diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000000..29288c6efe6 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,60 @@ +# SALTSTACK CODE OWNERS + +# See https://help.github.com/articles/about-codeowners/ +# for more info about CODEOWNERS file + +# Lines starting with '#' are comments. +# Each line is a file pattern followed by one or more owners. + +# See https://help.github.com/articles/about-codeowners/ +# for more info about the CODEOWNERS file + +# Team Boto +salt/**/*boto* @saltstack/team-boto + +# Team Core +salt/auth/ @saltstack/team-core +salt/cache/ @saltstack/team-core +salt/cli/ @saltstack/team-core +salt/client/* @saltstack/team-core +salt/config/* @saltstack/team-core +salt/daemons/ @saltstack/team-core +salt/pillar/ @saltstack/team-core +salt/loader.py @saltstack/team-core +salt/payload.py @saltstack/team-core +salt/**/master* @saltstack/team-core +salt/**/minion* @saltstack/team-core + +# Team Cloud +salt/cloud/ @saltstack/team-cloud +salt/utils/openstack/ @saltstack/team-cloud +salt/utils/aws.py @saltstack/team-cloud +salt/**/*cloud* @saltstack/team-cloud + +# Team NetAPI +salt/cli/api.py @saltstack/team-netapi +salt/client/netapi.py @saltstack/team-netapi +salt/netapi/ @saltstack/team-netapi + +# Team Network +salt/proxy/ @saltstack/team-proxy + +# Team SPM +salt/cli/spm.py @saltstack/team-spm +salt/spm/ @saltstack/team-spm + +# Team SSH +salt/cli/ssh.py @saltstack/team-ssh +salt/client/ssh/ @saltstack/team-ssh +salt/runners/ssh.py @saltstack/team-ssh +salt/**/thin.py @saltstack/team-ssh + +# Team State +salt/state.py @saltstack/team-state + +# Team Transport +salt/transport/ @saltstack/team-transport +salt/utils/zeromq.py @saltstack/team-transport + +# Team Windows +salt/**/*win* @saltstack/team-windows diff --git a/.github/stale.yml b/.github/stale.yml index 5b1a2c69e30..35928803a7f 100644 --- a/.github/stale.yml +++ b/.github/stale.yml @@ -1,8 +1,8 @@ # Probot Stale configuration file # Number of days of inactivity before an issue becomes stale -# 1200 is approximately 3 years and 3 months -daysUntilStale: 1200 +# 1000 is approximately 2 years and 9 months +daysUntilStale: 1000 # Number of days of inactivity before a stale issue is closed daysUntilClose: 7 diff --git a/.mention-bot b/.mention-bot index 56be9ab9e69..c07f85b9fc8 100644 --- a/.mention-bot +++ b/.mention-bot @@ -1,5 +1,17 @@ { + "alwaysNotifyForPaths": [ + { + "name": "ryan-lane", + "files": ["salt/**/*boto*.py"], + "skipTeamPrs": false + }, + { + "name": "tkwilliams", + "files": ["salt/**/*boto*.py"], + "skipTeamPrs": false + } + ], "skipTitle": "Merge forward", - "userBlacklist": ["cvrebert", "markusgattol", "olliewalsh"] + "userBlacklist": ["cvrebert", "markusgattol", "olliewalsh", "basepi"] } diff --git a/README.rst b/README.rst index d3917d6abe4..64b6fe1c5b2 100644 --- a/README.rst +++ b/README.rst @@ -67,10 +67,11 @@ Engage SaltStack `SaltConf`_, **User Groups and Meetups** - SaltStack has a vibrant and `global community`_ of customers, users, developers and enthusiasts. Connect with other -Salted folks in your area of the world, or join `SaltConf16`_, the SaltStack -annual user conference, April 19-21 in Salt Lake City. Please let us know if -you would like to start a user group or if we should add your existing -SaltStack user group to this list by emailing: info@saltstack.com +Salted folks in your area of the world, or join `SaltConf`_, the SaltStack +annual user conference held in Salt Lake City. Please visit the `SaltConf`_ site +for details of our next conference. Also, please let us know if you would like +to start a user group or if we should add your existing SaltStack user group to +this list by emailing: info@saltstack.com **SaltStack Training** - Get access to proprietary `SaltStack education offerings`_ through instructor-led training offered on-site, virtually or at @@ -89,9 +90,8 @@ services`_ offerings. * LinkedIn Group - ``_ * Google+ - ``_ -.. _SaltConf: http://www.youtube.com/user/saltstack .. _global community: http://www.meetup.com/pro/saltstack/ -.. _SaltConf16: http://saltconf.com/ +.. _SaltConf: http://saltconf.com/ .. _SaltStack education offerings: http://saltstack.com/training/ .. _SaltStack Certified Engineer (SSCE): http://saltstack.com/certification/ .. _SaltStack professional services: http://saltstack.com/services/ diff --git a/conf/cloud b/conf/cloud index 921cc048482..035cfea1014 100644 --- a/conf/cloud +++ b/conf/cloud @@ -97,3 +97,14 @@ # #delete_sshkeys: False +# Whether or not to include grains information in the /etc/salt/minion file +# which is generated when the minion is provisioned. For example... +# grains: +# salt-cloud: +# driver: ec2 +# provider: my_ec2:ec2 +# profile: micro_ec2 +# +# Default: 'True' +# +#enable_cloud_grains: 'True' diff --git a/conf/cloud.providers b/conf/cloud.providers index b4879432c48..8e9cc2ccf79 100644 --- a/conf/cloud.providers +++ b/conf/cloud.providers @@ -3,7 +3,7 @@ # directory is identical. #my-digitalocean-config: -# driver: digital_ocean +# driver: digitalocean # client_key: wFGEwgregeqw3435gDger # api_key: GDE43t43REGTrkilg43934t34qT43t4dgegerGEgg # location: New York 1 diff --git a/conf/cloud.providers.d/digitalocean.conf b/conf/cloud.providers.d/digitalocean.conf index 989758f1847..da3c13b45da 100644 --- a/conf/cloud.providers.d/digitalocean.conf +++ b/conf/cloud.providers.d/digitalocean.conf @@ -1,5 +1,5 @@ #my-digitalocean-config: -# driver: digital_ocean +# driver: digitalocean # client_key: wFGEwgregeqw3435gDger # api_key: GDE43t43REGTrkilg43934t34qT43t4dgegerGEgg # location: New York 1 diff --git a/conf/master b/conf/master index 50f27062d1f..08accd85cb4 100644 --- a/conf/master +++ b/conf/master @@ -59,15 +59,14 @@ # Directory for custom modules. This directory can contain subdirectories for # each of Salt's module types such as "runners", "output", "wheel", "modules", -# "states", "returners", etc. -#extension_modules: +# "states", "returners", "engines", "utils", etc. +#extension_modules: /var/cache/salt/master/extmods # Directory for custom modules. This directory can contain subdirectories for # each of Salt's module types such as "runners", "output", "wheel", "modules", -# "states", "returners", "engines", etc. +# "states", "returners", "engines", "utils", etc. # Like 'extension_modules' but can take an array of paths -#module_dirs: -# - /var/cache/salt/minion/extmods +#module_dirs: [] # Verify and set permissions on configuration directories at startup: #verify_env: True @@ -91,6 +90,10 @@ # Set the default outputter used by the salt command. The default is "nested". #output: nested +# To set a list of additional directories to search for salt outputters, set the +# outputter_dirs option. +#outputter_dirs: [] + # Set the default output file used by the salt command. Default is to output # to the CLI and not to a file. Functions the same way as the "--out-file" # CLI option, only sets this to a single file for all salt commands. @@ -99,6 +102,9 @@ # Return minions that timeout when running commands like test.ping #show_timeout: True +# Tell the client to display the jid when a job is published. +#show_jid: False + # By default, output is colored. To disable colored output, set the color value # to False. #color: True @@ -294,6 +300,22 @@ ##### Security settings ##### ########################################## +# Enable passphrase protection of Master private key. Although a string value +# is acceptable; passwords should be stored in an external vaulting mechanism +# and retrieved via sdb. See https://docs.saltstack.com/en/latest/topics/sdb/. +# Passphrase protection is off by default but an example of an sdb profile and +# query is as follows. +# masterkeyring: +# driver: keyring +# service: system +# +# key_pass: sdb://masterkeyring/key_pass + +# Enable passphrase protection of the Master signing_key. This only applies if +# master_sign_pubkey is set to True. This is disabled by default. +# master_sign_pubkey: True +# signing_key_pass: sdb://masterkeyring/signing_pass + # Enable "open mode", this mode still maintains encryption, but turns off # authentication, this is only intended for highly secure environments or for # the situation where your keys end up in a bad state. If you run in open mode @@ -304,6 +326,9 @@ # public keys from the minions. Note that this is insecure. #auto_accept: False +# The size of key that should be generated when creating new keys. +#keysize: 2048 + # Time in minutes that an incoming public key with a matching name found in # pki_dir/minion_autosign/keyid is automatically accepted. Expired autosign keys # are removed when the master checks the minion_autosign directory. @@ -454,6 +479,27 @@ # - /etc/salt/roster.d # - /opt/salt/some/more/rosters +# The ssh password to log in with. +#ssh_passwd: '' + +#The target system's ssh port number. +#ssh_port: 22 + +# Comma-separated list of ports to scan. +#ssh_scan_ports: 22 + +# Scanning socket timeout for salt-ssh. +#ssh_scan_timeout: 0.01 + +# Boolean to run command via sudo. +#ssh_sudo: False + +# Number of seconds to wait for a response when establishing an SSH connection. +#ssh_timeout: 60 + +# The user to log in as. +#ssh_user: root + # The log file of the salt-ssh command: #ssh_log_file: /var/log/salt/ssh @@ -467,6 +513,18 @@ # authentication with minions #ssh_use_home_key: False +# Set this to True to default salt-ssh to run with ``-o IdentitiesOnly=yes``. +# This option is intended for situations where the ssh-agent offers many +# different identities and allows ssh to ignore those identities and use the +# only one specified in options. +#ssh_identities_only: False + +# List-only nodegroups for salt-ssh. Each group must be formed as either a +# comma-separated list, or a YAML list. This option is useful to group minions +# into easy-to-target groups when using salt-ssh. These groups can then be +# targeted with the normal -N argument to salt-ssh. +#ssh_list_nodegroups: {} + ##### Master Module Management ##### ########################################## # Manage how master side modules are loaded. @@ -474,6 +532,9 @@ # Add any additional locations to look for master runners: #runner_dirs: [] +# Add any additional locations to look for master utils: +#utils_dirs: [] + # Enable Cython for master side modules: #cython_enable: False @@ -535,6 +596,11 @@ # If set to 'changes', the output will be full unless the state didn't change. #state_output: full +# The state_output_diff setting changes whether or not the output from +# successful states is returned. Useful when even the terse output of these +# states is cluttering the logs. Set it to True to ignore them. +#state_output_diff: False + # Automatically aggregate all states that have support for mod_aggregate by # setting to 'True'. Or pass a list of state module names to automatically # aggregate just those types. @@ -575,6 +641,10 @@ # - /srv/salt # +# The master_roots setting configures a master-only copy of the file_roots dictionary, +# used by the state compiler. +#master_roots: /srv/salt-master + # When using multiple environments, each with their own top file, the # default behaviour is an unordered merge. To prevent top files from # being merged together and instead to only use the top file from the @@ -919,6 +989,21 @@ #pillar_cache_backend: disk +###### Reactor Settings ##### +########################################### +# Define a salt reactor. See https://docs.saltstack.com/en/latest/topics/reactor/ +#reactor: [] + +#Set the TTL for the cache of the reactor configuration. +#reactor_refresh_interval: 60 + +#Configure the number of workers for the runner/wheel in the reactor. +#reactor_worker_threads: 10 + +#Define the queue size for workers in the reactor. +#reactor_worker_hwm: 10000 + + ##### Syndic settings ##### ########################################## # The Salt syndic is used to pass commands through a master from a higher diff --git a/conf/minion b/conf/minion index 1efea6fc91d..fa5caf317b9 100644 --- a/conf/minion +++ b/conf/minion @@ -151,7 +151,11 @@ # Set the default outputter used by the salt-call command. The default is # "nested". #output: nested -# + +# To set a list of additional directories to search for salt outputters, set the +# outputter_dirs option. +#outputter_dirs: [] + # By default output is colored. To disable colored output, set the color value # to False. #color: True @@ -231,7 +235,7 @@ # cause sub minion process to restart. #auth_safemode: False -# Ping Master to ensure connection is alive (minutes). +# Ping Master to ensure connection is alive (seconds). #ping_interval: 0 # To auto recover minions if master changes IP address (DDNS) @@ -369,6 +373,9 @@ # interface: eth0 # cidr: '10.0.0.0/8' +# The number of minutes between mine updates. +#mine_interval: 60 + # Windows platforms lack posix IPC and must rely on slower TCP based inter- # process communications. Set ipc_mode to 'tcp' on such systems #ipc_mode: ipc @@ -613,6 +620,9 @@ # you do so at your own risk! #open_mode: False +# The size of key that should be generated when creating new keys. +#keysize: 2048 + # Enable permissive access to the salt keys. This allows you to run the # master or minion as root, but have a non-root group be given access to # your pki_dir. To make the access explicit, root must belong to the group @@ -654,6 +664,21 @@ # ssl_version: PROTOCOL_TLSv1_2 +###### Reactor Settings ##### +########################################### +# Define a salt reactor. See https://docs.saltstack.com/en/latest/topics/reactor/ +#reactor: [] + +#Set the TTL for the cache of the reactor configuration. +#reactor_refresh_interval: 60 + +#Configure the number of workers for the runner/wheel in the reactor. +#reactor_worker_threads: 10 + +#Define the queue size for workers in the reactor. +#reactor_worker_hwm: 10000 + + ###### Thread settings ##### ########################################### # Disable multiprocessing support, by default when a minion receives a diff --git a/conf/suse/master b/conf/suse/master new file mode 100644 index 00000000000..aeaa1d88591 --- /dev/null +++ b/conf/suse/master @@ -0,0 +1,1230 @@ +##### Primary configuration settings ##### +########################################## +# This configuration file is used to manage the behavior of the Salt Master. +# Values that are commented out but have an empty line after the comment are +# defaults that do not need to be set in the config. If there is no blank line +# after the comment then the value is presented as an example and is not the +# default. + +# Per default, the master will automatically include all config files +# from master.d/*.conf (master.d is a directory in the same directory +# as the main master config file). +#default_include: master.d/*.conf + +# The address of the interface to bind to: +#interface: 0.0.0.0 + +# Whether the master should listen for IPv6 connections. If this is set to True, +# the interface option must be adjusted, too. (For example: "interface: '::'") +#ipv6: False + +# The tcp port used by the publisher: +#publish_port: 4505 + +# The user under which the salt master will run. Salt will update all +# permissions to allow the specified user to run the master. The exception is +# the job cache, which must be deleted if this user is changed. If the +# modified files cause conflicts, set verify_env to False. +user: salt +syndic_user: salt + +# The port used by the communication interface. The ret (return) port is the +# interface used for the file server, authentication, job returns, etc. +#ret_port: 4506 + +# Specify the location of the daemon process ID file: +#pidfile: /var/run/salt-master.pid + +# The root directory prepended to these options: pki_dir, cachedir, +# sock_dir, log_file, autosign_file, autoreject_file, extension_modules, +# key_logfile, pidfile: +#root_dir: / + +# The path to the master's configuration file. +#conf_file: /etc/salt/master + +# Directory used to store public key data: +#pki_dir: /etc/salt/pki/master + +# Key cache. Increases master speed for large numbers of accepted +# keys. Available options: 'sched'. (Updates on a fixed schedule.) +# Note that enabling this feature means that minions will not be +# available to target for up to the length of the maintanence loop +# which by default is 60s. +#key_cache: '' + +# Directory to store job and cache data: +# This directory may contain sensitive data and should be protected accordingly. +# +#cachedir: /var/cache/salt/master + +# Directory for custom modules. This directory can contain subdirectories for +# each of Salt's module types such as "runners", "output", "wheel", "modules", +# "states", "returners", etc. +#extension_modules: + +# Directory for custom modules. This directory can contain subdirectories for +# each of Salt's module types such as "runners", "output", "wheel", "modules", +# "states", "returners", "engines", etc. +# Like 'extension_modules' but can take an array of paths +#module_dirs: +# - /var/cache/salt/minion/extmods + +# Verify and set permissions on configuration directories at startup: +#verify_env: True + +# Set the number of hours to keep old job information in the job cache: +#keep_jobs: 24 + +# The number of seconds to wait when the client is requesting information +# about running jobs. +#gather_job_timeout: 10 + +# Set the default timeout for the salt command and api. The default is 5 +# seconds. +#timeout: 5 + +# The loop_interval option controls the seconds for the master's maintenance +# process check cycle. This process updates file server backends, cleans the +# job cache and executes the scheduler. +#loop_interval: 60 + +# Set the default outputter used by the salt command. The default is "nested". +#output: nested + +# To set a list of additional directories to search for salt outputters, set the +# outputter_dirs option. +#outputter_dirs: [] + +# Set the default output file used by the salt command. Default is to output +# to the CLI and not to a file. Functions the same way as the "--out-file" +# CLI option, only sets this to a single file for all salt commands. +#output_file: None + +# Return minions that timeout when running commands like test.ping +#show_timeout: True + +# Tell the client to display the jid when a job is published. +#show_jid: False + +# By default, output is colored. To disable colored output, set the color value +# to False. +#color: True + +# Do not strip off the colored output from nested results and state outputs +# (true by default). +# strip_colors: False + +# To display a summary of the number of minions targeted, the number of +# minions returned, and the number of minions that did not return, set the +# cli_summary value to True. (False by default.) +# +#cli_summary: False + +# Set the directory used to hold unix sockets: +#sock_dir: /var/run/salt/master + +# The master can take a while to start up when lspci and/or dmidecode is used +# to populate the grains for the master. Enable if you want to see GPU hardware +# data for your master. +# enable_gpu_grains: False + +# The master maintains a job cache. While this is a great addition, it can be +# a burden on the master for larger deployments (over 5000 minions). +# Disabling the job cache will make previously executed jobs unavailable to +# the jobs system and is not generally recommended. +#job_cache: True + +# Cache minion grains, pillar and mine data via the cache subsystem in the +# cachedir or a database. +#minion_data_cache: True + +# Cache subsystem module to use for minion data cache. +#cache: localfs +# Enables a fast in-memory cache booster and sets the expiration time. +#memcache_expire_seconds: 0 +# Set a memcache limit in items (bank + key) per cache storage (driver + driver_opts). +#memcache_max_items: 1024 +# Each time a cache storage got full cleanup all the expired items not just the oldest one. +#memcache_full_cleanup: False +# Enable collecting the memcache stats and log it on `debug` log level. +#memcache_debug: False + +# Store all returns in the given returner. +# Setting this option requires that any returner-specific configuration also +# be set. See various returners in salt/returners for details on required +# configuration values. (See also, event_return_queue below.) +# +#event_return: mysql + +# On busy systems, enabling event_returns can cause a considerable load on +# the storage system for returners. Events can be queued on the master and +# stored in a batched fashion using a single transaction for multiple events. +# By default, events are not queued. +#event_return_queue: 0 + +# Only return events matching tags in a whitelist, supports glob matches. +#event_return_whitelist: +# - salt/master/a_tag +# - salt/run/*/ret + +# Store all event returns **except** the tags in a blacklist, supports globs. +#event_return_blacklist: +# - salt/master/not_this_tag +# - salt/wheel/*/ret + +# Passing very large events can cause the minion to consume large amounts of +# memory. This value tunes the maximum size of a message allowed onto the +# master event bus. The value is expressed in bytes. +#max_event_size: 1048576 + +# By default, the master AES key rotates every 24 hours. The next command +# following a key rotation will trigger a key refresh from the minion which may +# result in minions which do not respond to the first command after a key refresh. +# +# To tell the master to ping all minions immediately after an AES key refresh, set +# ping_on_rotate to True. This should mitigate the issue where a minion does not +# appear to initially respond after a key is rotated. +# +# Note that ping_on_rotate may cause high load on the master immediately after +# the key rotation event as minions reconnect. Consider this carefully if this +# salt master is managing a large number of minions. +# +# If disabled, it is recommended to handle this event by listening for the +# 'aes_key_rotate' event with the 'key' tag and acting appropriately. +# ping_on_rotate: False + +# By default, the master deletes its cache of minion data when the key for that +# minion is removed. To preserve the cache after key deletion, set +# 'preserve_minion_cache' to True. +# +# WARNING: This may have security implications if compromised minions auth with +# a previous deleted minion ID. +#preserve_minion_cache: False + +# Allow or deny minions from requesting their own key revocation +#allow_minion_key_revoke: True + +# If max_minions is used in large installations, the master might experience +# high-load situations because of having to check the number of connected +# minions for every authentication. This cache provides the minion-ids of +# all connected minions to all MWorker-processes and greatly improves the +# performance of max_minions. +# con_cache: False + +# The master can include configuration from other files. To enable this, +# pass a list of paths to this option. The paths can be either relative or +# absolute; if relative, they are considered to be relative to the directory +# the main master configuration file lives in (this file). Paths can make use +# of shell-style globbing. If no files are matched by a path passed to this +# option, then the master will log a warning message. +# +# Include a config file from some other path: +# include: /etc/salt/extra_config +# +# Include config from several files and directories: +# include: +# - /etc/salt/extra_config + + +##### Large-scale tuning settings ##### +########################################## +# Max open files +# +# Each minion connecting to the master uses AT LEAST one file descriptor, the +# master subscription connection. If enough minions connect you might start +# seeing on the console (and then salt-master crashes): +# Too many open files (tcp_listener.cpp:335) +# Aborted (core dumped) +# +# By default this value will be the one of `ulimit -Hn`, ie, the hard limit for +# max open files. +# +# If you wish to set a different value than the default one, uncomment and +# configure this setting. Remember that this value CANNOT be higher than the +# hard limit. Raising the hard limit depends on your OS and/or distribution, +# a good way to find the limit is to search the internet. For example: +# raise max open files hard limit debian +# +#max_open_files: 100000 + +# The number of worker threads to start. These threads are used to manage +# return calls made from minions to the master. If the master seems to be +# running slowly, increase the number of threads. This setting can not be +# set lower than 3. +#worker_threads: 5 + +# Set the ZeroMQ high water marks +# http://api.zeromq.org/3-2:zmq-setsockopt + +# The listen queue size / backlog +#zmq_backlog: 1000 + +# The publisher interface ZeroMQPubServerChannel +#pub_hwm: 1000 + +# These two ZMQ HWM settings, salt_event_pub_hwm and event_publisher_pub_hwm +# are significant for masters with thousands of minions. When these are +# insufficiently high it will manifest in random responses missing in the CLI +# and even missing from the job cache. Masters that have fast CPUs and many +# cores with appropriate worker_threads will not need these set as high. + +# On deployment with 8,000 minions, 2.4GHz CPUs, 24 cores, 32GiB memory has +# these settings: +# +# salt_event_pub_hwm: 128000 +# event_publisher_pub_hwm: 64000 + +# ZMQ high-water-mark for SaltEvent pub socket +#salt_event_pub_hwm: 20000 + +# ZMQ high-water-mark for EventPublisher pub socket +#event_publisher_pub_hwm: 10000 + +# The master may allocate memory per-event and not +# reclaim it. +# To set a high-water mark for memory allocation, use +# ipc_write_buffer to set a high-water mark for message +# buffering. +# Value: In bytes. Set to 'dynamic' to have Salt select +# a value for you. Default is disabled. +# ipc_write_buffer: 'dynamic' + + +##### Security settings ##### +########################################## +# Enable "open mode", this mode still maintains encryption, but turns off +# authentication, this is only intended for highly secure environments or for +# the situation where your keys end up in a bad state. If you run in open mode +# you do so at your own risk! +#open_mode: False + +# Enable auto_accept, this setting will automatically accept all incoming +# public keys from the minions. Note that this is insecure. +#auto_accept: False + +# Time in minutes that an incoming public key with a matching name found in +# pki_dir/minion_autosign/keyid is automatically accepted. Expired autosign keys +# are removed when the master checks the minion_autosign directory. +# 0 equals no timeout +# autosign_timeout: 120 + +# If the autosign_file is specified, incoming keys specified in the +# autosign_file will be automatically accepted. This is insecure. Regular +# expressions as well as globing lines are supported. +#autosign_file: /etc/salt/autosign.conf + +# Works like autosign_file, but instead allows you to specify minion IDs for +# which keys will automatically be rejected. Will override both membership in +# the autosign_file and the auto_accept setting. +#autoreject_file: /etc/salt/autoreject.conf + +# Enable permissive access to the salt keys. This allows you to run the +# master or minion as root, but have a non-root group be given access to +# your pki_dir. To make the access explicit, root must belong to the group +# you've given access to. This is potentially quite insecure. If an autosign_file +# is specified, enabling permissive_pki_access will allow group access to that +# specific file. +#permissive_pki_access: False + +# Allow users on the master access to execute specific commands on minions. +# This setting should be treated with care since it opens up execution +# capabilities to non root users. By default this capability is completely +# disabled. +#publisher_acl: +# larry: +# - test.ping +# - network.* +# +# Blacklist any of the following users or modules +# +# This example would blacklist all non sudo users, including root from +# running any commands. It would also blacklist any use of the "cmd" +# module. This is completely disabled by default. +# +# +# Check the list of configured users in client ACL against users on the +# system and throw errors if they do not exist. +#client_acl_verify: True +# +#publisher_acl_blacklist: +# users: +# - root +# - '^(?!sudo_).*$' # all non sudo users +# modules: +# - cmd + +# Enforce publisher_acl & publisher_acl_blacklist when users have sudo +# access to the salt command. +# +#sudo_acl: False + +# The external auth system uses the Salt auth modules to authenticate and +# validate users to access areas of the Salt system. +#external_auth: +# pam: +# fred: +# - test.* +# +# Time (in seconds) for a newly generated token to live. Default: 12 hours +#token_expire: 43200 +# +# Allow eauth users to specify the expiry time of the tokens they generate. +# A boolean applies to all users or a dictionary of whitelisted eauth backends +# and usernames may be given. +# token_expire_user_override: +# pam: +# - fred +# - tom +# ldap: +# - gary +# +#token_expire_user_override: False + +# Set to True to enable keeping the calculated user's auth list in the token +# file. This is disabled by default and the auth list is calculated or requested +# from the eauth driver each time. +#keep_acl_in_token: False + +# Auth subsystem module to use to get authorized access list for a user. By default it's +# the same module used for external authentication. +#eauth_acl_module: django + +# Allow minions to push files to the master. This is disabled by default, for +# security purposes. +#file_recv: False + +# Set a hard-limit on the size of the files that can be pushed to the master. +# It will be interpreted as megabytes. Default: 100 +#file_recv_max_size: 100 + +# Signature verification on messages published from the master. +# This causes the master to cryptographically sign all messages published to its event +# bus, and minions then verify that signature before acting on the message. +# +# This is False by default. +# +# Note that to facilitate interoperability with masters and minions that are different +# versions, if sign_pub_messages is True but a message is received by a minion with +# no signature, it will still be accepted, and a warning message will be logged. +# Conversely, if sign_pub_messages is False, but a minion receives a signed +# message it will be accepted, the signature will not be checked, and a warning message +# will be logged. This behavior went away in Salt 2014.1.0 and these two situations +# will cause minion to throw an exception and drop the message. +# sign_pub_messages: False + +# Signature verification on messages published from minions +# This requires that minions cryptographically sign the messages they +# publish to the master. If minions are not signing, then log this information +# at loglevel 'INFO' and drop the message without acting on it. +# require_minion_sign_messages: False + +# The below will drop messages when their signatures do not validate. +# Note that when this option is False but `require_minion_sign_messages` is True +# minions MUST sign their messages but the validity of their signatures +# is ignored. +# These two config options exist so a Salt infrastructure can be moved +# to signing minion messages gradually. +# drop_messages_signature_fail: False + +# Use TLS/SSL encrypted connection between master and minion. +# Can be set to a dictionary containing keyword arguments corresponding to Python's +# 'ssl.wrap_socket' method. +# Default is None. +#ssl: +# keyfile: +# certfile: +# ssl_version: PROTOCOL_TLSv1_2 + +##### Salt-SSH Configuration ##### +########################################## + +# Pass in an alternative location for the salt-ssh roster file +#roster_file: /etc/salt/roster + +# Define locations for roster files so they can be chosen when using Salt API. +# An administrator can place roster files into these locations. Then when +# calling Salt API, parameter 'roster_file' should contain a relative path to +# these locations. That is, "roster_file=/foo/roster" will be resolved as +# "/etc/salt/roster.d/foo/roster" etc. This feature prevents passing insecure +# custom rosters through the Salt API. +# +#rosters: +# - /etc/salt/roster.d +# - /opt/salt/some/more/rosters + +# The ssh password to log in with. +#ssh_passwd: '' + +#The target system's ssh port number. +#ssh_port: 22 + +# Comma-separated list of ports to scan. +#ssh_scan_ports: 22 + +# Scanning socket timeout for salt-ssh. +#ssh_scan_timeout: 0.01 + +# Boolean to run command via sudo. +#ssh_sudo: False + +# Number of seconds to wait for a response when establishing an SSH connection. +#ssh_timeout: 60 + +# The user to log in as. +#ssh_user: root + +# The log file of the salt-ssh command: +#ssh_log_file: /var/log/salt/ssh + +# Pass in minion option overrides that will be inserted into the SHIM for +# salt-ssh calls. The local minion config is not used for salt-ssh. Can be +# overridden on a per-minion basis in the roster (`minion_opts`) +#ssh_minion_opts: +# gpg_keydir: /root/gpg + +# Set this to True to default to using ~/.ssh/id_rsa for salt-ssh +# authentication with minions +#ssh_use_home_key: False + +# Set this to True to default salt-ssh to run with ``-o IdentitiesOnly=yes``. +# This option is intended for situations where the ssh-agent offers many +# different identities and allows ssh to ignore those identities and use the +# only one specified in options. +#ssh_identities_only: False + +# List-only nodegroups for salt-ssh. Each group must be formed as either a +# comma-separated list, or a YAML list. This option is useful to group minions +# into easy-to-target groups when using salt-ssh. These groups can then be +# targeted with the normal -N argument to salt-ssh. +#ssh_list_nodegroups: {} + +##### Master Module Management ##### +########################################## +# Manage how master side modules are loaded. + +# Add any additional locations to look for master runners: +#runner_dirs: [] + +# Enable Cython for master side modules: +#cython_enable: False + + +##### State System settings ##### +########################################## +# The state system uses a "top" file to tell the minions what environment to +# use and what modules to use. The state_top file is defined relative to the +# root of the base environment as defined in "File Server settings" below. +#state_top: top.sls + +# The master_tops option replaces the external_nodes option by creating +# a plugable system for the generation of external top data. The external_nodes +# option is deprecated by the master_tops option. +# +# To gain the capabilities of the classic external_nodes system, use the +# following configuration: +# master_tops: +# ext_nodes: +# +#master_tops: {} + +# The external_nodes option allows Salt to gather data that would normally be +# placed in a top file. The external_nodes option is the executable that will +# return the ENC data. Remember that Salt will look for external nodes AND top +# files and combine the results if both are enabled! +#external_nodes: None + +# The renderer to use on the minions to render the state data +#renderer: yaml_jinja + +# The Jinja renderer can strip extra carriage returns and whitespace +# See http://jinja.pocoo.org/docs/api/#high-level-api +# +# If this is set to True the first newline after a Jinja block is removed +# (block, not variable tag!). Defaults to False, corresponds to the Jinja +# environment init variable "trim_blocks". +#jinja_trim_blocks: False +# +# If this is set to True leading spaces and tabs are stripped from the start +# of a line to a block. Defaults to False, corresponds to the Jinja +# environment init variable "lstrip_blocks". +#jinja_lstrip_blocks: False + +# The failhard option tells the minions to stop immediately after the first +# failure detected in the state execution, defaults to False +#failhard: False + +# The state_verbose and state_output settings can be used to change the way +# state system data is printed to the display. By default all data is printed. +# The state_verbose setting can be set to True or False, when set to False +# all data that has a result of True and no changes will be suppressed. +#state_verbose: True + +# The state_output setting changes if the output is the full multi line +# output for each changed state if set to 'full', but if set to 'terse' +# the output will be shortened to a single line. If set to 'mixed', the output +# will be terse unless a state failed, in which case that output will be full. +# If set to 'changes', the output will be full unless the state didn't change. +#state_output: full + +# The state_output_diff setting changes whether or not the output from +# successful states is returned. Useful when even the terse output of these +# states is cluttering the logs. Set it to True to ignore them. +#state_output_diff: False + +# Automatically aggregate all states that have support for mod_aggregate by +# setting to 'True'. Or pass a list of state module names to automatically +# aggregate just those types. +# +# state_aggregate: +# - pkg +# +#state_aggregate: False + +# Send progress events as each function in a state run completes execution +# by setting to 'True'. Progress events are in the format +# 'salt/job//prog//'. +#state_events: False + +##### File Server settings ##### +########################################## +# Salt runs a lightweight file server written in zeromq to deliver files to +# minions. This file server is built into the master daemon and does not +# require a dedicated port. + +# The file server works on environments passed to the master, each environment +# can have multiple root directories, the subdirectories in the multiple file +# roots cannot match, otherwise the downloaded files will not be able to be +# reliably ensured. A base environment is required to house the top file. +# Example: +# file_roots: +# base: +# - /srv/salt/ +# dev: +# - /srv/salt/dev/services +# - /srv/salt/dev/states +# prod: +# - /srv/salt/prod/services +# - /srv/salt/prod/states +# +#file_roots: +# base: +# - /srv/salt +# + +# The master_roots setting configures a master-only copy of the file_roots dictionary, +# used by the state compiler. +#master_roots: /srv/salt-master + +# When using multiple environments, each with their own top file, the +# default behaviour is an unordered merge. To prevent top files from +# being merged together and instead to only use the top file from the +# requested environment, set this value to 'same'. +#top_file_merging_strategy: merge + +# To specify the order in which environments are merged, set the ordering +# in the env_order option. Given a conflict, the last matching value will +# win. +#env_order: ['base', 'dev', 'prod'] + +# If top_file_merging_strategy is set to 'same' and an environment does not +# contain a top file, the top file in the environment specified by default_top +# will be used instead. +#default_top: base + +# The hash_type is the hash to use when discovering the hash of a file on +# the master server. The default is sha256, but md5, sha1, sha224, sha384 and +# sha512 are also supported. +# +# WARNING: While md5 and sha1 are also supported, do not use them due to the +# high chance of possible collisions and thus security breach. +# +# Prior to changing this value, the master should be stopped and all Salt +# caches should be cleared. +#hash_type: sha256 + +# The buffer size in the file server can be adjusted here: +#file_buffer_size: 1048576 + +# A regular expression (or a list of expressions) that will be matched +# against the file path before syncing the modules and states to the minions. +# This includes files affected by the file.recurse state. +# For example, if you manage your custom modules and states in subversion +# and don't want all the '.svn' folders and content synced to your minions, +# you could set this to '/\.svn($|/)'. By default nothing is ignored. +#file_ignore_regex: +# - '/\.svn($|/)' +# - '/\.git($|/)' + +# A file glob (or list of file globs) that will be matched against the file +# path before syncing the modules and states to the minions. This is similar +# to file_ignore_regex above, but works on globs instead of regex. By default +# nothing is ignored. +# file_ignore_glob: +# - '*.pyc' +# - '*/somefolder/*.bak' +# - '*.swp' + +# File Server Backend +# +# Salt supports a modular fileserver backend system, this system allows +# the salt master to link directly to third party systems to gather and +# manage the files available to minions. Multiple backends can be +# configured and will be searched for the requested file in the order in which +# they are defined here. The default setting only enables the standard backend +# "roots" which uses the "file_roots" option. +#fileserver_backend: +# - roots +# +# To use multiple backends list them in the order they are searched: +#fileserver_backend: +# - git +# - roots +# +# Uncomment the line below if you do not want the file_server to follow +# symlinks when walking the filesystem tree. This is set to True +# by default. Currently this only applies to the default roots +# fileserver_backend. +#fileserver_followsymlinks: False +# +# Uncomment the line below if you do not want symlinks to be +# treated as the files they are pointing to. By default this is set to +# False. By uncommenting the line below, any detected symlink while listing +# files on the Master will not be returned to the Minion. +#fileserver_ignoresymlinks: True +# +# By default, the Salt fileserver recurses fully into all defined environments +# to attempt to find files. To limit this behavior so that the fileserver only +# traverses directories with SLS files and special Salt directories like _modules, +# enable the option below. This might be useful for installations where a file root +# has a very large number of files and performance is impacted. Default is False. +# fileserver_limit_traversal: False +# +# The fileserver can fire events off every time the fileserver is updated, +# these are disabled by default, but can be easily turned on by setting this +# flag to True +#fileserver_events: False + +# Git File Server Backend Configuration +# +# Optional parameter used to specify the provider to be used for gitfs. Must be +# either pygit2 or gitpython. If unset, then both will be tried (in that +# order), and the first one with a compatible version installed will be the +# provider that is used. +# +#gitfs_provider: pygit2 + +# Along with gitfs_password, is used to authenticate to HTTPS remotes. +# gitfs_user: '' + +# Along with gitfs_user, is used to authenticate to HTTPS remotes. +# This parameter is not required if the repository does not use authentication. +#gitfs_password: '' + +# By default, Salt will not authenticate to an HTTP (non-HTTPS) remote. +# This parameter enables authentication over HTTP. Enable this at your own risk. +#gitfs_insecure_auth: False + +# Along with gitfs_privkey (and optionally gitfs_passphrase), is used to +# authenticate to SSH remotes. This parameter (or its per-remote counterpart) +# is required for SSH remotes. +#gitfs_pubkey: '' + +# Along with gitfs_pubkey (and optionally gitfs_passphrase), is used to +# authenticate to SSH remotes. This parameter (or its per-remote counterpart) +# is required for SSH remotes. +#gitfs_privkey: '' + +# This parameter is optional, required only when the SSH key being used to +# authenticate is protected by a passphrase. +#gitfs_passphrase: '' + +# When using the git fileserver backend at least one git remote needs to be +# defined. The user running the salt master will need read access to the repo. +# +# The repos will be searched in order to find the file requested by a client +# and the first repo to have the file will return it. +# When using the git backend branches and tags are translated into salt +# environments. +# Note: file:// repos will be treated as a remote, so refs you want used must +# exist in that repo as *local* refs. +#gitfs_remotes: +# - git://github.com/saltstack/salt-states.git +# - file:///var/git/saltmaster +# +# The gitfs_ssl_verify option specifies whether to ignore ssl certificate +# errors when contacting the gitfs backend. You might want to set this to +# false if you're using a git backend that uses a self-signed certificate but +# keep in mind that setting this flag to anything other than the default of True +# is a security concern, you may want to try using the ssh transport. +#gitfs_ssl_verify: True +# +# The gitfs_root option gives the ability to serve files from a subdirectory +# within the repository. The path is defined relative to the root of the +# repository and defaults to the repository root. +#gitfs_root: somefolder/otherfolder +# +# The refspecs fetched by gitfs remotes +#gitfs_refspecs: +# - '+refs/heads/*:refs/remotes/origin/*' +# - '+refs/tags/*:refs/tags/*' +# +# +##### Pillar settings ##### +########################################## +# Salt Pillars allow for the building of global data that can be made selectively +# available to different minions based on minion grain filtering. The Salt +# Pillar is laid out in the same fashion as the file server, with environments, +# a top file and sls files. However, pillar data does not need to be in the +# highstate format, and is generally just key/value pairs. +#pillar_roots: +# base: +# - /srv/pillar +# +#ext_pillar: +# - hiera: /etc/hiera.yaml +# - cmd_yaml: cat /etc/salt/yaml + + +# A list of paths to be recursively decrypted during pillar compilation. +# Entries in this list can be formatted either as a simple string, or as a +# key/value pair, with the key being the pillar location, and the value being +# the renderer to use for pillar decryption. If the former is used, the +# renderer specified by decrypt_pillar_default will be used. +#decrypt_pillar: +# - 'foo:bar': gpg +# - 'lorem:ipsum:dolor' + +# The delimiter used to distinguish nested data structures in the +# decrypt_pillar option. +#decrypt_pillar_delimiter: ':' + +# The default renderer used for decryption, if one is not specified for a given +# pillar key in decrypt_pillar. +#decrypt_pillar_default: gpg + +# List of renderers which are permitted to be used for pillar decryption. +#decrypt_pillar_renderers: +# - gpg + +# The ext_pillar_first option allows for external pillar sources to populate +# before file system pillar. This allows for targeting file system pillar from +# ext_pillar. +#ext_pillar_first: False + +# The external pillars permitted to be used on-demand using pillar.ext +#on_demand_ext_pillar: +# - libvirt +# - virtkey + +# The pillar_gitfs_ssl_verify option specifies whether to ignore ssl certificate +# errors when contacting the pillar gitfs backend. You might want to set this to +# false if you're using a git backend that uses a self-signed certificate but +# keep in mind that setting this flag to anything other than the default of True +# is a security concern, you may want to try using the ssh transport. +#pillar_gitfs_ssl_verify: True + +# The pillar_opts option adds the master configuration file data to a dict in +# the pillar called "master". This is used to set simple configurations in the +# master config file that can then be used on minions. +#pillar_opts: False + +# The pillar_safe_render_error option prevents the master from passing pillar +# render errors to the minion. This is set on by default because the error could +# contain templating data which would give that minion information it shouldn't +# have, like a password! When set true the error message will only show: +# Rendering SLS 'my.sls' failed. Please see master log for details. +#pillar_safe_render_error: True + +# The pillar_source_merging_strategy option allows you to configure merging strategy +# between different sources. It accepts five values: none, recurse, aggregate, overwrite, +# or smart. None will not do any merging at all. Recurse will merge recursively mapping of data. +# Aggregate instructs aggregation of elements between sources that use the #!yamlex renderer. Overwrite +# will overwrite elements according the order in which they are processed. This is +# behavior of the 2014.1 branch and earlier. Smart guesses the best strategy based +# on the "renderer" setting and is the default value. +#pillar_source_merging_strategy: smart + +# Recursively merge lists by aggregating them instead of replacing them. +#pillar_merge_lists: False + +# Set this option to True to force the pillarenv to be the same as the effective +# saltenv when running states. If pillarenv is specified this option will be +# ignored. +#pillarenv_from_saltenv: False + +# Set this option to 'True' to force a 'KeyError' to be raised whenever an +# attempt to retrieve a named value from pillar fails. When this option is set +# to 'False', the failed attempt returns an empty string. Default is 'False'. +#pillar_raise_on_missing: False + +# Git External Pillar (git_pillar) Configuration Options +# +# Specify the provider to be used for git_pillar. Must be either pygit2 or +# gitpython. If unset, then both will be tried in that same order, and the +# first one with a compatible version installed will be the provider that +# is used. +#git_pillar_provider: pygit2 + +# If the desired branch matches this value, and the environment is omitted +# from the git_pillar configuration, then the environment for that git_pillar +# remote will be base. +#git_pillar_base: master + +# If the branch is omitted from a git_pillar remote, then this branch will +# be used instead +#git_pillar_branch: master + +# Environment to use for git_pillar remotes. This is normally derived from +# the branch/tag (or from a per-remote env parameter), but if set this will +# override the process of deriving the env from the branch/tag name. +#git_pillar_env: '' + +# Path relative to the root of the repository where the git_pillar top file +# and SLS files are located. +#git_pillar_root: '' + +# Specifies whether or not to ignore SSL certificate errors when contacting +# the remote repository. +#git_pillar_ssl_verify: False + +# When set to False, if there is an update/checkout lock for a git_pillar +# remote and the pid written to it is not running on the master, the lock +# file will be automatically cleared and a new lock will be obtained. +#git_pillar_global_lock: True + +# Git External Pillar Authentication Options +# +# Along with git_pillar_password, is used to authenticate to HTTPS remotes. +#git_pillar_user: '' + +# Along with git_pillar_user, is used to authenticate to HTTPS remotes. +# This parameter is not required if the repository does not use authentication. +#git_pillar_password: '' + +# By default, Salt will not authenticate to an HTTP (non-HTTPS) remote. +# This parameter enables authentication over HTTP. +#git_pillar_insecure_auth: False + +# Along with git_pillar_privkey (and optionally git_pillar_passphrase), +# is used to authenticate to SSH remotes. +#git_pillar_pubkey: '' + +# Along with git_pillar_pubkey (and optionally git_pillar_passphrase), +# is used to authenticate to SSH remotes. +#git_pillar_privkey: '' + +# This parameter is optional, required only when the SSH key being used +# to authenticate is protected by a passphrase. +#git_pillar_passphrase: '' + +# The refspecs fetched by git_pillar remotes +#git_pillar_refspecs: +# - '+refs/heads/*:refs/remotes/origin/*' +# - '+refs/tags/*:refs/tags/*' + +# A master can cache pillars locally to bypass the expense of having to render them +# for each minion on every request. This feature should only be enabled in cases +# where pillar rendering time is known to be unsatisfactory and any attendant security +# concerns about storing pillars in a master cache have been addressed. +# +# When enabling this feature, be certain to read through the additional ``pillar_cache_*`` +# configuration options to fully understand the tunable parameters and their implications. +# +# Note: setting ``pillar_cache: True`` has no effect on targeting Minions with Pillars. +# See https://docs.saltstack.com/en/latest/topics/targeting/pillar.html +#pillar_cache: False + +# If and only if a master has set ``pillar_cache: True``, the cache TTL controls the amount +# of time, in seconds, before the cache is considered invalid by a master and a fresh +# pillar is recompiled and stored. +#pillar_cache_ttl: 3600 + +# If and only if a master has set `pillar_cache: True`, one of several storage providers +# can be utililzed. +# +# `disk`: The default storage backend. This caches rendered pillars to the master cache. +# Rendered pillars are serialized and deserialized as msgpack structures for speed. +# Note that pillars are stored UNENCRYPTED. Ensure that the master cache +# has permissions set appropriately. (Same defaults are provided.) +# +# memory: [EXPERIMENTAL] An optional backend for pillar caches which uses a pure-Python +# in-memory data structure for maximal performance. There are several caveats, +# however. First, because each master worker contains its own in-memory cache, +# there is no guarantee of cache consistency between minion requests. This +# works best in situations where the pillar rarely if ever changes. Secondly, +# and perhaps more importantly, this means that unencrypted pillars will +# be accessible to any process which can examine the memory of the ``salt-master``! +# This may represent a substantial security risk. +# +#pillar_cache_backend: disk + + +##### Syndic settings ##### +########################################## +# The Salt syndic is used to pass commands through a master from a higher +# master. Using the syndic is simple. If this is a master that will have +# syndic servers(s) below it, then set the "order_masters" setting to True. +# +# If this is a master that will be running a syndic daemon for passthrough, then +# the "syndic_master" setting needs to be set to the location of the master server +# to receive commands from. + +# Set the order_masters setting to True if this master will command lower +# masters' syndic interfaces. +#order_masters: False + +# If this master will be running a salt syndic daemon, syndic_master tells +# this master where to receive commands from. +#syndic_master: masterofmasters + +# This is the 'ret_port' of the MasterOfMaster: +#syndic_master_port: 4506 + +# PID file of the syndic daemon: +#syndic_pidfile: /var/run/salt-syndic.pid + +# The log file of the salt-syndic daemon: +#syndic_log_file: /var/log/salt/syndic + +# The behaviour of the multi-syndic when connection to a master of masters failed. +# Can specify ``random`` (default) or ``ordered``. If set to ``random``, masters +# will be iterated in random order. If ``ordered`` is specified, the configured +# order will be used. +#syndic_failover: random + +# The number of seconds for the salt client to wait for additional syndics to +# check in with their lists of expected minions before giving up. +#syndic_wait: 5 + + +##### Peer Publish settings ##### +########################################## +# Salt minions can send commands to other minions, but only if the minion is +# allowed to. By default "Peer Publication" is disabled, and when enabled it +# is enabled for specific minions and specific commands. This allows secure +# compartmentalization of commands based on individual minions. + +# The configuration uses regular expressions to match minions and then a list +# of regular expressions to match functions. The following will allow the +# minion authenticated as foo.example.com to execute functions from the test +# and pkg modules. +#peer: +# foo.example.com: +# - test.* +# - pkg.* +# +# This will allow all minions to execute all commands: +#peer: +# .*: +# - .* +# +# This is not recommended, since it would allow anyone who gets root on any +# single minion to instantly have root on all of the minions! + +# Minions can also be allowed to execute runners from the salt master. +# Since executing a runner from the minion could be considered a security risk, +# it needs to be enabled. This setting functions just like the peer setting +# except that it opens up runners instead of module functions. +# +# All peer runner support is turned off by default and must be enabled before +# using. This will enable all peer runners for all minions: +#peer_run: +# .*: +# - .* +# +# To enable just the manage.up runner for the minion foo.example.com: +#peer_run: +# foo.example.com: +# - manage.up +# +# +##### Mine settings ##### +##################################### +# Restrict mine.get access from minions. By default any minion has a full access +# to get all mine data from master cache. In acl definion below, only pcre matches +# are allowed. +# mine_get: +# .*: +# - .* +# +# The example below enables minion foo.example.com to get 'network.interfaces' mine +# data only, minions web* to get all network.* and disk.* mine data and all other +# minions won't get any mine data. +# mine_get: +# foo.example.com: +# - network.interfaces +# web.*: +# - network.* +# - disk.* + + +##### Logging settings ##### +########################################## +# The location of the master log file +# The master log can be sent to a regular file, local path name, or network +# location. Remote logging works best when configured to use rsyslogd(8) (e.g.: +# ``file:///dev/log``), with rsyslogd(8) configured for network logging. The URI +# format is: ://:/ +#log_file: /var/log/salt/master +#log_file: file:///dev/log +#log_file: udp://loghost:10514 + +#log_file: /var/log/salt/master +#key_logfile: /var/log/salt/key + +# The level of messages to send to the console. +# One of 'garbage', 'trace', 'debug', info', 'warning', 'error', 'critical'. +# +# The following log levels are considered INSECURE and may log sensitive data: +# ['garbage', 'trace', 'debug'] +# +#log_level: warning + +# The level of messages to send to the log file. +# One of 'garbage', 'trace', 'debug', info', 'warning', 'error', 'critical'. +# If using 'log_granular_levels' this must be set to the highest desired level. +#log_level_logfile: warning + +# The date and time format used in log messages. Allowed date/time formatting +# can be seen here: http://docs.python.org/library/time.html#time.strftime +#log_datefmt: '%H:%M:%S' +#log_datefmt_logfile: '%Y-%m-%d %H:%M:%S' + +# The format of the console logging messages. Allowed formatting options can +# be seen here: http://docs.python.org/library/logging.html#logrecord-attributes +# +# Console log colors are specified by these additional formatters: +# +# %(colorlevel)s +# %(colorname)s +# %(colorprocess)s +# %(colormsg)s +# +# Since it is desirable to include the surrounding brackets, '[' and ']', in +# the coloring of the messages, these color formatters also include padding as +# well. Color LogRecord attributes are only available for console logging. +# +#log_fmt_console: '%(colorlevel)s %(colormsg)s' +#log_fmt_console: '[%(levelname)-8s] %(message)s' +# +#log_fmt_logfile: '%(asctime)s,%(msecs)03d [%(name)-17s][%(levelname)-8s] %(message)s' + +# This can be used to control logging levels more specificically. This +# example sets the main salt library at the 'warning' level, but sets +# 'salt.modules' to log at the 'debug' level: +# log_granular_levels: +# 'salt': 'warning' +# 'salt.modules': 'debug' +# +#log_granular_levels: {} + + +##### Node Groups ###### +########################################## +# Node groups allow for logical groupings of minion nodes. A group consists of +# a group name and a compound target. Nodgroups can reference other nodegroups +# with 'N@' classifier. Ensure that you do not have circular references. +# +#nodegroups: +# group1: 'L@foo.domain.com,bar.domain.com,baz.domain.com or bl*.domain.com' +# group2: 'G@os:Debian and foo.domain.com' +# group3: 'G@os:Debian and N@group1' +# group4: +# - 'G@foo:bar' +# - 'or' +# - 'G@foo:baz' + + +##### Range Cluster settings ##### +########################################## +# The range server (and optional port) that serves your cluster information +# https://github.com/ytoolshed/range/wiki/%22yamlfile%22-module-file-spec +# +#range_server: range:80 + + +##### Windows Software Repo settings ##### +########################################### +# Location of the repo on the master: +#winrepo_dir_ng: '/srv/salt/win/repo-ng' +# +# List of git repositories to include with the local repo: +#winrepo_remotes_ng: +# - 'https://github.com/saltstack/salt-winrepo-ng.git' + + +##### Windows Software Repo settings - Pre 2015.8 ##### +######################################################## +# Legacy repo settings for pre-2015.8 Windows minions. +# +# Location of the repo on the master: +#winrepo_dir: '/srv/salt/win/repo' +# +# Location of the master's repo cache file: +#winrepo_mastercachefile: '/srv/salt/win/repo/winrepo.p' +# +# List of git repositories to include with the local repo: +#winrepo_remotes: +# - 'https://github.com/saltstack/salt-winrepo.git' + +# The refspecs fetched by winrepo remotes +#winrepo_refspecs: +# - '+refs/heads/*:refs/remotes/origin/*' +# - '+refs/tags/*:refs/tags/*' +# + +##### Returner settings ###### +############################################ +# Which returner(s) will be used for minion's result: +#return: mysql + + +###### Miscellaneous settings ###### +############################################ +# Default match type for filtering events tags: startswith, endswith, find, regex, fnmatch +#event_match_type: startswith + +# Save runner returns to the job cache +#runner_returns: True + +# Permanently include any available Python 3rd party modules into thin and minimal Salt +# when they are generated for Salt-SSH or other purposes. +# The modules should be named by the names they are actually imported inside the Python. +# The value of the parameters can be either one module or a comma separated list of them. +#thin_extra_mods: foo,bar +#min_extra_mods: foo,bar,baz + + +###### Keepalive settings ###### +############################################ +# Warning: Failure to set TCP keepalives on the salt-master can result in +# not detecting the loss of a minion when the connection is lost or when +# it's host has been terminated without first closing the socket. +# Salt's Presence System depends on this connection status to know if a minion +# is "present". +# ZeroMQ now includes support for configuring SO_KEEPALIVE if supported by +# the OS. If connections between the minion and the master pass through +# a state tracking device such as a firewall or VPN gateway, there is +# the risk that it could tear down the connection the master and minion +# without informing either party that their connection has been taken away. +# Enabling TCP Keepalives prevents this from happening. + +# Overall state of TCP Keepalives, enable (1 or True), disable (0 or False) +# or leave to the OS defaults (-1), on Linux, typically disabled. Default True, enabled. +#tcp_keepalive: True + +# How long before the first keepalive should be sent in seconds. Default 300 +# to send the first keepalive after 5 minutes, OS default (-1) is typically 7200 seconds +# on Linux see /proc/sys/net/ipv4/tcp_keepalive_time. +#tcp_keepalive_idle: 300 + +# How many lost probes are needed to consider the connection lost. Default -1 +# to use OS defaults, typically 9 on Linux, see /proc/sys/net/ipv4/tcp_keepalive_probes. +#tcp_keepalive_cnt: -1 + +# How often, in seconds, to send keepalives after the first one. Default -1 to +# use OS defaults, typically 75 seconds on Linux, see +# /proc/sys/net/ipv4/tcp_keepalive_intvl. +#tcp_keepalive_intvl: -1 + diff --git a/doc/_themes/saltstack2/layout.html b/doc/_themes/saltstack2/layout.html index 293f5e39d1f..10ba2374f2b 100644 --- a/doc/_themes/saltstack2/layout.html +++ b/doc/_themes/saltstack2/layout.html @@ -255,8 +255,8 @@
- +
diff --git a/doc/_themes/saltstack2/static/images/DOCBANNER.jpg b/doc/_themes/saltstack2/static/images/DOCBANNER.jpg new file mode 100644 index 00000000000..f626a181623 Binary files /dev/null and b/doc/_themes/saltstack2/static/images/DOCBANNER.jpg differ diff --git a/doc/conf.py b/doc/conf.py index 3b3db95061d..f7c329db11f 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -72,6 +72,7 @@ MOCK_MODULES = [ 'Crypto.Signature', 'Crypto.Signature.PKCS1_v1_5', 'M2Crypto', + 'msgpack', 'yaml', 'yaml.constructor', 'yaml.nodes', @@ -244,9 +245,9 @@ on_saltstack = 'SALT_ON_SALTSTACK' in os.environ project = 'Salt' version = salt.version.__version__ -latest_release = '2016.11.5' # latest release -previous_release = '2016.3.6' # latest release from previous branch -previous_release_dir = '2016.3' # path on web server for previous branch +latest_release = '2017.7.1' # latest release +previous_release = '2016.11.7' # latest release from previous branch +previous_release_dir = '2016.11' # path on web server for previous branch next_release = '' # next release next_release_dir = '' # path on web server for next release branch @@ -319,11 +320,21 @@ rst_prolog = """\ .. _`salt-packagers`: https://groups.google.com/forum/#!forum/salt-packagers .. |windownload| raw:: html -

x86: Salt-Minion-{release}-x86-Setup.exe - | md5

+

Python2 x86: Salt-Minion-{release}-x86-Setup.exe + | md5

+ +

Python2 AMD64: Salt-Minion-{release}-AMD64-Setup.exe + | md5

+

Python3 x86: Salt-Minion-{release}-x86-Setup.exe + | md5

+ +

Python3 AMD64: Salt-Minion-{release}-AMD64-Setup.exe + | md5

-

AMD64: Salt-Minion-{release}-AMD64-Setup.exe - | md5

.. |osxdownload| raw:: html diff --git a/doc/faq.rst b/doc/faq.rst index a81cd45dfdf..18674b370ba 100644 --- a/doc/faq.rst +++ b/doc/faq.rst @@ -190,6 +190,8 @@ PATH using a :mod:`file.symlink ` state. file.symlink: - target: /usr/local/bin/foo +.. _which-version: + Can I run different versions of Salt on my Master and Minion? ------------------------------------------------------------- @@ -319,7 +321,27 @@ Restart using states ******************** Now we can apply the workaround to restart the Minion in reliable way. -The following example works on both UNIX-like and Windows operating systems: +The following example works on UNIX-like operating systems: + +.. code-block:: jinja + + {%- if grains['os'] != 'Windows' % + Restart Salt Minion: + cmd.run: + - name: 'salt-call --local service.restart salt-minion' + - bg: True + - onchanges: + - pkg: Upgrade Salt Minion + {%- endif %} + +Note that restarting the ``salt-minion`` service on Windows operating systems is +not always necessary when performing an upgrade. The installer stops the +``salt-minion`` service, removes it, deletes the contents of the ``\salt\bin`` +directory, installs the new code, re-creates the ``salt-minion`` service, and +starts it (by default). The restart step **would** be necessary during the +upgrade process, however, if the minion config was edited after the upgrade or +installation. If a minion restart is necessary, the state above can be edited +as follows: .. code-block:: jinja @@ -335,8 +357,8 @@ The following example works on both UNIX-like and Windows operating systems: - pkg: Upgrade Salt Minion However, it requires more advanced tricks to upgrade from legacy version of -Salt (before ``2016.3.0``), where executing commands in the background is not -supported: +Salt (before ``2016.3.0``) on UNIX-like operating systems, where executing +commands in the background is not supported: .. code-block:: jinja diff --git a/doc/man/salt.7 b/doc/man/salt.7 index d6cfe937a1d..86c463b7713 100644 --- a/doc/man/salt.7 +++ b/doc/man/salt.7 @@ -10795,6 +10795,7 @@ cmd_whitelist_glob: .UNINDENT .UNINDENT .SS Thread Settings +.SS \fBmultiprocessing\fP .sp Default: \fBTrue\fP .sp diff --git a/doc/ref/auth/all/index.rst b/doc/ref/auth/all/index.rst index f12d6f28bff..d421efbbe70 100644 --- a/doc/ref/auth/all/index.rst +++ b/doc/ref/auth/all/index.rst @@ -19,5 +19,4 @@ auth modules pki rest sharedsecret - stormpath yubico diff --git a/doc/ref/auth/all/salt.auth.stormpath.rst b/doc/ref/auth/all/salt.auth.stormpath.rst deleted file mode 100644 index ea5e365ffa5..00000000000 --- a/doc/ref/auth/all/salt.auth.stormpath.rst +++ /dev/null @@ -1,6 +0,0 @@ -=================== -salt.auth.stormpath -=================== - -.. automodule:: salt.auth.stormpath - :members: \ No newline at end of file diff --git a/doc/ref/beacons/all/index.rst b/doc/ref/beacons/all/index.rst index c0970f4f6c0..7fccfc5b151 100644 --- a/doc/ref/beacons/all/index.rst +++ b/doc/ref/beacons/all/index.rst @@ -22,6 +22,7 @@ beacon modules load log memusage + napalm_beacon network_info network_settings pkg diff --git a/doc/ref/beacons/all/salt.beacons.napalm_beacon.rst b/doc/ref/beacons/all/salt.beacons.napalm_beacon.rst new file mode 100644 index 00000000000..ff5bbc4b01e --- /dev/null +++ b/doc/ref/beacons/all/salt.beacons.napalm_beacon.rst @@ -0,0 +1,6 @@ +========================== +salt.beacons.napalm_beacon +========================== + +.. automodule:: salt.beacons.napalm_beacon + :members: diff --git a/doc/ref/cli/_includes/output-options.rst b/doc/ref/cli/_includes/output-options.rst index 0128a1cfb07..c679269e763 100644 --- a/doc/ref/cli/_includes/output-options.rst +++ b/doc/ref/cli/_includes/output-options.rst @@ -33,6 +33,10 @@ Output Options Write the output to the specified file. +.. option:: --out-file-append, --output-file-append + + Append the output to the specified file. + .. option:: --no-color Disable all colored output @@ -46,3 +50,14 @@ Output Options ``green`` denotes success, ``red`` denotes failure, ``blue`` denotes changes and success and ``yellow`` denotes a expected future change in configuration. + +.. option:: --state-output=STATE_OUTPUT, --state_output=STATE_OUTPUT + + Override the configured state_output value for minion + output. One of 'full', 'terse', 'mixed', 'changes' or + 'filter'. Default: 'none'. + +.. option:: --state-verbose=STATE_VERBOSE, --state_verbose=STATE_VERBOSE + + Override the configured state_verbose value for minion + output. Set to True or False. Default: none. diff --git a/doc/ref/cli/salt-cloud.rst b/doc/ref/cli/salt-cloud.rst index a9f3123756d..a64c6ba83bc 100644 --- a/doc/ref/cli/salt-cloud.rst +++ b/doc/ref/cli/salt-cloud.rst @@ -136,7 +136,7 @@ Query Options .. versionadded:: 2014.7.0 Display a list of configured profiles. Pass in a cloud provider to view - the provider's associated profiles, such as ``digital_ocean``, or pass in + the provider's associated profiles, such as ``digitalocean``, or pass in ``all`` to list all the configured profiles. diff --git a/doc/ref/cli/salt-cp.rst b/doc/ref/cli/salt-cp.rst index 3922163b320..c2087c022a2 100644 --- a/doc/ref/cli/salt-cp.rst +++ b/doc/ref/cli/salt-cp.rst @@ -39,6 +39,13 @@ specified target expression. desitination will be assumed to be a directory. Finally, recursion is now supported, allowing for entire directories to be copied. +.. versionchanged:: 2016.11.7,2017.7.2 + Reverted back to the old copy mode to preserve backward compatibility. The + new functionality added in 2016.6.6 and 2017.7.0 is now available using the + ``-C`` or ``--chunked`` CLI arguments. Note that compression, recursive + copying, and support for copying large files is only available in chunked + mode. + Options ======= @@ -56,9 +63,16 @@ Options .. include:: _includes/target-selection.rst +.. option:: -C, --chunked + + Use new chunked mode to copy files. This mode supports large files, recursive + directories copying and compression. + + .. versionadded:: 2016.11.7,2017.7.2 + .. option:: -n, --no-compression - Disable gzip compression. + Disable gzip compression in chunked mode. .. versionadded:: 2016.3.7,2016.11.6,2017.7.0 diff --git a/doc/ref/cli/salt.rst b/doc/ref/cli/salt.rst index f2fb693a1ce..6eaa9e56ff1 100644 --- a/doc/ref/cli/salt.rst +++ b/doc/ref/cli/salt.rst @@ -81,7 +81,7 @@ Options Pass in an external authentication medium to validate against. The credentials will be prompted for. The options are `auto`, - `keystone`, `ldap`, `pam`, and `stormpath`. Can be used with the -T + `keystone`, `ldap`, and `pam`. Can be used with the -T option. .. option:: -T, --make-token diff --git a/doc/ref/clouds/all/index.rst b/doc/ref/clouds/all/index.rst index 5c5a3a9f5ce..15fb4b1ae37 100644 --- a/doc/ref/clouds/all/index.rst +++ b/doc/ref/clouds/all/index.rst @@ -13,7 +13,7 @@ Full list of Salt Cloud modules aliyun azurearm cloudstack - digital_ocean + digitalocean dimensiondata ec2 gce diff --git a/doc/ref/clouds/all/salt.cloud.clouds.digital_ocean.rst b/doc/ref/clouds/all/salt.cloud.clouds.digital_ocean.rst index 71917c87659..1eeb2b2a411 100644 --- a/doc/ref/clouds/all/salt.cloud.clouds.digital_ocean.rst +++ b/doc/ref/clouds/all/salt.cloud.clouds.digital_ocean.rst @@ -1,6 +1,6 @@ =============================== -salt.cloud.clouds.digital_ocean +salt.cloud.clouds.digitalocean =============================== -.. automodule:: salt.cloud.clouds.digital_ocean +.. automodule:: salt.cloud.clouds.digitalocean :members: \ No newline at end of file diff --git a/doc/ref/clouds/all/salt.cloud.clouds.oneandone.rst b/doc/ref/clouds/all/salt.cloud.clouds.oneandone.rst new file mode 100644 index 00000000000..4355d7570aa --- /dev/null +++ b/doc/ref/clouds/all/salt.cloud.clouds.oneandone.rst @@ -0,0 +1,6 @@ +=========================== +salt.cloud.clouds.oneandone +=========================== + +.. automodule:: salt.cloud.clouds.oneandone + :members: diff --git a/doc/ref/configuration/master.rst b/doc/ref/configuration/master.rst index 6a9bd7ae94b..ff6e49d3498 100644 --- a/doc/ref/configuration/master.rst +++ b/doc/ref/configuration/master.rst @@ -94,64 +94,6 @@ The user to run the Salt processes user: root -.. conf_master:: max_open_files - -``max_open_files`` ------------------- - -Default: ``100000`` - -Each minion connecting to the master uses AT LEAST one file descriptor, the -master subscription connection. If enough minions connect you might start -seeing on the console(and then salt-master crashes): - -.. code-block:: bash - - Too many open files (tcp_listener.cpp:335) - Aborted (core dumped) - -.. code-block:: yaml - - max_open_files: 100000 - -By default this value will be the one of `ulimit -Hn`, i.e., the hard limit for -max open files. - -To set a different value than the default one, uncomment, and configure this -setting. Remember that this value CANNOT be higher than the hard limit. Raising -the hard limit depends on the OS and/or distribution, a good way to find the -limit is to search the internet for something like this: - -.. code-block:: text - - raise max open files hard limit debian - -.. conf_master:: worker_threads - -``worker_threads`` ------------------- - -Default: ``5`` - -The number of threads to start for receiving commands and replies from minions. -If minions are stalling on replies because you have many minions, raise the -worker_threads value. - -Worker threads should not be put below 3 when using the peer system, but can -drop down to 1 worker otherwise. - -.. note:: - When the master daemon starts, it is expected behaviour to see - multiple salt-master processes, even if 'worker_threads' is set to '1'. At - a minimum, a controlling process will start along with a Publisher, an - EventPublisher, and a number of MWorker processes will be started. The - number of MWorker processes is tuneable by the 'worker_threads' - configuration value while the others are not. - -.. code-block:: yaml - - worker_threads: 5 - .. conf_master:: ret_port ``ret_port`` @@ -241,13 +183,16 @@ The directory to store the pki authentication keys. Directory for custom modules. This directory can contain subdirectories for each of Salt's module types such as ``runners``, ``output``, ``wheel``, -``modules``, ``states``, ``returners``, ``engines``, etc. This path is appended to -:conf_master:`root_dir`. +``modules``, ``states``, ``returners``, ``engines``, ``utils``, etc. +This path is appended to :conf_master:`root_dir`. .. code-block:: yaml extension_modules: /root/salt_extmods +.. conf_master:: extmod_whitelist +.. conf_master:: extmod_blacklist + ``extmod_whitelist/extmod_blacklist`` ------------------------------------- @@ -289,6 +234,7 @@ Valid options: - clouds - tops - roster + - tokens .. conf_master:: module_dirs @@ -393,6 +339,19 @@ Default: ``nested`` Set the default outputter used by the salt command. +.. conf_master:: outputter_dirs + +``outputter_dirs`` +------------------ + +Default: ``[]`` + +A list of additional directories to search for salt outputters in. + +.. code-block:: yaml + + outputter_dirs: [] + .. conf_master:: output_file ``output_file`` @@ -408,6 +367,32 @@ CLI option, only sets this to a single file for all salt commands. output_file: /path/output/file +.. conf_master:: show_timeout + +``show_timeout`` +---------------- + +Default: ``True`` + +Tell the client to show minions that have timed out. + +.. code-block:: yaml + + show_timeout: True + +.. conf_master:: show_jid + +``show_jid`` +------------ + +Default: ``False`` + +Tell the client to display the jid when a job is published. + +.. code-block:: yaml + + show_jid: False + .. conf_master:: color ``color`` @@ -819,6 +804,32 @@ that connect to a master via localhost. presence_events: False +.. conf_master:: ping_on_rotate + +``ping_on_rotate`` +------------------ + +Default: ``False`` + +By default, the master AES key rotates every 24 hours. The next command +following a key rotation will trigger a key refresh from the minion which may +result in minions which do not respond to the first command after a key refresh. + +To tell the master to ping all minions immediately after an AES key refresh, +set ``ping_on_rotate`` to ``True``. This should mitigate the issue where a +minion does not appear to initially respond after a key is rotated. + +Note that ping_on_rotate may cause high load on the master immediately after +the key rotation event as minions reconnect. Consider this carefully if this +salt master is managing a large number of minions. + +If disabled, it is recommended to handle this event by listening for the +``aes_key_rotate`` event with the ``key`` tag and acting appropriately. + +.. code-block:: yaml + + ping_on_rotate: False + .. conf_master:: transport ``transport`` @@ -837,6 +848,8 @@ what you are doing! Transports are explained in :ref:`Salt Transports transport: zeromq +.. conf_master:: transport_opts + ``transport_opts`` ------------------ @@ -855,6 +868,95 @@ what you are doing! Transports are explained in :ref:`Salt Transports ret_port: 4606 zeromq: [] +.. conf_master:: sock_pool_size + +``sock_pool_size`` +------------------ + +Default: 1 + +To avoid blocking waiting while writing a data to a socket, we support +socket pool for Salt applications. For example, a job with a large number +of target host list can cause long period blocking waiting. The option +is used by ZMQ and TCP transports, and the other transport methods don't +need the socket pool by definition. Most of Salt tools, including CLI, +are enough to use a single bucket of socket pool. On the other hands, +it is highly recommended to set the size of socket pool larger than 1 +for other Salt applications, especially Salt API, which must write data +to socket concurrently. + +.. code-block:: yaml + + sock_pool_size: 15 + +.. conf_master:: ipc_mode + +``ipc_mode`` +------------ + +Default: ``ipc`` + +The ipc strategy. (i.e., sockets versus tcp, etc.) Windows platforms lack +POSIX IPC and must rely on TCP based inter-process communications. ``ipc_mode`` +is set to ``tcp`` by default on Windows. + +.. code-block:: yaml + + ipc_mode: ipc + +.. conf_master:: + +``tcp_master_pub_port`` +----------------------- + +Default: ``4512`` + +The TCP port on which events for the master should be published if ``ipc_mode`` is TCP. + +.. code-block:: yaml + + tcp_master_pub_port: 4512 + +.. conf_master:: tcp_master_pull_port + +``tcp_master_pull_port`` +------------------------ + +Default: ``4513`` + +The TCP port on which events for the master should be pulled if ``ipc_mode`` is TCP. + +.. code-block:: yaml + + tcp_master_pull_port: 4513 + +.. conf_master:: tcp_master_publish_pull + +``tcp_master_publish_pull`` +--------------------------- + +Default: ``4514`` + +The TCP port on which events for the master should be pulled fom and then republished onto +the event bus on the master. + +.. code-block:: yaml + + tcp_master_publish_pull: 4514 + +.. conf_master:: tcp_master_workers + +``tcp_master_workers`` +---------------------- + +Default: ``4515`` + +The TCP port for ``mworkers`` to connect to on the master. + +.. code-block:: yaml + + tcp_master_workers: 4515 + .. _salt-ssh-configuration: @@ -874,6 +976,97 @@ Pass in an alternative location for the salt-ssh roster file. roster_file: /root/roster +.. conf_master:: ssh_passwd + +``ssh_passwd`` +-------------- + +Default: ``''`` + +The ssh password to log in with. + +.. code-block:: yaml + + ssh_passwd: '' + +.. conf_master:: ssh_port + +``ssh_port`` +------------ + +Default: ``22`` + +The target system's ssh port number. + +.. code-block:: yaml + + ssh_port: 22 + +.. conf_master:: ssh_scan_ports + +``ssh_scan_ports`` +------------------ + +Default: ``22`` + +Comma-separated list of ports to scan. + +.. code-block:: yaml + + ssh_scan_ports: 22 + +.. conf_master:: ssh_scan_timeout + +``ssh_scan_timeout`` +-------------------- + +Default: ``0.01`` + +Scanning socket timeout for salt-ssh. + +.. code-block:: yaml + + ssh_scan_timeout: 0.01 + +.. conf_master:: ssh_sudo + +``ssh_sudo`` +------------ + +Default: ``False`` + +Boolean to run command via sudo. + +.. code-block:: yaml + + ssh_sudo: False + +.. conf_master:: ssh_timeout + +``ssh_timeout`` +--------------- + +Default: ``60`` + +Number of seconds to wait for a response when establishing an SSH connection. + +.. code-block:: yaml + + ssh_timeout: 60 + +.. conf_master:: ssh_user + +``ssh_user`` +------------ + +Default: ``root`` + +The user to log in as. + +.. code-block:: yaml + + ssh_user: root + .. conf_master:: ssh_log_file ``ssh_log_file`` @@ -905,6 +1098,8 @@ overridden on a per-minion basis in the roster (``minion_opts``) ssh_minion_opts: gpg_keydir: /root/gpg +.. conf_master:: ssh_use_home_key + ``ssh_use_home_key`` -------------------- @@ -917,6 +1112,41 @@ authentication with minions ssh_use_home_key: False +.. conf_master:: ssh_identities_only + +``ssh_identities_only`` +----------------------- + +Default: ``False`` + +Set this to ``True`` to default salt-ssh to run with ``-o IdentitiesOnly=yes``. This +option is intended for situations where the ssh-agent offers many different identities +and allows ssh to ignore those identities and use the only one specified in options. + +.. code-block:: yaml + + ssh_identities_only: False + +.. conf_master:: ssh_list_nodegroups + +``ssh_list_nodegroups`` +----------------------- + +Default: ``{}`` + +List-only nodegroups for salt-ssh. Each group must be formed as either a comma-separated +list, or a YAML list. This option is useful to group minions into easy-to-target groups +when using salt-ssh. These groups can then be targeted with the normal -N argument to +salt-ssh. + +.. code-block:: yaml + + ssh_list_nodegroups: + groupA: minion1,minion2 + groupB: minion1,minion3 + +.. conf_master:: thin_extra_mods + ``thin_extra_mods`` ------------------- @@ -973,6 +1203,19 @@ public keys from minions. auto_accept: False +.. conf_master:: keysize + +``keysize`` +----------- + +Default: ``2048`` + +The size of key that should be generated when creating new keys. + +.. code-block:: yaml + + keysize: 2048 + .. conf_master:: autosign_timeout ``autosign_timeout`` @@ -1017,6 +1260,24 @@ minion IDs for which keys will automatically be rejected. Will override both membership in the :conf_master:`autosign_file` and the :conf_master:`auto_accept` setting. +.. conf_master:: permissive_pki_access + +``permissive_pki_access`` +------------------------- + +Default: ``False`` + +Enable permissive access to the salt keys. This allows you to run the +master or minion as root, but have a non-root group be given access to +your pki_dir. To make the access explicit, root must belong to the group +you've given access to. This is potentially quite insecure. If an autosign_file +is specified, enabling permissive_pki_access will allow group access to that +specific file. + +.. code-block:: yaml + + permissive_pki_access: False + .. conf_master:: publisher_acl ``publisher_acl`` @@ -1059,6 +1320,20 @@ This is completely disabled by default. - cmd.* - test.echo +.. conf_master:: sudo_acl + +``sudo_acl`` +------------ + +Default: ``False`` + +Enforce ``publisher_acl`` and ``publisher_acl_blacklist`` when users have sudo +access to the salt command. + +.. code-block:: yaml + + sudo_acl: False + .. conf_master:: external_auth ``external_auth`` @@ -1243,6 +1518,19 @@ Do not disable this unless it is absolutely clear what this does. rotate_aes_key: True +.. conf_master:: publish_session + +``publish_session`` +------------------- + +Default: ``86400`` + +The number of seconds between AES key rotations on the master. + +.. code-block:: yaml + + publish_session: Default: 86400 + .. conf_master:: ssl ``ssl`` @@ -1273,6 +1561,24 @@ constant names without ssl module prefix: ``CERT_REQUIRED`` or ``PROTOCOL_SSLv23 ``allow_minion_key_revoke`` --------------------------- +Default: ``False`` + +By default, the master deletes its cache of minion data when the key for that +minion is removed. To preserve the cache after key deletion, set +``preserve_minion_cache`` to True. + +WARNING: This may have security implications if compromised minions auth with +a previous deleted minion ID. + +.. code-block:: yaml + + preserve_minion_cache: False + +.. conf_master:: allow_minion_key_revoke + +``allow_minion_key_revoke`` +--------------------------- + Default: ``True`` Controls whether a minion can request its own key revocation. When True @@ -1285,6 +1591,127 @@ the master will drop the request and the minion's key will remain accepted. rotate_aes_key: True +Master Large Scale Tuning Settings +================================== + +.. conf_master:: max_open_files + +``max_open_files`` +------------------ + +Default: ``100000`` + +Each minion connecting to the master uses AT LEAST one file descriptor, the +master subscription connection. If enough minions connect you might start +seeing on the console(and then salt-master crashes): + +.. code-block:: bash + + Too many open files (tcp_listener.cpp:335) + Aborted (core dumped) + +.. code-block:: yaml + + max_open_files: 100000 + +By default this value will be the one of `ulimit -Hn`, i.e., the hard limit for +max open files. + +To set a different value than the default one, uncomment, and configure this +setting. Remember that this value CANNOT be higher than the hard limit. Raising +the hard limit depends on the OS and/or distribution, a good way to find the +limit is to search the internet for something like this: + +.. code-block:: text + + raise max open files hard limit debian + +.. conf_master:: worker_threads + +``worker_threads`` +------------------ + +Default: ``5`` + +The number of threads to start for receiving commands and replies from minions. +If minions are stalling on replies because you have many minions, raise the +worker_threads value. + +Worker threads should not be put below 3 when using the peer system, but can +drop down to 1 worker otherwise. + +.. note:: + When the master daemon starts, it is expected behaviour to see + multiple salt-master processes, even if 'worker_threads' is set to '1'. At + a minimum, a controlling process will start along with a Publisher, an + EventPublisher, and a number of MWorker processes will be started. The + number of MWorker processes is tuneable by the 'worker_threads' + configuration value while the others are not. + +.. code-block:: yaml + + worker_threads: 5 + +.. conf_master:: pub_hwm + +``pub_hwm`` +----------- + +Default: ``1000`` + +The zeromq high water mark on the publisher interface. + +.. code-block:: yaml + + pub_hwm: 1000 + +.. conf_master:: zmq_backlog + +``zmq_backlog`` +--------------- + +Default: ``1000`` + +The listen queue size of the ZeroMQ backlog. + +.. code-block:: yaml + + zmq_backlog: 1000 + +.. conf_master:: salt_event_pub_hwm +.. conf_master:: event_publisher_pub_hwm + +``salt_event_pub_hwm`` and ``event_publisher_pub_hwm`` +------------------------------------------------------ + +These two ZeroMQ High Water Mark settings, ``salt_event_pub_hwm`` and +``event_publisher_pub_hwm`` are significant for masters with thousands of +minions. When these are insufficiently high it will manifest in random +responses missing in the CLI and even missing from the job cache. Masters +that have fast CPUs and many cores with appropriate ``worker_threads`` +will not need these set as high. + +The ZeroMQ high-water-mark for the ``SaltEvent`` pub socket default is: + +.. code-block:: yaml + + salt_event_pub_hwm: 20000 + +The ZeroMQ high-water-mark for the ``EventPublisher`` pub socket default is: + +.. code-block:: yaml + + event_publisher_pub_hwm: 10000 + +As an example, on single master deployment with 8,000 minions, 2.4GHz CPUs, +24 cores, and 32GiB memory has these settings: + +.. code-block:: yaml + + salt_event_pub_hwm: 128000 + event_publisher_pub_hwm: 64000 + + .. _master-module-management: Master Module Management @@ -1304,6 +1731,22 @@ Set additional directories to search for runner modules. runner_dirs: - /var/lib/salt/runners +.. conf_master:: utils_dirs + +``utils_dirs`` +--------------- + +.. versionadded:: Oxygen + +Default: ``[]`` + +Set additional directories to search for util modules. + +.. code-block:: yaml + + utils_dirs: + - /var/lib/salt/utils + .. conf_master:: cython_enable ``cython_enable`` @@ -1333,7 +1776,8 @@ Default: ``top.sls`` The state system uses a "top" file to tell the minions what environment to use and what modules to use. The state_top file is defined relative to the -root of the base environment. +root of the base environment. The value of "state_top" is also used for the +pillar top file .. code-block:: yaml @@ -1577,6 +2021,21 @@ If set to 'changes', the output will be full unless the state didn't change. state_output: full +.. conf_master:: state_output_diff + +``state_output_diff`` +--------------------- + +Default: ``False`` + +The state_output_diff setting changes whether or not the output from +successful states is returned. Useful when even the terse output of these +states is cluttering the logs. Set it to True to ignore them. + +.. code-block:: yaml + + state_output_diff: False + .. conf_master:: state_aggregate ``state_aggregate`` @@ -1889,6 +2348,19 @@ Example: For masterless Salt, this parameter must be specified in the minion config file. +.. conf_master:: master_roots + +``master_roots`` +---------------- + +Default: ``/srv/salt-master`` + +A master-only copy of the file_roots dictionary, used by the state compiler. + +.. code-block:: yaml + + master_roots: /srv/salt-master + git: Git Remote File Server Backend ----------------------------------- @@ -2056,6 +2528,60 @@ gitfs remotes. - dev: - ref: develop +.. conf_master:: gitfs_disable_saltenv_mapping + +``gitfs_disable_saltenv_mapping`` +********************************* + +.. versionadded:: Oxygen + +Default: ``False`` + +When set to ``True``, all saltenv mapping logic is disregarded (aside from +which branch/tag is mapped to the ``base`` saltenv). To use any other +environments, they must then be defined using :ref:`per-saltenv configuration +parameters `. + +.. code-block:: yaml + + gitfs_disable_saltenv_mapping: True + +.. note:: + This is is a global configuration option, see :ref:`here + ` for examples of configuring it for individual + repositories. + +.. conf_master:: gitfs_ref_types + +``gitfs_ref_types`` +******************* + +.. versionadded:: Oxygen + +Default: ``['branch', 'tag', 'sha']`` + +This option defines what types of refs are mapped to fileserver environments +(i.e. saltenvs). It also sets the order of preference when there are +ambiguously-named refs (i.e. when a branch and tag both have the same name). +The below example disables mapping of both tags and SHAs, so that only branches +are mapped as saltenvs: + +.. code-block:: yaml + + gitfs_ref_types: + - branch + +.. note:: + This is is a global configuration option, see :ref:`here + ` for examples of configuring it for individual + repositories. + +.. note:: + ``sha`` is special in that it will not show up when listing saltenvs (e.g. + with the :py:func:`fileserver.envs ` runner), + but works within states and with :py:func:`cp.cache_file + ` to retrieve a file from a specific git SHA. + .. conf_master:: gitfs_saltenv_whitelist ``gitfs_saltenv_whitelist`` @@ -2944,6 +3470,26 @@ configuration. pillar_opts: False +.. conf_master:: pillar_safe_render_error + +``pillar_safe_render_error`` +---------------------------- + +Default: ``True`` + +The pillar_safe_render_error option prevents the master from passing pillar +render errors to the minion. This is set on by default because the error could +contain templating data which would give that minion information it shouldn't +have, like a password! When set ``True`` the error message will only show: + +.. code-block:: shell + + Rendering SLS 'my.sls' failed. Please see master log for details. + +.. code-block:: yaml + + pillar_safe_render_error: True + .. _master-configuration-ext-pillar: .. conf_master:: ext_pillar @@ -3241,7 +3787,7 @@ they were created by a different master. Default: ``True`` Normally, when processing :ref:`git_pillar remotes -`, if more than one repo under the same ``git`` +`, if more than one repo under the same ``git`` section in the ``ext_pillar`` configuration refers to the same pillar environment, then each repo in a given environment will have access to the other repos' files to be referenced in their top files. However, it may be @@ -3614,6 +4160,62 @@ can be utilized: pillar_cache_backend: disk +Master Reactor Settings +======================= + +.. conf_master:: reactor + +``reactor`` +----------- + +Default: ``[]`` + +Defines a salt reactor. See the :ref:`Reactor ` documentation for more +information. + +.. code-block:: yaml + + reactor: [] + +.. conf_master:: reactor_refresh_interval + +``reactor_refresh_interval`` +---------------------------- + +Default: ``60`` + +The TTL for the cache of the reactor configuration. + +.. code-block:: yaml + + reactor_refresh_interval: 60 + +.. conf_master:: reactor_worker_threads + +``reactor_worker_threads`` +-------------------------- + +Default: ``10`` + +The number of workers for the runner/wheel in the reactor. + +.. code-block:: yaml + reactor_worker_threads: 10 + +.. conf_master:: reactor_worker_hwm + +``reactor_worker_hwm`` +---------------------- + +Default: ``10000`` + +The queue size for workers in the reactor. + +.. code-block:: yaml + + reactor_worker_hwm: 10000 + + .. _syndic-server-settings: Syndic Server Settings @@ -4080,6 +4682,63 @@ option then the master will log a warning message. - /etc/roles/webserver +Keepalive Settings +================== + +.. conf_master:: tcp_keepalive + +``tcp_keepalive`` +----------------- + +Default: ``True`` + +The tcp keepalive interval to set on TCP ports. This setting can be used to tune Salt +connectivity issues in messy network environments with misbehaving firewalls. + +.. code-block:: yaml + + tcp_keepalive: True + +.. conf_master:: tcp_keepalive_cnt + +``tcp_keepalive_cnt`` +--------------------- + +Default: ``-1`` + +Sets the ZeroMQ TCP keepalive count. May be used to tune issues with minion disconnects. + +.. code-block:: yaml + + tcp_keepalive_cnt: -1 + +.. conf_master:: tcp_keepalive_idle + +``tcp_keepalive_idle`` +---------------------- + +Default: ``300`` + +Sets ZeroMQ TCP keepalive idle. May be used to tune issues with minion disconnects. + +.. code-block:: yaml + + tcp_keepalive_idle: 300 + +.. conf_master:: tcp_keepalive_intvl + +``tcp_keepalive_intvl`` +----------------------- + +Default: ``-1`` + +Sets ZeroMQ TCP keepalive interval. May be used to tune issues with minion disconnects. + +.. code-block:: yaml + + tcp_keepalive_intvl': -1 + + .. _winrepo-master-config-opts: Windows Software Repo Settings @@ -4218,7 +4877,7 @@ URL of the repository: .. code-block:: yaml - winrepo_remotes: + winrepo_remotes_ng: - ' https://github.com/saltstack/salt-winrepo-ng.git' Replace ```` with the SHA1 hash of a commit ID. Specifying a commit diff --git a/doc/ref/configuration/minion.rst b/doc/ref/configuration/minion.rst index 9919e136626..3438bfca035 100644 --- a/doc/ref/configuration/minion.rst +++ b/doc/ref/configuration/minion.rst @@ -133,6 +133,24 @@ name) is set in the :conf_minion:`master` configuration setting. master_uri_format: ip_only +.. conf_minion:: master_tops_first + +``master_tops_first`` +--------------------- + +.. versionadded:: Oxygen + +Default: ``False`` + +SLS targets defined using the :ref:`Master Tops ` system +are normally executed *after* any matches defined in the :ref:`Top File +`. Set this option to ``True`` to have the minion execute the +:ref:`Master Tops ` states first. + +.. code-block:: yaml + + master_tops_first: True + .. conf_minion:: master_type ``master_type`` @@ -319,7 +337,7 @@ The user to run the Salt processes .. conf_minion:: sudo_user ``sudo_user`` --------------- +------------- Default: ``''`` @@ -610,6 +628,26 @@ With ``grains_deep_merge``, the result will be: k1: v1 k2: v2 +.. conf_minion:: grains_refresh_every + +``grains_refresh_every`` +------------------------ + +Default: ``0`` + +The ``grains_refresh_every`` setting allows for a minion to periodically +check its grains to see if they have changed and, if so, to inform the master +of the new grains. This operation is moderately expensive, therefore care +should be taken not to set this value too low. + +Note: This value is expressed in minutes. + +A value of 10 minutes is a reasonable default. + +.. code-block:: yaml + + grains_refresh_every: 0 + .. conf_minion:: mine_enabled ``mine_enabled`` @@ -643,7 +681,7 @@ return for the job cache. mine_return_job: False ``mine_functions`` -------------------- +------------------ Default: Empty @@ -661,6 +699,18 @@ Note these can be defined in the pillar for a minion as well. interface: eth0 cidr: '10.0.0.0/8' +.. conf_minion:: mine_interval + +``mine_interval`` +----------------- + +Default: ``60`` + +The number of minutes between mine updates. + +.. code-block:: yaml + + mine_interval: 60 .. conf_minion:: sock_dir @@ -675,6 +725,19 @@ The directory where Unix sockets will be kept. sock_dir: /var/run/salt/minion +.. conf_minion:: outputter_dirs + +``outputter_dirs`` +------------------ + +Default: ``[]`` + +A list of additional directories to search for salt outputters in. + +.. code-block:: yaml + + outputter_dirs: [] + .. conf_minion:: backup_mode ``backup_mode`` @@ -719,6 +782,20 @@ seconds each iteration. acceptance_wait_time_max: 0 +.. conf_minion:: rejected_retry + +``rejected_retry`` +------------------ + +Default: ``False`` + +If the master rejects the minion's public key, retry instead of exiting. +Rejected keys will be handled the same as waiting on acceptance. + +.. code-block:: yaml + + rejected_retry: False + .. conf_minion:: random_reauth_delay ``random_reauth_delay`` @@ -817,6 +894,20 @@ restart. auth_safemode: False +.. conf_minion:: ping_interval + +``ping_interval`` +----------------- + +Default: ``0`` + +Instructs the minion to ping its master(s) every n number of seconds. Used +primarily as a mitigation technique against minion disconnects. + +.. code-block:: yaml + + ping_interval: 0 + .. conf_minion:: recon_default ``random_startup_delay`` @@ -1135,7 +1226,7 @@ If certain returners should be disabled, this is the place .. conf_minion:: enable_whitelist_modules ``whitelist_modules`` ----------------------------- +--------------------- Default: ``[]`` (Module whitelisting is disabled. Adding anything to the config option will cause only the listed modules to be enabled. Modules not in the list will @@ -1227,6 +1318,20 @@ A list of extra directories to search for Salt renderers render_dirs: - /var/lib/salt/renderers +.. conf_minion:: utils_dirs + +``utils_dirs`` +-------------- + +Default: ``[]`` + +A list of extra directories to search for Salt utilities + +.. code-block:: yaml + + utils_dirs: + - /var/lib/salt/utils + .. conf_minion:: cython_enable ``cython_enable`` @@ -1275,6 +1380,23 @@ below. providers: service: systemd +.. conf_minion:: modules_max_memory + +``modules_max_memory`` +---------------------- + +Default: ``-1`` + +Specify a max size (in bytes) for modules on import. This feature is currently +only supported on *nix operating systems and requires psutil. + +.. code-block:: yaml + + modules_max_memory: -1 + +.. conf_minion:: extmod_whitelist +.. conf_minion:: extmod_blacklist + ``extmod_whitelist/extmod_blacklist`` ------------------------------------- @@ -1297,8 +1419,8 @@ whitelist an empty list. modules: - specific_module - Valid options: + - beacons - clouds - sdb @@ -1444,6 +1566,52 @@ environment lacks one. default_top: dev +.. conf_minion:: startup_states + +``startup_states`` +------------------ + +Default: ``''`` + +States to run when the minion daemon starts. To enable, set ``startup_states`` to: + +- ``highstate``: Execute state.highstate +- ``sls``: Read in the sls_list option and execute the named sls files +- ``top``: Read top_file option and execute based on that file on the Master + +.. code-block:: yaml + + startup_states: '' + +.. conf_minion:: sls_list + +``sls_list`` +------------ + +Default: ``[]`` + +List of states to run when the minion starts up if ``startup_states`` is set to ``sls``. + +.. code-block:: yaml + + sls_list: + - edit.vim + - hyper + +.. conf_minion:: top_file + +``top_file`` +------------ + +Default: ``''`` + +Top file to execute if ``startup_states`` is set to ``top``. + +.. code-block:: yaml + + top_file: '' + + State Management Settings ========================= @@ -1460,7 +1628,7 @@ The default renderer used for local state executions renderer: yaml_jinja -.. conf_master:: test +.. conf_minion:: test ``test`` -------- @@ -1504,6 +1672,22 @@ the output will be shortened to a single line. state_output: full + +.. conf_minion:: state_output_diff + +``state_output_diff`` +--------------------- + +Default: ``False`` + +The state_output_diff setting changes whether or not the output from +successful states is returned. Useful when even the terse output of these +states is cluttering the logs. Set it to True to ignore them. + +.. code-block:: yaml + + state_output_diff: False + .. conf_minion:: autoload_dynamic_modules ``autoload_dynamic_modules`` @@ -1929,6 +2113,41 @@ It will be interpreted as megabytes. file_recv_max_size: 100 +.. conf_minion:: pass_to_ext_pillars + +``pass_to_ext_pillars`` +----------------------- + +Specify a list of configuration keys whose values are to be passed to +external pillar functions. + +Suboptions can be specified using the ':' notation (i.e. ``option:suboption``) + +The values are merged and included in the ``extra_minion_data`` optional +parameter of the external pillar function. The ``extra_minion_data`` parameter +is passed only to the external pillar functions that have it explicitly +specified in their definition. + +If the config contains + +.. code-block:: yaml + + opt1: value1 + opt2: + subopt1: value2 + subopt2: value3 + + pass_to_ext_pillars: + - opt1 + - opt2: subopt1 + +the ``extra_minion_data`` parameter will be + +.. code-block:: python + + {'opt1': 'value1', + 'opt2': {'subopt1': 'value2'}} + Security Settings ================= @@ -1962,6 +2181,35 @@ before the initial key exchange. The master fingerprint can be found by running master_finger: 'ba:30:65:2a:d6:9e:20:4f:d8:b2:f3:a7:d4:65:11:13' +.. conf_minion:: keysize + +``keysize`` +----------- + +Default: ``2048`` + +The size of key that should be generated when creating new keys. + +.. code-block:: yaml + + keysize: 2048 + +.. conf_minion:: permissive_pki_access + +``permissive_pki_access`` +------------------------- + +Default: ``False`` + +Enable permissive access to the salt keys. This allows you to run the +master or minion as root, but have a non-root group be given access to +your pki_dir. To make the access explicit, root must belong to the group +you've given access to. This is potentially quite insecure. + +.. code-block:: yaml + + permissive_pki_access: False + .. conf_minion:: verify_master_pubkey_sign ``verify_master_pubkey_sign`` @@ -2069,7 +2317,7 @@ blocked. If `cmd_whitelist_glob` is NOT SET, then all shell commands are permitt - 'cat /etc/fstab' -.. conf_master:: ssl +.. conf_minion:: ssl ``ssl`` ------- @@ -2095,16 +2343,75 @@ constant names without ssl module prefix: ``CERT_REQUIRED`` or ``PROTOCOL_SSLv23 ssl_version: PROTOCOL_TLSv1_2 +Reactor Settings +================ + +.. conf_minion:: reactor + +``reactor`` +----------- + +Default: ``[]`` + +Defines a salt reactor. See the :ref:`Reactor ` documentation for more +information. + +.. code-block:: yaml + + reactor: [] + +.. conf_minion:: reactor_refresh_interval + +``reactor_refresh_interval`` +---------------------------- + +Default: ``60`` + +The TTL for the cache of the reactor configuration. + +.. code-block:: yaml + + reactor_refresh_interval: 60 + +.. conf_minion:: reactor_worker_threads + +``reactor_worker_threads`` +-------------------------- + +Default: ``10`` + +The number of workers for the runner/wheel in the reactor. + +.. code-block:: yaml + reactor_worker_threads: 10 + +.. conf_minion:: reactor_worker_hwm + +``reactor_worker_hwm`` +---------------------- + +Default: ``10000`` + +The queue size for workers in the reactor. + +.. code-block:: yaml + + reactor_worker_hwm: 10000 + + Thread Settings =============== .. conf_minion:: multiprocessing +``multiprocessing`` +------- + Default: ``True`` -If `multiprocessing` is enabled when a minion receives a +If ``multiprocessing`` is enabled when a minion receives a publication a new process is spawned and the command is executed therein. -Conversely, if `multiprocessing` is disabled the new publication will be run +Conversely, if ``multiprocessing`` is disabled the new publication will be run executed in a thread. @@ -2365,6 +2672,62 @@ option then the minion will log a warning message. - /etc/roles/webserver +Keepalive Settings +================== + +.. conf_minion:: tcp_keepalive + +``tcp_keepalive`` +----------------- + +Default: ``True`` + +The tcp keepalive interval to set on TCP ports. This setting can be used to tune Salt +connectivity issues in messy network environments with misbehaving firewalls. + +.. code-block:: yaml + + tcp_keepalive: True + +.. conf_minion:: tcp_keepalive_cnt + +``tcp_keepalive_cnt`` +--------------------- + +Default: ``-1`` + +Sets the ZeroMQ TCP keepalive count. May be used to tune issues with minion disconnects. + +.. code-block:: yaml + + tcp_keepalive_cnt: -1 + +.. conf_minion:: tcp_keepalive_idle + +``tcp_keepalive_idle`` +---------------------- + +Default: ``300`` + +Sets ZeroMQ TCP keepalive idle. May be used to tune issues with minion disconnects. + +.. code-block:: yaml + + tcp_keepalive_idle: 300 + +.. conf_minion:: tcp_keepalive_intvl + +``tcp_keepalive_intvl`` +----------------------- + +Default: ``-1`` + +Sets ZeroMQ TCP keepalive interval. May be used to tune issues with minion disconnects. + +.. code-block:: yaml + + tcp_keepalive_intvl': -1 + Frozen Build Update Settings ============================ @@ -2466,6 +2829,36 @@ out. winrepo_dir: 'D:\winrepo' +.. conf_minion:: winrepo_dir_ng + +``winrepo_dir_ng`` +------------------ + +.. versionadded:: 2015.8.0 + A new :ref:`ng ` repo was added. + +Default: ``/srv/salt/win/repo-ng`` + +Location on the minion where the :conf_minion:`winrepo_remotes_ng` are checked +out for 2015.8.0 and later minions. + +.. code-block:: yaml + + winrepo_dir_ng: /srv/salt/win/repo-ng + +.. conf_minion:: winrepo_source_dir + +``winrepo_source_dir`` +---------------------- + +Default: ``salt://win/repo-ng/`` + +The source location for the winrepo sls files. + +.. code-block:: yaml + + winrepo_source_dir: salt://win/repo-ng/ + .. conf_minion:: winrepo_cachefile .. conf_minion:: win_repo_cachefile @@ -2518,3 +2911,33 @@ URL of the repository: Replace ```` with the SHA1 hash of a commit ID. Specifying a commit ID is useful in that it allows one to revert back to a previous version in the event that an error is introduced in the latest revision of the repo. + +.. conf_minion:: winrepo_remotes_ng + +``winrepo_remotes_ng`` +---------------------- + +.. versionadded:: 2015.8.0 + A new :ref:`ng ` repo was added. + +Default: ``['https://github.com/saltstack/salt-winrepo-ng.git']`` + +List of git repositories to checkout and include in the winrepo for +2015.8.0 and later minions. + +.. code-block:: yaml + + winrepo_remotes_ng: + - https://github.com/saltstack/salt-winrepo-ng.git + +To specify a specific revision of the repository, prepend a commit ID to the +URL of the repository: + +.. code-block:: yaml + + winrepo_remotes_ng: + - ' https://github.com/saltstack/salt-winrepo-ng.git' + +Replace ```` with the SHA1 hash of a commit ID. Specifying a commit +ID is useful in that it allows one to revert back to a previous version in the +event that an error is introduced in the latest revision of the repo. diff --git a/doc/ref/configuration/proxy.rst b/doc/ref/configuration/proxy.rst index 974e0af8902..e55f3fc01b5 100644 --- a/doc/ref/configuration/proxy.rst +++ b/doc/ref/configuration/proxy.rst @@ -118,3 +118,53 @@ has to be closed after every command. .. code-block:: yaml proxy_always_alive: False + +``proxy_merge_pillar_in_opts`` +------------------------------ + +.. versionadded:: 2017.7.3 + +Default: ``False``. + +Wheter the pillar data to be merged into the proxy configuration options. +As multiple proxies can run on the same server, we may need different +configuration options for each, while there's one single configuration file. +The solution is merging the pillar data of each proxy minion into the opts. + +.. code-block:: yaml + + proxy_merge_pillar_in_opts: True + +``proxy_deep_merge_pillar_in_opts`` +----------------------------------- + +.. versionadded:: 2017.7.3 + +Default: ``False``. + +Deep merge of pillar data into configuration opts. +This option is evaluated only when :conf_proxy:`proxy_merge_pillar_in_opts` is +enabled. + +``proxy_merge_pillar_in_opts_strategy`` +--------------------------------------- + +.. versionadded:: 2017.7.3 + +Default: ``smart``. + +The strategy used when merging pillar configuration into opts. +This option is evaluated only when :conf_proxy:`proxy_merge_pillar_in_opts` is +enabled. + +``proxy_mines_pillar`` +---------------------- + +.. versionadded:: 2017.7.3 + +Default: ``True``. + +Allow enabling mine details using pillar data. This evaluates the mine +configuration under the pillar, for the following regular minion options that +are also equally available on the proxy minion: :conf_minion:`mine_interval`, +and :conf_minion:`mine_functions`. diff --git a/doc/ref/modules/all/index.rst b/doc/ref/modules/all/index.rst index fe7bbf5ffaf..b9f4b3f35cd 100644 --- a/doc/ref/modules/all/index.rst +++ b/doc/ref/modules/all/index.rst @@ -44,6 +44,7 @@ execution modules boto_apigateway boto_asg boto_cfn + boto_cloudfront boto_cloudtrail boto_cloudwatch boto_cloudwatch_event @@ -97,6 +98,7 @@ execution modules cytest daemontools data + datadog_api ddns deb_apache deb_postgres @@ -197,6 +199,7 @@ execution modules keyboard keystone kmod + kubernetes launchctl layman ldap3 @@ -298,6 +301,7 @@ execution modules opkg oracle osquery + out pacman pagerduty pagerduty_util @@ -323,6 +327,7 @@ execution modules ps publish puppet + purefa pushbullet pushover_notify pw_group @@ -397,7 +402,6 @@ execution modules state status statuspage - stormpath supervisord suse_apache svn @@ -415,6 +419,7 @@ execution modules test testinframod test_virtual + textfsm_mod timezone tls tomcat diff --git a/doc/ref/modules/all/salt.modules.boto_cloudfront.rst b/doc/ref/modules/all/salt.modules.boto_cloudfront.rst new file mode 100644 index 00000000000..a76ea991fcf --- /dev/null +++ b/doc/ref/modules/all/salt.modules.boto_cloudfront.rst @@ -0,0 +1,6 @@ +============================ +salt.modules.boto_cloudfront +============================ + +.. automodule:: salt.modules.boto_cloudfront + :members: diff --git a/doc/ref/modules/all/salt.modules.datadog_api.rst b/doc/ref/modules/all/salt.modules.datadog_api.rst new file mode 100644 index 00000000000..9f6afa953e3 --- /dev/null +++ b/doc/ref/modules/all/salt.modules.datadog_api.rst @@ -0,0 +1,6 @@ +======================== +salt.modules.datadog_api +======================== + +.. automodule:: salt.modules.datadog_api + :members: \ No newline at end of file diff --git a/doc/ref/modules/all/salt.modules.kernelpkg.rst b/doc/ref/modules/all/salt.modules.kernelpkg.rst index 6eada674288..9b4d8b2beb6 100644 --- a/doc/ref/modules/all/salt.modules.kernelpkg.rst +++ b/doc/ref/modules/all/salt.modules.kernelpkg.rst @@ -13,7 +13,7 @@ salt.modules.kernelpkg Execution Module Used for ============================================ ======================================== :py:mod:`~salt.modules.kernelpkg_linux_apt` Debian/Ubuntu-based distros which use - ``apt-get(8)`` for package management + ``apt-get`` for package management :py:mod:`~salt.modules.kernelpkg_linux_yum` RedHat-based distros and derivatives - using ``yum(8)`` or ``dnf(8)`` + using ``yum`` or ``dnf`` ============================================ ======================================== diff --git a/doc/ref/modules/all/salt.modules.kubernetes.rst b/doc/ref/modules/all/salt.modules.kubernetes.rst new file mode 100644 index 00000000000..a0f715d617f --- /dev/null +++ b/doc/ref/modules/all/salt.modules.kubernetes.rst @@ -0,0 +1,6 @@ +======================= +salt.modules.kubernetes +======================= + +.. automodule:: salt.modules.kubernetes + :members: diff --git a/doc/ref/modules/all/salt.modules.out.rst b/doc/ref/modules/all/salt.modules.out.rst new file mode 100644 index 00000000000..ef1a927d6e3 --- /dev/null +++ b/doc/ref/modules/all/salt.modules.out.rst @@ -0,0 +1,5 @@ +salt.modules.out module +======================= + +.. automodule:: salt.modules.out + :members: diff --git a/doc/ref/modules/all/salt.modules.purefa.rst b/doc/ref/modules/all/salt.modules.purefa.rst new file mode 100644 index 00000000000..1a1b80ce1e0 --- /dev/null +++ b/doc/ref/modules/all/salt.modules.purefa.rst @@ -0,0 +1,6 @@ +=================== +salt.modules.purefa +=================== + +.. automodule:: salt.modules.purefa + :members: diff --git a/doc/ref/modules/all/salt.modules.stormpath.rst b/doc/ref/modules/all/salt.modules.stormpath.rst deleted file mode 100644 index 708cfb2ab66..00000000000 --- a/doc/ref/modules/all/salt.modules.stormpath.rst +++ /dev/null @@ -1,6 +0,0 @@ -====================== -salt.modules.stormpath -====================== - -.. automodule:: salt.modules.stormpath - :members: diff --git a/doc/ref/modules/all/salt.modules.test.rst b/doc/ref/modules/all/salt.modules.test.rst index 5163f48e43b..0aa6ccf6a8e 100644 --- a/doc/ref/modules/all/salt.modules.test.rst +++ b/doc/ref/modules/all/salt.modules.test.rst @@ -3,4 +3,5 @@ salt.modules.test ================= .. automodule:: salt.modules.test - :members: \ No newline at end of file + :members: + :exclude-members: rand_str diff --git a/doc/ref/modules/all/salt.modules.textfsm_mod.rst b/doc/ref/modules/all/salt.modules.textfsm_mod.rst new file mode 100644 index 00000000000..7b2c64b9565 --- /dev/null +++ b/doc/ref/modules/all/salt.modules.textfsm_mod.rst @@ -0,0 +1,5 @@ +salt.modules.textfsm_mod module +=============================== + +.. automodule:: salt.modules.textfsm_mod + :members: diff --git a/doc/ref/modules/index.rst b/doc/ref/modules/index.rst index 9f81170fe4d..d6c7fe96105 100644 --- a/doc/ref/modules/index.rst +++ b/doc/ref/modules/index.rst @@ -13,7 +13,7 @@ Writing Salt execution modules is straightforward. A Salt execution module is a Python or `Cython`_ module placed in a directory called ``_modules/`` at the root of the Salt fileserver. When using the default -fileserver backend (i.e. :py:mod:`roots `), unless environments are otherwise defined in the :conf_master:`file_roots` config option, the ``_modules/`` directory would be located in ``/srv/salt/_modules`` on most systems. @@ -209,6 +209,29 @@ default configuration file for the minion contains the information and format used to pass data to the modules. :mod:`salt.modules.test`, :file:`conf/minion`. +.. _module_init: + +``__init__`` Function +--------------------- + +If you want your module to have different execution modes based on minion +configuration, you can use the ``__init__(opts)`` function to perform initial +module setup. The parameter ``opts`` is the complete minion configuration, +as also available in the ``__opts__`` dict. + +.. code-block:: python + + ''' + Cheese module initialization example + ''' + def __init__(opts): + ''' + Allow foreign imports if configured to do so + ''' + if opts.get('cheese.allow_foreign', False): + _enable_foreign_products() + + Strings and Unicode =================== @@ -273,8 +296,9 @@ module is not loaded. ``False`` lets the module perform system checks and prevent loading if dependencies are not met. Since ``__virtual__`` is called before the module is loaded, ``__salt__`` will -be unavailable as it will not have been packed into the module at this point in -time. +be unreliable as not all modules will be available at this point in time. The +``__pillar`` and ``__grains__`` :ref:`"dunder" dictionaries ` +are available however. .. note:: Modules which return a string from ``__virtual__`` that is already used by @@ -313,10 +337,14 @@ the case when the dependency is unavailable. else: return False, 'The cheese execution module cannot be loaded: enzymes unavailable.' + def slice(): + pass + .. code-block:: python ''' - Cheese state module + Cheese state module. Note that this works in state modules because it is + guaranteed that execution modules are loaded first ''' def __virtual__(): @@ -401,10 +429,33 @@ similar to the following: Confine this module to Mac OS with Homebrew. ''' - if salt.utils.which('brew') and __grains__['os'] == 'MacOS': + if salt.utils.path.which('brew') and __grains__['os'] == 'MacOS': return __virtualname__ return False +The ``__virtual__()`` function can return a ``True`` or ``False`` boolean, a tuple, +or a string. If it returns a ``True`` value, this ``__virtualname__`` module-level +attribute can be set as seen in the above example. This is the string that the module +should be referred to as. + +When ``__virtual__()`` returns a tuple, the first item should be a boolean and the +second should be a string. This is typically done when the module should not load. The +first value of the tuple is ``False`` and the second is the error message to display +for why the module did not load. + +For example: + +.. code-block:: python + + def __virtual__(): + ''' + Only load if git exists on the system + ''' + if salt.utils.path.which('git') is None: + return (False, + 'The git execution module cannot be loaded: git unavailable.') + else: + return True Documentation ============= diff --git a/doc/ref/renderers/index.rst b/doc/ref/renderers/index.rst index bea2b7798fb..8399f7f6049 100644 --- a/doc/ref/renderers/index.rst +++ b/doc/ref/renderers/index.rst @@ -146,8 +146,10 @@ Here is a simple YAML renderer example: import yaml from salt.utils.yamlloader import SaltYamlSafeLoader + from salt.ext import six + def render(yaml_data, saltenv='', sls='', **kws): - if not isinstance(yaml_data, basestring): + if not isinstance(yaml_data, six.string_types): yaml_data = yaml_data.read() data = yaml.load( yaml_data, diff --git a/doc/ref/states/aggregate.rst b/doc/ref/states/aggregate.rst index e8aa61f6892..ce25507a1ca 100644 --- a/doc/ref/states/aggregate.rst +++ b/doc/ref/states/aggregate.rst @@ -122,7 +122,7 @@ This example, simplified from the pkg state, shows how to create mod_aggregate f for chunk in chunks: # The state runtime uses "tags" to track completed jobs, it may # look familiar with the _|- - tag = salt.utils.gen_state_tag(chunk) + tag = __utils__['state.gen_tag'](chunk) if tag in running: # Already ran the pkg state, skip aggregation continue diff --git a/doc/ref/states/all/index.rst b/doc/ref/states/all/index.rst index 8c02f0d7e3f..0b681ace7e3 100644 --- a/doc/ref/states/all/index.rst +++ b/doc/ref/states/all/index.rst @@ -31,6 +31,7 @@ state modules boto_apigateway boto_asg boto_cfn + boto_cloudfront boto_cloudtrail boto_cloudwatch_alarm boto_cloudwatch_event @@ -74,6 +75,10 @@ state modules dellchassis disk docker + docker_container + docker_image + docker_network + docker_volume drac elasticsearch elasticsearch_index @@ -132,6 +137,7 @@ state modules keyboard keystone kmod + kubernetes layman ldap libcloud_dns @@ -174,6 +180,7 @@ state modules netusers network netyang + nfs_export nftables npm ntp @@ -245,7 +252,6 @@ state modules stateconf status statuspage - stormpath_account supervisord svn sysctl diff --git a/doc/ref/states/all/salt.states.boto_cloudfront.rst b/doc/ref/states/all/salt.states.boto_cloudfront.rst new file mode 100644 index 00000000000..671965b2dc2 --- /dev/null +++ b/doc/ref/states/all/salt.states.boto_cloudfront.rst @@ -0,0 +1,6 @@ +=========================== +salt.states.boto_cloudfront +=========================== + +.. automodule:: salt.states.boto_cloudfront + :members: diff --git a/doc/ref/states/all/salt.states.kubernetes.rst b/doc/ref/states/all/salt.states.kubernetes.rst new file mode 100644 index 00000000000..55bd416492a --- /dev/null +++ b/doc/ref/states/all/salt.states.kubernetes.rst @@ -0,0 +1,6 @@ +====================== +salt.states.kubernetes +====================== + +.. automodule:: salt.states.kubernetes + :members: diff --git a/doc/ref/states/all/salt.states.nfs_export.rst b/doc/ref/states/all/salt.states.nfs_export.rst new file mode 100644 index 00000000000..231992626bb --- /dev/null +++ b/doc/ref/states/all/salt.states.nfs_export.rst @@ -0,0 +1,6 @@ +====================== +salt.states.nfs_export +====================== + +.. automodule:: salt.states.nfs_export + :members: \ No newline at end of file diff --git a/doc/ref/states/all/salt.states.stormpath_account.rst b/doc/ref/states/all/salt.states.stormpath_account.rst deleted file mode 100644 index f361ec2bb75..00000000000 --- a/doc/ref/states/all/salt.states.stormpath_account.rst +++ /dev/null @@ -1,6 +0,0 @@ -============================= -salt.states.stormpath_account -============================= - -.. automodule:: salt.states.stormpath_account - :members: diff --git a/doc/ref/states/requisites.rst b/doc/ref/states/requisites.rst index 6b00b3e8566..3e3d29213c0 100644 --- a/doc/ref/states/requisites.rst +++ b/doc/ref/states/requisites.rst @@ -519,7 +519,8 @@ runas .. versionadded:: 2017.7.0 -The ``runas`` global option is used to set the user which will be used to run the command in the ``cmd.run`` module. +The ``runas`` global option is used to set the user which will be used to run +the command in the ``cmd.run`` module. .. code-block:: yaml @@ -532,6 +533,26 @@ The ``runas`` global option is used to set the user which will be used to run th In the above state, the pip command run by ``cmd.run`` will be run by the daniel user. +runas_password +~~~~~~~~~~~~~~ + +.. versionadded:: 2017.7.2 + +The ``runas_password`` global option is used to set the password used by the +runas global option. This is required by ``cmd.run`` on Windows when ``runas`` +is specified. It will be set when ``runas_password`` is defined in the state. + +.. code-block:: yaml + + run_script: + cmd.run: + - name: Powershell -NonInteractive -ExecutionPolicy Bypass -File C:\\Temp\\script.ps1 + - runas: frank + - runas_password: supersecret + +In the above state, the Powershell script run by ``cmd.run`` will be run by the +frank user with the password ``supersecret``. + .. _requisites-require-in: .. _requisites-watch-in: .. _requisites-onchanges-in: diff --git a/doc/ref/states/writing.rst b/doc/ref/states/writing.rst index 365e1b581b2..5e94c1ccc34 100644 --- a/doc/ref/states/writing.rst +++ b/doc/ref/states/writing.rst @@ -135,21 +135,30 @@ A State Module must return a dict containing the following keys/values: ``test=True``, and changes would have been made if the state was not run in test mode. - +--------------------+-----------+-----------+ - | | live mode | test mode | - +====================+===========+===========+ - | no changes | ``True`` | ``True`` | - +--------------------+-----------+-----------+ - | successful changes | ``True`` | ``None`` | - +--------------------+-----------+-----------+ - | failed changes | ``False`` | ``None`` | - +--------------------+-----------+-----------+ + +--------------------+-----------+------------------------+ + | | live mode | test mode | + +====================+===========+========================+ + | no changes | ``True`` | ``True`` | + +--------------------+-----------+------------------------+ + | successful changes | ``True`` | ``None`` | + +--------------------+-----------+------------------------+ + | failed changes | ``False`` | ``False`` or ``None`` | + +--------------------+-----------+------------------------+ .. note:: - Test mode does not predict if the changes will be successful or not. + Test mode does not predict if the changes will be successful or not, + and hence the result for pending changes is usually ``None``. -- **comment:** A string containing a summary of the result. + However, if a state is going to fail and this can be determined + in test mode without applying the change, ``False`` can be returned. + +- **comment:** A list of strings or a single string summarizing the result. + Note that support for lists of strings is available as of Salt Oxygen. + Lists of strings will be joined with newlines to form the final comment; + this is useful to allow multiple comments from subparts of a state. + Prefer to keep line lengths short (use multiple lines as needed), + and end with punctuation (e.g. a period) to delimit multiple comments. The return data can also, include the **pchanges** key, this stands for `predictive changes`. The **pchanges** key informs the State system what diff --git a/doc/spelling_wordlist.txt b/doc/spelling_wordlist.txt index f2eb2a614b6..c097ef1c591 100644 --- a/doc/spelling_wordlist.txt +++ b/doc/spelling_wordlist.txt @@ -777,8 +777,6 @@ Stateconf stderr stdin stdout -stormpath -Stormpath str strftime subfolder diff --git a/doc/topics/beacons/index.rst b/doc/topics/beacons/index.rst index 6dae8dca091..e68d3d6dd5a 100644 --- a/doc/topics/beacons/index.rst +++ b/doc/topics/beacons/index.rst @@ -34,7 +34,7 @@ monitored, everything is configured using Salt. Beacons are typically enabled by placing a ``beacons:`` top level block in ``/etc/salt/minion`` or any file in ``/etc/salt/minion.d/`` such as -``/etc/salt/minion.d/beacons.conf``: +``/etc/salt/minion.d/beacons.conf`` or add it to pillars for that minion: .. code-block:: yaml diff --git a/doc/topics/cloud/action.rst b/doc/topics/cloud/action.rst index 0e10a956531..f36211d01a1 100644 --- a/doc/topics/cloud/action.rst +++ b/doc/topics/cloud/action.rst @@ -21,7 +21,7 @@ Or you may specify a map which includes all VMs to perform the action on: $ salt-cloud -a reboot -m /path/to/mapfile -The following is a list of actions currently supported by salt-cloud: +The following is an example list of actions currently supported by ``salt-cloud``: .. code-block:: yaml @@ -36,5 +36,5 @@ The following is a list of actions currently supported by salt-cloud: - start - stop -Another useful reference for viewing more salt-cloud actions is the -:ref:Salt Cloud Feature Matrix +Another useful reference for viewing more ``salt-cloud`` actions is the +:ref:`Salt Cloud Feature Matrix `. diff --git a/doc/topics/cloud/aws.rst b/doc/topics/cloud/aws.rst index 0512da051b0..9ef8c679a57 100644 --- a/doc/topics/cloud/aws.rst +++ b/doc/topics/cloud/aws.rst @@ -78,6 +78,7 @@ parameters are discussed in more detail below. # RHEL -> ec2-user # CentOS -> ec2-user # Ubuntu -> ubuntu + # Debian -> admin # ssh_username: ec2-user @@ -470,7 +471,7 @@ EC2 API or AWS Console. By default, the spot instance type is set to 'one-time', meaning it will be launched and, if it's ever terminated for whatever reason, it will not be recreated. If you would like your spot instances to be relaunched after -a termination (by your or AWS), set the ``type`` to 'persistent'. +a termination (by you or AWS), set the ``type`` to 'persistent'. NOTE: Spot instances are a great way to save a bit of money, but you do run the risk of losing your spot instances if the current price for the diff --git a/doc/topics/cloud/azurearm.rst b/doc/topics/cloud/azurearm.rst index bad3ae2abc5..f77895475a2 100644 --- a/doc/topics/cloud/azurearm.rst +++ b/doc/topics/cloud/azurearm.rst @@ -6,7 +6,7 @@ Getting Started With Azure ARM Azure is a cloud service by Microsoft providing virtual machines, SQL services, media services, and more. Azure ARM (aka, the Azure Resource Manager) is a next -generatiom version of the Azure portal and API. This document describes how to +generation version of the Azure portal and API. This document describes how to use Salt Cloud to create a virtual machine on Azure ARM, with Salt installed. More information about Azure is located at `http://www.windowsazure.com/ @@ -15,9 +15,7 @@ More information about Azure is located at `http://www.windowsazure.com/ Dependencies ============ -* `Microsoft Azure SDK for Python `_ >= 2.0rc6 -* `Microsoft Azure Storage SDK for Python `_ >= 0.32 -* The python-requests library, for Python < 2.7.9. +* Azure Cli ```pip install 'azure-cli>=2.0.12'``` * A Microsoft Azure account * `Salt `_ @@ -238,6 +236,20 @@ iface_name Optional. The name to apply to the VM's network interface. If not supplied, the value will be set to ``-iface0``. +dns_servers +----------- +Optional. A **list** of the DNS servers to configure for the network interface +(will be set on the VM by the DHCP of the VNET). + +.. code-block:: yaml + + my-azurearm-profile: + provider: azurearm-provider + network: mynetwork + dns_servers: + - 10.1.1.4 + - 10.1.1.5 + availability_set ---------------- Optional. If set, the VM will be added to the specified availability set. diff --git a/doc/topics/cloud/cloud.rst b/doc/topics/cloud/cloud.rst index 0ca59b83f33..568866a7e47 100644 --- a/doc/topics/cloud/cloud.rst +++ b/doc/topics/cloud/cloud.rst @@ -146,24 +146,24 @@ library. The following two lines set up the imports: .. code-block:: python from salt.cloud.libcloudfuncs import * # pylint: disable=W0614,W0401 - from salt.utils import namespaced_function + import salt.utils And then a series of declarations will make the necessary functions available within the cloud module. .. code-block:: python - get_size = namespaced_function(get_size, globals()) - get_image = namespaced_function(get_image, globals()) - avail_locations = namespaced_function(avail_locations, globals()) - avail_images = namespaced_function(avail_images, globals()) - avail_sizes = namespaced_function(avail_sizes, globals()) - script = namespaced_function(script, globals()) - destroy = namespaced_function(destroy, globals()) - list_nodes = namespaced_function(list_nodes, globals()) - list_nodes_full = namespaced_function(list_nodes_full, globals()) - list_nodes_select = namespaced_function(list_nodes_select, globals()) - show_instance = namespaced_function(show_instance, globals()) + get_size = salt.utils.namespaced_function(get_size, globals()) + get_image = salt.utils.namespaced_function(get_image, globals()) + avail_locations = salt.utils.namespaced_function(avail_locations, globals()) + avail_images = salt.utils.namespaced_function(avail_images, globals()) + avail_sizes = salt.utils.namespaced_function(avail_sizes, globals()) + script = salt.utils.namespaced_function(script, globals()) + destroy = salt.utils.namespaced_function(destroy, globals()) + list_nodes = salt.utils.namespaced_function(list_nodes, globals()) + list_nodes_full = salt.utils.namespaced_function(list_nodes_full, globals()) + list_nodes_select = salt.utils.namespaced_function(list_nodes_select, globals()) + show_instance = salt.utils.namespaced_function(show_instance, globals()) If necessary, these functions may be replaced by removing the appropriate declaration line, and then adding the function as normal. @@ -183,7 +183,7 @@ imports should be absent from the Salt Cloud module. A good example of a non-libcloud driver is the DigitalOcean driver: -https://github.com/saltstack/salt/tree/develop/salt/cloud/clouds/digital_ocean.py +https://github.com/saltstack/salt/tree/develop/salt/cloud/clouds/digitalocean.py The ``create()`` Function ------------------------- diff --git a/doc/topics/cloud/config.rst b/doc/topics/cloud/config.rst index a1739bc7015..173ea4e6928 100644 --- a/doc/topics/cloud/config.rst +++ b/doc/topics/cloud/config.rst @@ -56,6 +56,24 @@ settings can be placed in the provider or profile: sls_list: - web + +When salt cloud creates a new minon, it can automatically add grain information +to the minion configuration file identifying the sources originally used +to define it. + +The generated grain information will appear similar to: + +.. code-block:: yaml + + grains: + salt-cloud: + driver: ec2 + provider: my_ec2:ec2 + profile: ec2-web + +The generation of the salt-cloud grain can be surpressed by the +option ``enable_cloud_grains: 'False'`` in the cloud configuration file. + Cloud Configuration Syntax ========================== @@ -97,7 +115,7 @@ Using the example configuration above: .. note:: - Salt Cloud provider configurations within ``/etc/cloud.provider.d/ should not + Salt Cloud provider configurations within ``/etc/cloud.provider.d/`` should not specify the ``providers`` starting key. It is also possible to have multiple cloud configuration blocks within the same alias block. @@ -371,7 +389,6 @@ both. compute_name: cloudServersOpenStack protocol: ipv4 compute_region: DFW - protocol: ipv4 user: myuser tenant: 5555555 password: mypass @@ -427,7 +444,7 @@ under the API Access tab. .. code-block:: yaml my-digitalocean-config: - driver: digital_ocean + driver: digitalocean personal_access_token: xxx location: New York 1 diff --git a/doc/topics/cloud/deploy.rst b/doc/topics/cloud/deploy.rst index 689a5d6030f..00a8ff2979c 100644 --- a/doc/topics/cloud/deploy.rst +++ b/doc/topics/cloud/deploy.rst @@ -89,7 +89,7 @@ functions include: A good, well commented example of this process is the Fedora deployment script: -https://github.com/saltstack/salt-cloud/blob/master/saltcloud/deploy/Fedora.sh +https://github.com/saltstack/salt/blob/develop/salt/cloud/deploy/Fedora.sh A number of legacy deploy scripts are included with the release tarball. None of them are as functional or complete as Salt Bootstrap, and are still included diff --git a/doc/topics/cloud/digitalocean.rst b/doc/topics/cloud/digitalocean.rst index e89faf1a5c7..dd7c76d91fb 100644 --- a/doc/topics/cloud/digitalocean.rst +++ b/doc/topics/cloud/digitalocean.rst @@ -19,7 +19,7 @@ under the "SSH Keys" section. # /etc/salt/cloud.providers.d/ directory. my-digitalocean-config: - driver: digital_ocean + driver: digitalocean personal_access_token: xxx ssh_key_file: /path/to/ssh/key/file ssh_key_names: my-key-name,my-key-name-2 @@ -63,7 +63,7 @@ command: # salt-cloud --list-locations my-digitalocean-config my-digitalocean-config: ---------- - digital_ocean: + digitalocean: ---------- Amsterdam 1: ---------- @@ -87,7 +87,7 @@ command: # salt-cloud --list-sizes my-digitalocean-config my-digitalocean-config: ---------- - digital_ocean: + digitalocean: ---------- 512MB: ---------- @@ -117,7 +117,7 @@ command: # salt-cloud --list-images my-digitalocean-config my-digitalocean-config: ---------- - digital_ocean: + digitalocean: ---------- 10.1: ---------- @@ -142,7 +142,7 @@ Profile Specifics: ssh_username ------------ -If using a FreeBSD image from Digital Ocean, you'll need to set the ``ssh_username`` +If using a FreeBSD image from DigitalOcean, you'll need to set the ``ssh_username`` setting to ``freebsd`` in your profile configuration. .. code-block:: yaml diff --git a/doc/topics/cloud/function.rst b/doc/topics/cloud/function.rst index 3368cd820b3..c83487d36b8 100644 --- a/doc/topics/cloud/function.rst +++ b/doc/topics/cloud/function.rst @@ -26,5 +26,5 @@ gathering information about instances on a provider basis: $ salt-cloud -f list_nodes_full linode $ salt-cloud -f list_nodes_select linode -Another useful reference for viewing salt-cloud functions is the +Another useful reference for viewing ``salt-cloud`` functions is the :ref:`Salt Cloud Feature Matrix `. diff --git a/doc/topics/cloud/index.rst b/doc/topics/cloud/index.rst index 9cbb789b0c7..eff8e2aa8f2 100644 --- a/doc/topics/cloud/index.rst +++ b/doc/topics/cloud/index.rst @@ -64,7 +64,9 @@ automatically installed salt-cloud for you. Use your distribution's package manager to install the ``salt-cloud`` package from the same repo that you used to install Salt. These repos will automatically be setup by Salt Bootstrap. -If there is no salt-cloud package, install with ``pip install salt-cloud``. +Alternatively, the ``-L`` option can be passed to the `Salt Bootstrap`_ script when +installing Salt. The ``-L`` option will install ``salt-cloud`` and the required +``libcloud`` package. .. _`Salt Bootstrap`: https://github.com/saltstack/salt-bootstrap @@ -117,6 +119,7 @@ Cloud Provider Specifics Getting Started With Libvirt Getting Started With Linode Getting Started With LXC + Getting Started With OneAndOne Getting Started With OpenNebula Getting Started With OpenStack Getting Started With Parallels diff --git a/doc/topics/cloud/joyent.rst b/doc/topics/cloud/joyent.rst index 8d4ca2c100d..a56d5338402 100644 --- a/doc/topics/cloud/joyent.rst +++ b/doc/topics/cloud/joyent.rst @@ -49,7 +49,7 @@ Set up an initial profile at ``/etc/salt/cloud.profiles`` or in the .. code-block:: yaml - joyent_512 + joyent_512: provider: my-joyent-config size: g4-highcpu-512M image: ubuntu-16.04 diff --git a/doc/topics/cloud/libvirt.rst b/doc/topics/cloud/libvirt.rst index 48ba5ba8d46..7fb0ff25ef8 100644 --- a/doc/topics/cloud/libvirt.rst +++ b/doc/topics/cloud/libvirt.rst @@ -8,10 +8,14 @@ libvirt with qemu-kvm. http://www.libvirt.org/ -Dependencies +Host Dependencies ============ * libvirt >= 1.2.18 (older might work) +Salt-Cloud Dependencies +============ +* libvirt-python + Provider Configuration ====================== diff --git a/doc/topics/cloud/misc.rst b/doc/topics/cloud/misc.rst index 3ef1518923c..2e44aa5612f 100644 --- a/doc/topics/cloud/misc.rst +++ b/doc/topics/cloud/misc.rst @@ -386,3 +386,42 @@ script, a cloud profile using ``file_map`` might look like: file_map: /local/path/to/custom/script: /remote/path/to/use/custom/script /local/path/to/package: /remote/path/to/store/package + +Running Pre-Flight Commands +=========================== + +.. versionadded:: Oxygen + +To execute specified preflight shell commands on a VM before the deploy script is +run, use the ``preflight_cmds`` option. These must be defined as a list in a cloud +configuration file. For example: + +.. code-block:: yaml + + my-cloud-profile: + provider: linode-config + image: Ubuntu 16.04 LTS + size: Linode 2048 + preflight_cmds: + - whoami + - echo 'hello world!' + +These commands will run in sequence **before** the bootstrap script is executed. + +Force Minion Config +=================== + +.. versionadded:: Oxygen + +The ``force_minion_config`` option requests the bootstrap process to overwrite +an existing minion configuration file and public/private key files. +Default: False + +This might be important for drivers (such as ``saltify``) which are expected to +take over a connection from a former salt master. + +.. code-block:: yaml + + my_saltify_provider: + driver: saltify + force_minion_config: true diff --git a/doc/topics/cloud/oneandone.rst b/doc/topics/cloud/oneandone.rst new file mode 100644 index 00000000000..e0e52e739ab --- /dev/null +++ b/doc/topics/cloud/oneandone.rst @@ -0,0 +1,146 @@ +========================== +Getting Started With 1and1 +========================== + +1&1 is one of the world’s leading Web hosting providers. 1&1 currently offers +a wide range of Web hosting products, including email solutions and high-end +servers in 10 different countries including Germany, Spain, Great Britain +and the United States. From domains to 1&1 MyWebsite to eBusiness solutions +like Cloud Hosting and Web servers for complex tasks, 1&1 is well placed to deliver +a high quality service to its customers. All 1&1 products are hosted in +1&1‘s high-performance, green data centers in the USA and Europe. + +Dependencies +============ + +* 1and1 >= 1.2.0 + +Configuration +============= + +* Using the new format, set up the cloud configuration at + ``/etc/salt/cloud.providers`` or + ``/etc/salt/cloud.providers.d/oneandone.conf``: + +.. code-block:: yaml + + my-oneandone-config: + driver: oneandone + + # Set the location of the salt-master + # + minion: + master: saltmaster.example.com + + # Configure oneandone authentication credentials + # + api_token: + ssh_private_key: /path/to/id_rsa + ssh_public_key: /path/to/id_rsa.pub + +Authentication +============== + +The ``api_key`` is used for API authorization. This token can be obtained +from the CloudPanel in the Management section below Users. + +Profiles +======== + +Here is an example of a profile: + +.. code-block:: yaml + + oneandone_fixed_size: + provider: my-oneandone-config + description: Small instance size server + fixed_instance_size: S + appliance_id: 8E3BAA98E3DFD37857810E0288DD8FBA + + oneandone_custom_size: + provider: my-oneandone-config + description: Custom size server + vcore: 2 + cores_per_processor: 2 + ram: 8 + appliance_id: 8E3BAA98E3DFD37857810E0288DD8FBA + hdds: + - + is_main: true + size: 20 + - + is_main: false + size: 20 + +The following list explains some of the important properties. + +fixed_instance_size_id + When creating a server, either ``fixed_instance_size_id`` or custom hardware params + containing ``vcore``, ``cores_per_processor``, ``ram``, and ``hdds`` must be provided. + Can be one of the IDs listed among the output of the following command: + +.. code-block:: bash + + salt-cloud --list-sizes oneandone + +vcore + Total amount of processors. + +cores_per_processor + Number of cores per processor. + +ram + RAM memory size in GB. + +hdds + Hard disks. + +appliance_id + ID of the image that will be installed on server. + Can be one of the IDs listed in the output of the following command: + +.. code-block:: bash + + salt-cloud --list-images oneandone + +datacenter_id + ID of the datacenter where the server will be created. + Can be one of the IDs listed in the output of the following command: + +.. code-block:: bash + + salt-cloud --list-locations oneandone + +description + Description of the server. + +password + Password of the server. Password must contain more than 8 characters + using uppercase letters, numbers and other special symbols. + +power_on + Power on server after creation. Default is set to true. + +firewall_policy_id + Firewall policy ID. If it is not provided, the server will assign + the best firewall policy, creating a new one if necessary. If the parameter + is sent with a 0 value, the server will be created with all ports blocked. + +ip_id + IP address ID. + +load_balancer_id + Load balancer ID. + +monitoring_policy_id + Monitoring policy ID. + +deploy + Set to False if Salt should not be installed on the node. + +wait_for_timeout + The timeout to wait in seconds for provisioning resources such as servers. + The default wait_for_timeout is 15 minutes. + +For more information concerning cloud profiles, see :ref:`here +`. diff --git a/doc/topics/cloud/profitbricks.rst b/doc/topics/cloud/profitbricks.rst index 4189f6107dc..b25fb6d082e 100644 --- a/doc/topics/cloud/profitbricks.rst +++ b/doc/topics/cloud/profitbricks.rst @@ -34,8 +34,8 @@ Configuration # username: user@domain.com password: 123456 - # datacenter is the UUID of a pre-existing virtual data center. - datacenter: 9e6709a0-6bf9-4bd6-8692-60349c70ce0e + # datacenter_id is the UUID of a pre-existing virtual data center. + datacenter_id: 9e6709a0-6bf9-4bd6-8692-60349c70ce0e # Connect to public LAN ID 1. public_lan: 1 ssh_public_key: /path/to/id_rsa.pub diff --git a/doc/topics/cloud/qs.rst b/doc/topics/cloud/qs.rst index 18434e249b4..9b37c3ea16f 100644 --- a/doc/topics/cloud/qs.rst +++ b/doc/topics/cloud/qs.rst @@ -12,7 +12,9 @@ automatically installed salt-cloud for you. Use your distribution's package manager to install the ``salt-cloud`` package from the same repo that you used to install Salt. These repos will automatically be setup by Salt Bootstrap. -If there is no salt-cloud package, install with ``pip install salt-cloud``. +Alternatively, the ``-L`` option can be passed to the `Salt Bootstrap`_ script when +installing Salt. The ``-L`` option will install ``salt-cloud`` and the required +``libcloud`` package. .. _`Salt Bootstrap`: https://github.com/saltstack/salt-bootstrap diff --git a/doc/topics/cloud/saltify.rst b/doc/topics/cloud/saltify.rst index 2e8fa156612..dda98015221 100644 --- a/doc/topics/cloud/saltify.rst +++ b/doc/topics/cloud/saltify.rst @@ -16,7 +16,7 @@ The Saltify driver has no external dependencies. Configuration ============= -Because the Saltify driver does not use an actual cloud provider host, it has a +Because the Saltify driver does not use an actual cloud provider host, it can have a simple provider configuration. The only thing that is required to be set is the driver name, and any other potentially useful information, like the location of the salt-master: @@ -31,6 +31,12 @@ the salt-master: master: 111.222.333.444 provider: saltify +However, if you wish to use the more advanced capabilities of salt-cloud, such as +rebooting, listing, and disconnecting machines, then the salt master must fill +the role usually performed by a vendor's cloud management system. In order to do +that, you must configure your salt master as a salt-api server, and supply credentials +to use it. (See ``salt-api setup`` below.) + Profiles ======== @@ -72,6 +78,30 @@ to it can be verified with Salt: salt my-machine test.ping +Destroy Options +--------------- + +For obvious reasons, the ``destroy`` action does not actually vaporize hardware. +If the salt master is connected using salt-api, it can tear down parts of +the client machines. It will remove the client's key from the salt master, +and will attempt the following options: + +.. code-block:: yaml + + - remove_config_on_destroy: true + # default: true + # Deactivate salt-minion on reboot and + # delete the minion config and key files from its ``/etc/salt`` directory, + # NOTE: If deactivation is unsuccessful (older Ubuntu machines) then when + # salt-minion restarts it will automatically create a new, unwanted, set + # of key files. The ``force_minion_config`` option must be used in that case. + + - shutdown_on_destroy: false + # default: false + # send a ``shutdown`` command to the client. + +.. versionadded:: Oxygen + Using Map Files --------------- The settings explained in the section above may also be set in a map file. An @@ -135,3 +165,67 @@ Return values: - ``True``: Credential verification succeeded - ``False``: Credential verification succeeded - ``None``: Credential verification was not attempted. + +Provisioning salt-api +===================== + +In order to query or control minions it created, saltify needs to send commands +to the salt master. It does that using the network interface to salt-api. + +The salt-api is not enabled by default. The following example will provide a +simple installation. + +.. code-block:: yaml + + # file /etc/salt/cloud.profiles.d/my_saltify_profiles.conf + hw_41: # a theoretical example hardware machine + ssh_host: 10.100.9.41 # the hard address of your target + ssh_username: vagrant # a user name which has passwordless sudo + password: vagrant # on your target machine + provider: my_saltify_provider + + +.. code-block:: yaml + + # file /etc/salt/cloud.providers.d/saltify_provider.conf + my_saltify_provider: + driver: saltify + eauth: pam + username: vagrant # supply some sudo-group-member's name + password: vagrant # and password on the salt master + minion: + master: 10.100.9.5 # the hard address of the master + + +.. code-block:: yaml + + # file /etc/salt/master.d/auth.conf + # using salt-api ... members of the 'sudo' group can do anything ... + external_auth: + pam: + sudo%: + - .* + - '@wheel' + - '@runner' + - '@jobs' + + +.. code-block:: yaml + + # file /etc/salt/master.d/api.conf + # see https://docs.saltstack.com/en/latest/ref/netapi/all/salt.netapi.rest_cherrypy.html + rest_cherrypy: + host: localhost + port: 8000 + ssl_crt: /etc/pki/tls/certs/localhost.crt + ssl_key: /etc/pki/tls/certs/localhost.key + thread_pool: 30 + socket_queue_size: 10 + + +Start your target machine as a Salt minion named "node41" by: + +.. code-block:: bash + + $ sudo salt-cloud -p hw_41 node41 + diff --git a/doc/topics/cloud/windows.rst b/doc/topics/cloud/windows.rst index 84ccd05c745..c48b403b72a 100644 --- a/doc/topics/cloud/windows.rst +++ b/doc/topics/cloud/windows.rst @@ -48,6 +48,15 @@ from saltstack.com: .. __: https://repo.saltstack.com/windows/ +.. _new-pywinrm: + +Self Signed Certificates with WinRM +=================================== + +Salt-Cloud can use versions of ``pywinrm<=0.1.1`` or ``pywinrm>=0.2.1``. + +For versions greater than `0.2.1`, ``winrm_verify_ssl`` needs to be set to +`False` if the certificate is self signed and not verifiable. Firewall Settings ================= @@ -179,7 +188,8 @@ The default Windows user is `Administrator`, and the default Windows password is blank. If WinRM is to be used ``use_winrm`` needs to be set to `True`. ``winrm_port`` -can be used to specify a custom port (must be HTTPS listener). +can be used to specify a custom port (must be HTTPS listener). And +``winrm_verify_ssl`` can be set to `False` to use a self signed certificate. Auto-Generated Passwords on EC2 diff --git a/doc/topics/development/contributing.rst b/doc/topics/development/contributing.rst index 57e3b3c677c..79413976829 100644 --- a/doc/topics/development/contributing.rst +++ b/doc/topics/development/contributing.rst @@ -260,6 +260,21 @@ The Salt development team will back-port bug fixes made to ``develop`` to the current release branch if the contributor cannot create the pull request against that branch. +Release Branches +---------------- + +For each release, a branch will be created when the SaltStack release team is +ready to tag. The release branch is created from the parent branch and will be +the same name as the tag minus the ``v``. For example, the ``2017.7.1`` release +branch was created from the ``2017.7`` parent branch and the ``v2017.7.1`` +release was tagged at the ``HEAD`` of the ``2017.7.1`` branch. This branching +strategy will allow for more stability when there is a need for a re-tag during +the testing phase of the release process. + +Once the release branch is created, the fixes required for a given release, as +determined by the SaltStack release team, will be added to this branch. All +commits in this branch will be merged forward into the parent branch as well. + Keeping Salt Forks in Sync ========================== diff --git a/doc/topics/development/conventions/formulas.rst b/doc/topics/development/conventions/formulas.rst index 29f968faf97..4fc8f722f38 100644 --- a/doc/topics/development/conventions/formulas.rst +++ b/doc/topics/development/conventions/formulas.rst @@ -214,18 +214,34 @@ Writing Formulas Each Formula is a separate repository in the `saltstack-formulas`_ organization on GitHub. -.. note:: Get involved creating new Formulas +Get involved creating new Formulas +---------------------------------- - The best way to create new Formula repositories for now is to create a - repository in your own account on GitHub and notify a SaltStack employee - when it is ready. We will add you to the contributors team on the - `saltstack-formulas`_ organization and help you transfer the repository - over. Ping a SaltStack employee on IRC (``#salt`` on Freenode) or send an - email to the `salt-users`_ mailing list. +The best way to create new Formula repositories for now is to create a +repository in your own account on GitHub and notify a SaltStack employee when +it is ready. We will add you to the Contributors team on the +`saltstack-formulas`_ organization and help you transfer the repository over. +Ping a SaltStack employee on IRC (``#salt`` on Freenode) or send an email to +the `salt-users`_ mailing list. - There are a lot of repositories in that organization! Team members can - manage which repositories they are subscribed to on GitHub's watching page: - https://github.com/watching. +There are a lot of repositories in that organization! Team members can manage +which repositories they are subscribed to on GitHub's watching page: +https://github.com/watching. + +Members of the Contributors team are welcome to participate in reviewing pull +requests across the Organization. Some repositories will have regular +contributors and some repositories will not. As you get involved in a +repository be sure to communicate with any other contributors there on pull +requests that are large or have breaking changes. + +In general it is best to have another Contributor review and merge any pull +requests that you open. Feel free to `at-mention`__ other regular contributors +to a repository and request a review. However, there are a lot of formula +repositories so if a repository does not yet have regular contributors or if +your pull request has stayed open for more than a couple days feel free to +"selfie-merge" your own pull request. + +__: https://help.github.com/articles/basic-writing-and-formatting-syntax/#mentioning-users-and-teams Style ----- diff --git a/doc/topics/development/deprecations.rst b/doc/topics/development/deprecations.rst index c13a5b32cc7..fc89c106bda 100644 --- a/doc/topics/development/deprecations.rst +++ b/doc/topics/development/deprecations.rst @@ -18,10 +18,10 @@ on the significance and complexity of the changes required by the user. Salt feature releases are based on the Periodic Table. Any new features going into the develop branch will be named after the next element in the Periodic -Table. For example, Beryllium was the feature release name of the develop branch -before the 2015.8 branch was tagged. At that point in time, any new features going -into the develop branch after 2015.8 was branched were part of the Boron feature -release. +Table. For example, Beryllium was the feature release name of the develop +branch before the 2015.8 branch was tagged. At that point in time, any new +features going into the develop branch after 2015.8 was branched were part of +the Boron feature release. A deprecation warning should be in place for at least two major releases before the deprecated code and its accompanying deprecation warning are removed. More @@ -29,14 +29,14 @@ time should be given for more complex changes. For example, if the current release under development is ``Sodium``, the deprecated code and associated warnings should remain in place and warn for at least ``Aluminum``. -To help in this deprecation task, salt provides :func:`salt.utils.warn_until -`. The idea behind this helper function is to show the -deprecation warning to the user until salt reaches the provided version. Once -that provided version is equaled :func:`salt.utils.warn_until -` will raise a :py:exc:`RuntimeError` making salt stop -its execution. This stoppage is unpleasant and will remind the developer that -the deprecation limit has been reached and that the code can then be safely -removed. +To help in this deprecation task, salt provides +:func:`salt.utils.versions.warn_until `. The +idea behind this helper function is to show the deprecation warning to the user +until salt reaches the provided version. Once that provided version is equaled +:func:`salt.utils.versions.warn_until ` will +raise a :py:exc:`RuntimeError` making salt stop its execution. This stoppage is +unpleasant and will remind the developer that the deprecation limit has been +reached and that the code can then be safely removed. Consider the following example: @@ -44,7 +44,7 @@ Consider the following example: def some_function(bar=False, foo=None): if foo is not None: - salt.utils.warn_until( + salt.utils.versions.warn_until( 'Aluminum', 'The \'foo\' argument has been deprecated and its ' 'functionality removed, as such, its usage is no longer ' diff --git a/doc/topics/development/dunder_dictionaries.rst b/doc/topics/development/dunder_dictionaries.rst index 64df937a152..798e3e4a5fd 100644 --- a/doc/topics/development/dunder_dictionaries.rst +++ b/doc/topics/development/dunder_dictionaries.rst @@ -1,3 +1,5 @@ +.. _dunder-dictionaries: + =================== Dunder Dictionaries =================== diff --git a/doc/topics/development/tests/index.rst b/doc/topics/development/tests/index.rst index 2679ebf0fd7..f743b1da09e 100644 --- a/doc/topics/development/tests/index.rst +++ b/doc/topics/development/tests/index.rst @@ -219,7 +219,7 @@ the default cloud provider configuration file for DigitalOcean looks like this: .. code-block:: yaml digitalocean-config: - driver: digital_ocean + driver: digitalocean client_key: '' api_key: '' location: New York 1 @@ -230,7 +230,7 @@ must be provided: .. code-block:: yaml digitalocean-config: - driver: digital_ocean + driver: digitalocean client_key: wFGEwgregeqw3435gDger api_key: GDE43t43REGTrkilg43934t34qT43t4dgegerGEgg location: New York 1 diff --git a/doc/topics/development/tests/integration.rst b/doc/topics/development/tests/integration.rst index 6eb1b40480f..79c1eb3a487 100644 --- a/doc/topics/development/tests/integration.rst +++ b/doc/topics/development/tests/integration.rst @@ -371,6 +371,7 @@ on a minion event bus. .. code-block:: python import tests.integration as integration + import salt.utils.event class TestEvent(integration.SaltEventAssertsMixin): ''' @@ -443,7 +444,7 @@ to test states: from tests.support.mixins import SaltReturnAssertsMixin # Import salt libs - import salt.utils + import salt.utils.files HFILE = os.path.join(TMP, 'hosts') @@ -470,7 +471,7 @@ to test states: ip = '10.10.10.10' ret = self.run_state('host.present', name=name, ip=ip) self.assertSaltTrueReturn(ret) - with salt.utils.fopen(HFILE) as fp_: + with salt.utils.files.fopen(HFILE) as fp_: output = fp_.read() self.assertIn('{0}\t\t{1}'.format(ip, name), output) @@ -540,7 +541,7 @@ provider configuration file in the integration test file directory located at ``tests/integration/files/conf/cloud.*.d/``. The following is an example of the default profile configuration file for Digital -Ocean, located at: ``tests/integration/files/conf/cloud.profiles.d/digital_ocean.conf``: +Ocean, located at: ``tests/integration/files/conf/cloud.profiles.d/digitalocean.conf``: .. code-block:: yaml @@ -556,12 +557,12 @@ be provided by the user by editing the provider configuration file before runnin tests. The following is an example of the default provider configuration file for Digital -Ocean, located at: ``tests/integration/files/conf/cloud.providers.d/digital_ocean.conf``: +Ocean, located at: ``tests/integration/files/conf/cloud.providers.d/digitalocean.conf``: .. code-block:: yaml digitalocean-config: - driver: digital_ocean + driver: digitalocean client_key: '' api_key: '' location: New York 1 diff --git a/doc/topics/development/tests/unit.rst b/doc/topics/development/tests/unit.rst index d90c770178b..595a14d6367 100644 --- a/doc/topics/development/tests/unit.rst +++ b/doc/topics/development/tests/unit.rst @@ -319,7 +319,7 @@ function into ``__salt__`` that's actually a MagicMock instance. def show_patch(self): with patch.dict(my_module.__salt__, - {'function.to_replace': MagicMock()}: + {'function.to_replace': MagicMock()}): # From this scope, carry on with testing, with a modified __salt__! diff --git a/doc/topics/eauth/index.rst b/doc/topics/eauth/index.rst index 05a405a5601..d484136a0f4 100644 --- a/doc/topics/eauth/index.rst +++ b/doc/topics/eauth/index.rst @@ -93,6 +93,26 @@ By user, by minion: : - +By user, by runner/wheel: + +.. code-block:: yaml + + external_auth: + : + : + <@runner or @wheel>: + - + +By user, by runner+wheel module: + +.. code-block:: yaml + + external_auth: + : + : + <@module_name>: + - + Groups ------ @@ -127,6 +147,14 @@ Positional arguments or keyword arguments to functions can also be whitelisted. kwargs: 'kwa': 'kwa.*' 'kwb': 'kwb' + - '@runner': + - 'runner_mod.*': + args: + - 'a.*' + - 'b.*' + kwargs: + 'kwa': 'kwa.*' + 'kwb': 'kwb' The rules: @@ -246,6 +274,10 @@ Server configuration values and their defaults: # Redhat Identity Policy Audit auth.ldap.freeipa: False + +Authenticating to the LDAP Server ++++++++++++++++++++++++++++++++++ + There are two phases to LDAP authentication. First, Salt authenticates to search for a users' Distinguished Name and group membership. The user it authenticates as in this phase is often a special LDAP system user with read-only access to the LDAP directory. After Salt searches the directory to determine the actual user's DN @@ -276,6 +308,10 @@ substitutes the ``{{ username }}`` value for the username when querying LDAP auth.ldap.filter: uid={{ username }} + +Determining Group Memberships (OpenLDAP / non-Active Directory) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + For OpenLDAP, to determine group membership, one can specify an OU that contains group data. This is prepended to the basedn to create a search path. Then the results are filtered against ``auth.ldap.groupclass``, default @@ -285,7 +321,16 @@ the results are filtered against ``auth.ldap.groupclass``, default auth.ldap.groupou: Groups -When using the `ldap('DC=domain,DC=com')` eauth operator, sometimes the records returned +Note that as of 2017.7, auth.ldap.groupclass can refer to either a groupclass or an objectClass. +For some LDAP servers (notably OpenLDAP without the ``memberOf`` overlay enabled) to determine group +membership we need to know both the ``objectClass`` and the ``memberUid`` attributes. Usually for these +servers you will want a ``auth.ldap.groupclass`` of ``posixGroup`` and an ``auth.ldap.groupattribute`` of +``memberUid``. + +LDAP servers with the ``memberOf`` overlay will have entries similar to ``auth.ldap.groupclass: person`` and +``auth.ldap.groupattribute: memberOf``. + +When using the ``ldap('DC=domain,DC=com')`` eauth operator, sometimes the records returned from LDAP or Active Directory have fully-qualified domain names attached, while minion IDs instead are simple hostnames. The parameter below allows the administrator to strip off a certain set of domain names so the hostnames looked up in the directory service @@ -295,8 +340,9 @@ can match the minion IDs. auth.ldap.minion_stripdomains: ['.external.bigcorp.com', '.internal.bigcorp.com'] -Active Directory ----------------- + +Determining Group Memberships (Active Directory) +++++++++++++++++++++++++++++++++++++++++++++++++ Active Directory handles group membership differently, and does not utilize the ``groupou`` configuration variable. AD needs the following options in @@ -361,5 +407,5 @@ be part of the eAuth definition, they can be specified like this: - ldap('DC=corp,DC=example,DC=com'): - test.echo -The string inside `ldap()` above is any valid LDAP/AD tree limiter. `OU=` in +The string inside ``ldap()`` above is any valid LDAP/AD tree limiter. ``OU=`` in particular is permitted as long as it would return a list of computer objects. diff --git a/doc/topics/installation/debian.rst b/doc/topics/installation/debian.rst index 36a47fa8ffe..369991ebaa4 100644 --- a/doc/topics/installation/debian.rst +++ b/doc/topics/installation/debian.rst @@ -18,7 +18,7 @@ Installation from official Debian and Raspbian repositories is described Installation from the Official SaltStack Repository =================================================== -Packages for Debian 8 (Jessie) and Debian 7 (Wheezy) are available in the +Packages for Debian 9 (Stretch) and Debian 8 (Jessie) are available in the Official SaltStack repository. Instructions are at https://repo.saltstack.com/#debian. diff --git a/doc/topics/installation/eos.rst b/doc/topics/installation/eos.rst new file mode 100644 index 00000000000..c8c7787be70 --- /dev/null +++ b/doc/topics/installation/eos.rst @@ -0,0 +1,154 @@ +========================================= +Arista EOS Salt minion installation guide +========================================= + +The Salt minion for Arista EOS is distributed as a SWIX extension and can be installed directly on the switch. The EOS network operating system is based on old Fedora distributions and the installation of the ``salt-minion`` requires backports. This SWIX extension contains the necessary backports, together with the Salt basecode. + +.. note:: + + This SWIX extension has been tested on Arista DCS-7280SE-68-R, running EOS 4.17.5M and vEOS 4.18.3F. + +Important Notes +=============== + +This package is in beta, make sure to test it carefully before running it in production. + +If confirmed working correctly, please report and add a note on this page with the platform model and EOS version. + +If you want to uninstall this package, please refer to the uninstalling_ section. + +Installation from the Official SaltStack Repository +=================================================== + +Download the swix package and save it to flash. + +.. code-block:: bash + + veos#copy https://salt-eos.netops.life/salt-eos-latest.swix flash: + veos#copy https://salt-eos.netops.life/startup.sh flash: + +Install the Extension +===================== + +Copy the Salt package to extension + +.. code-block:: bash + + veos#copy flash:salt-eos-latest.swix extension: + +Install the SWIX + +.. code-block:: bash + + veos#extension salt-eos-latest.swix force + +Verify the installation + +.. code-block:: bash + + veos#show extensions | include salt-eos + salt-eos-2017-07-19.swix 1.0.11/1.fc25 A, F 27 + +Change the Salt master IP address or FQDN, by edit the variable (SALT_MASTER) + +.. code-block:: bash + + veos#bash vi /mnt/flash/startup.sh + +Make sure you enable the eAPI with unix-socket + +.. code-block:: bash + + veos(config)#management api http-commands + protocol unix-socket + no shutdown + +Post-installation tasks +======================= + +Generate Keys and host record and start Salt minion + +.. code-block:: bash + + veos#bash + #sudo /mnt/flash/startup.sh + +``salt-minion`` should be running + +Copy the installed extensions to boot-extensions + +.. code-block:: bash + + veos#copy installed-extensions boot-extensions + +Apply event-handler to let EOS start salt-minion during boot-up + +.. code-block:: bash + + veos(config)#event-handler boot-up-script + trigger on-boot + action bash sudo /mnt/flash/startup.sh + +For more specific installation details of the ``salt-minion``, please refer to :ref:`Configuring Salt`. + +.. _uninstalling: + +Uninstalling +============ + +If you decide to uninstall this package, the following steps are recommended for safety: + +1. Remove the extension from boot-extensions + +.. code-block:: bash + + veos#bash rm /mnt/flash/boot-extensions + +2. Remove the extension from extensions folder + +.. code-block:: bash + + veos#bash rm /mnt/flash/.extensions/salt-eos-latest.swix + +2. Remove boot-up script + +.. code-block:: bash + + veos(config)#no event-handler boot-up-script + +Additional Information +====================== + +This SWIX extension contains the following RPM packages: + +.. code-block:: text + + libsodium-1.0.11-1.fc25.i686.rpm + libstdc++-6.2.1-2.fc25.i686.rpm + openpgm-5.2.122-6.fc24.i686.rpm + python-Jinja2-2.8-0.i686.rpm + python-PyYAML-3.12-0.i686.rpm + python-babel-0.9.6-5.fc18.noarch.rpm + python-backports-1.0-3.fc18.i686.rpm + python-backports-ssl_match_hostname-3.4.0.2-1.fc18.noarch.rpm + python-backports_abc-0.5-0.i686.rpm + python-certifi-2016.9.26-0.i686.rpm + python-chardet-2.0.1-5.fc18.noarch.rpm + python-crypto-1.4.1-1.noarch.rpm + python-crypto-2.6.1-1.fc18.i686.rpm + python-futures-3.1.1-1.noarch.rpm + python-jtextfsm-0.3.1-0.noarch.rpm + python-kitchen-1.1.1-2.fc18.noarch.rpm + python-markupsafe-0.18-1.fc18.i686.rpm + python-msgpack-python-0.4.8-0.i686.rpm + python-napalm-base-0.24.3-1.noarch.rpm + python-napalm-eos-0.6.0-1.noarch.rpm + python-netaddr-0.7.18-0.noarch.rpm + python-pyeapi-0.7.0-0.noarch.rpm + python-salt-2017.7.0_1414_g2fb986f-1.noarch.rpm + python-singledispatch-3.4.0.3-0.i686.rpm + python-six-1.10.0-0.i686.rpm + python-tornado-4.4.2-0.i686.rpm + python-urllib3-1.5-7.fc18.noarch.rpm + python2-zmq-15.3.0-2.fc25.i686.rpm + zeromq-4.1.4-5.fc25.i686.rpm diff --git a/doc/topics/installation/index.rst b/doc/topics/installation/index.rst index 0ee4b38d7a5..2f96830c4e8 100644 --- a/doc/topics/installation/index.rst +++ b/doc/topics/installation/index.rst @@ -46,6 +46,7 @@ These guides go into detail how to install Salt on a given platform. arch debian + eos fedora freebsd gentoo diff --git a/doc/topics/jinja/index.rst b/doc/topics/jinja/index.rst index e028403a061..d57fbcbb043 100644 --- a/doc/topics/jinja/index.rst +++ b/doc/topics/jinja/index.rst @@ -150,6 +150,22 @@ starts at the root of the state tree or pillar. .. _`macro`: http://jinja.pocoo.org/docs/templates/#macros .. _`whitespace control`: http://jinja.pocoo.org/docs/templates/#whitespace-control +Errors +====== + +Saltstack allows to raise custom errors using the ``raise`` jinja function. + +.. code-block:: jinja + + {{ raise('Custom Error') }} + +When rendering the template containing the above statement, a ``TemplateError`` +exception is raised, causing the rendering to fail with the following message: + +.. code-block:: text + + TemplateError: Custom Error + Filters ======= @@ -335,7 +351,7 @@ Returns: .. versionadded:: 2017.7.0 -Wraps a text around quoutes. +This text will be wrapped in quotes. .. jinja_ref:: regex_search @@ -750,19 +766,43 @@ Returns: Check a whitelist and/or blacklist to see if the value matches it. -Example: +This filter can be used with either a whitelist or a blacklist individually, +or a whitelist and a blacklist can be passed simultaneously. + +If whitelist is used alone, value membership is checked against the +whitelist only. If the value is found, the function returns ``True``. +Otherwise, it returns ``False``. + +If blacklist is used alone, value membership is checked against the +blacklist only. If the value is found, the function returns ``False``. +Otherwise, it returns ``True``. + +If both a whitelist and a blacklist are provided, value membership in the +blacklist will be examined first. If the value is not found in the blacklist, +then the whitelist is checked. If the value isn't found in the whitelist, +the function returns ``False``. + +Whitelist Example: .. code-block:: jinja - {{ 5 | check_whitelist_blacklist(whitelist=[5, 6, 7]) }} - {{ 5 | check_whitelist_blacklist(blacklist=[5, 6, 7]) }} + {{ 5 | check_whitelist_blacklist(whitelist=[5, 6, 7]) }} Returns: .. code-block:: python - True + True +Blacklist Example: + +.. code-block:: jinja + + {{ 5 | check_whitelist_blacklist(blacklist=[5, 6, 7]) }} + +.. code-block:: python + + False .. jinja_ref:: date_format @@ -788,12 +828,14 @@ Returns: 08.03.2017 17:00 -.. jinja_ref:: str_to_num +.. jinja_ref:: to_num -``str_to_num`` --------------- +``to_num`` +---------- .. versionadded:: 2017.7.0 +.. versionadded:: Oxygen + Renamed from ``str_to_num`` to ``to_num``. Converts a string to its numerical value. @@ -801,7 +843,7 @@ Example: .. code-block:: jinja - {{ '5' | str_to_num }} + {{ '5' | to_num }} Returns: @@ -825,6 +867,13 @@ Example: {{ 'wall of text' | to_bytes }} +.. note:: + + This option may have adverse effects when using the default renderer, ``yaml_jinja``. + This is due to the fact that YAML requires proper handling in regard to special + characters. Please see the section on :ref:`YAML ASCII support ` + in the :ref:`YAML Idiosyncracies ` documentation for more + information. .. jinja_ref:: json_decode_list @@ -870,22 +919,28 @@ Returns: {'a': 'b'} -.. jinja_ref:: rand_str +.. jinja_ref:: random_hash -``rand_str`` ------------- +``random_hash`` +--------------- .. versionadded:: 2017.7.0 +.. versionadded:: Oxygen + Renamed from ``rand_str`` to ``random_hash`` to more accurately describe + what the filter does. -Generate a random string and applies a hash. Default hashing: md5. +Generates a random number between 1 and the number passed to the filter, and +then hashes it. The default hash type is the one specified by the minion's +:conf_minion:`hash_type` config option, but an alternate hash type can be +passed to the filter as an argument. Example: .. code-block:: jinja - {% set passwd_length = 17 %} - {{ passwd_length | rand_str }} - {{ passwd_length | rand_str('sha512') }} + {% set num_range = 99999999 %} + {{ num_range | random_hash }} + {{ num_range | random_hash('sha512') }} Returns: @@ -1186,7 +1241,7 @@ Example: .. code-block:: jinja - {{ ['192.168.0.1', 'foo', 'bar', 'fe80::'] | ipv4 }} + {{ ['192.168.0.1', 'foo', 'bar', 'fe80::'] | ipv6 }} Returns: @@ -1202,7 +1257,12 @@ Returns: .. versionadded:: 2017.7.0 -Return the list of hosts within a networks. +Return the list of hosts within a networks. This utility works for both IPv4 and IPv6. + +.. note:: + + When running this command with a large IPv6 network, the command will + take a long time to gather all of the hosts. Example: @@ -1224,7 +1284,7 @@ Returns: .. versionadded:: 2017.7.0 -Return the size of the network. +Return the size of the network. This utility works for both IPv4 and IPv6. Example: @@ -1284,6 +1344,13 @@ Example: {{ '00:11:22:33:44:55' | mac_str_to_bytes }} +.. note:: + + This option may have adverse effects when using the default renderer, ``yaml_jinja``. + This is due to the fact that YAML requires proper handling in regard to special + characters. Please see the section on :ref:`YAML ASCII support ` + in the :ref:`YAML Idiosyncracies ` documentation for more + information. .. jinja_ref:: dns_check @@ -1684,6 +1751,23 @@ Will insert the following message in the minion logs: .. jinja_ref:: custom-execution-modules +Python Methods +==================== + +A powerful feature of jinja that is only hinted at in the official jinja +documentation is that you can use the native python methods of the +variable type. Here is the python documentation for `string methods`_. + +.. code-block:: jinja + + {% set hostname,domain = grains.id.partition('.')[::2] %}{{ hostname }} + +.. code-block:: jinja + + {% set strings = grains.id.split('-') %}{{ strings[0] }} + +.. _`string methods`: https://docs.python.org/2/library/stdtypes.html#string-methods + Custom Execution Modules ======================== diff --git a/doc/topics/network_automation/index.rst b/doc/topics/network_automation/index.rst index bbc5a1382a4..0ba8bea94fc 100644 --- a/doc/topics/network_automation/index.rst +++ b/doc/topics/network_automation/index.rst @@ -40,7 +40,7 @@ and the interaction with the network device does not rely on a particular vendor .. image:: /_static/napalm_logo.png -Beginning with Nitrogen, the NAPALM modules have been transformed so they can +Beginning with 2017.7.0, the NAPALM modules have been transformed so they can run in both proxy and regular minions. That means, if the operating system allows, the salt-minion package can be installed directly on the network gear. The interface between the network operating system and Salt in that case would @@ -50,7 +50,7 @@ For example, if the user installs the salt-minion on a Arista switch, the only requirement is `napalm-eos `_. -The following modules are available in Nitrogen: +The following modules are available in 2017.7.0: - :mod:`NAPALM grains ` - :mod:`NET execution module ` - Networking basic diff --git a/doc/topics/proxyminion/index.rst b/doc/topics/proxyminion/index.rst index 80806c646ac..bfa5383155f 100644 --- a/doc/topics/proxyminion/index.rst +++ b/doc/topics/proxyminion/index.rst @@ -89,7 +89,7 @@ they are being loaded for the correct proxytype, example below: Only work on proxy ''' try: - if salt.utils.is_proxy() and \ + if salt.utils.platform.is_proxy() and \ __opts__['proxy']['proxytype'] == 'ssh_sample': return __virtualname__ except KeyError: @@ -156,20 +156,23 @@ will need to be restarted to pick up any changes. A corresponding utility funct ``saltutil.sync_proxymodules``, has been added to sync these modules to minions. In addition, a salt.utils helper function called `is_proxy()` was added to make -it easier to tell when the running minion is a proxy minion. +it easier to tell when the running minion is a proxy minion. **NOTE: This +function was renamed to salt.utils.platform.is_proxy() for the Oxygen release** New in 2015.8 ------------- -Starting with the 2015.8 release of Salt, proxy processes are no longer forked off from a controlling minion. -Instead, they have their own script ``salt-proxy`` which takes mostly the same arguments that the -standard Salt minion does with the addition of ``--proxyid``. This is the id that the salt-proxy will -use to identify itself to the master. Proxy configurations are still best kept in Pillar and their format -has not changed. +Starting with the 2015.8 release of Salt, proxy processes are no longer forked +off from a controlling minion. Instead, they have their own script +``salt-proxy`` which takes mostly the same arguments that the standard Salt +minion does with the addition of ``--proxyid``. This is the id that the +salt-proxy will use to identify itself to the master. Proxy configurations are +still best kept in Pillar and their format has not changed. -This change allows for better process control and logging. Proxy processes can now be listed with standard -process management utilities (``ps`` from the command line). Also, a full Salt minion is no longer -required (though it is still strongly recommended) on machines hosting proxies. +This change allows for better process control and logging. Proxy processes can +now be listed with standard process management utilities (``ps`` from the +command line). Also, a full Salt minion is no longer required (though it is +still strongly recommended) on machines hosting proxies. Getting Started @@ -619,9 +622,10 @@ in the proxymodule itself. This might be useful if a proxymodule author wants t all the code for the proxy interface in the same place instead of splitting it between the proxy and grains directories. -This function will only be called automatically if the configuration variable ``proxy_merge_grains_in_module`` -is set to True in the proxy configuration file (default ``/etc/salt/proxy``). This -variable defaults to ``True`` in the release code-named *2017.7.0*. +This function will only be called automatically if the configuration variable +``proxy_merge_grains_in_module`` is set to True in the proxy configuration file +(default ``/etc/salt/proxy``). This variable defaults to ``True`` in the +release code-named *2017.7.0*. .. code: python:: @@ -640,7 +644,7 @@ variable defaults to ``True`` in the release code-named *2017.7.0*. def __virtual__(): try: - if salt.utils.is_proxy() and __opts__['proxy']['proxytype'] == 'rest_sample': + if salt.utils.platform.is_proxy() and __opts__['proxy']['proxytype'] == 'rest_sample': return __virtualname__ except KeyError: pass @@ -708,7 +712,7 @@ Example from ``salt/grains/rest_sample.py``: def __virtual__(): try: - if salt.utils.is_proxy() and __opts__['proxy']['proxytype'] == 'rest_sample': + if salt.utils.platform.is_proxy() and __opts__['proxy']['proxytype'] == 'rest_sample': return __virtualname__ except KeyError: pass diff --git a/doc/topics/releases/2015.8.0.rst b/doc/topics/releases/2015.8.0.rst index ac9b03ff11a..5a34397a929 100644 --- a/doc/topics/releases/2015.8.0.rst +++ b/doc/topics/releases/2015.8.0.rst @@ -106,7 +106,7 @@ bringing with it the ability to access authenticated repositories. Using the new features will require updates to the git ext_pillar configuration, further details can be found in the :ref:`pillar.git_pillar -` docs. +` docs. .. _pygit2: https://github.com/libgit2/pygit2 diff --git a/doc/topics/releases/2016.11.6.rst b/doc/topics/releases/2016.11.6.rst index b1e80b3c7e5..dda24ba5c7e 100644 --- a/doc/topics/releases/2016.11.6.rst +++ b/doc/topics/releases/2016.11.6.rst @@ -9,40 +9,212 @@ Changes for v2016.11.5..v2016.11.6 Extended changelog courtesy of Todd Stansell (https://github.com/tjstansell/salt-changelogs): -*Generated at: 2017-06-14T19:58:30Z* +*Generated at: 2017-06-21T21:12:22Z* Statistics: -- Total Merges: **117** -- Total Issue references: **57** -- Total PR references: **141** +- Total Merges: **137** +- Total Issue references: **65** +- Total PR references: **167** Changes: +- **PR** `#41861`_: (*twangboy*) Fix problems with get_rule and delete_rule + @ *2017-06-20T20:37:23Z* + + * afc61ffe63 Merge pull request `#41861`_ from twangboy/fix_win_firewall + * 78892074f5 Fix problems with get_rule and delete_rule + +- **PR** `#41787`_: (*skizunov*) Fix `#41778`_ + @ *2017-06-20T20:11:23Z* + + - **ISSUE** `#41778`_: (*frogunder*) 2016.11.6 - TCP Transport gives Exception + | refs: `#41787`_ `#41787`_ + - **PR** `#41436`_: (*skizunov*) TCP transport: Fix occasional errors when using salt command + | refs: `#41787`_ `#41787`_ `#41787`_ + * 938d4fddf1 Merge pull request `#41787`_ from skizunov/develop3 + * 2ffd20cede Fix `#41778`_ + +- **PR** `#41812`_: (*skizunov*) TCP: Fix salt-master in bad state if remote side closed connection + @ *2017-06-20T19:46:53Z* + + * 03b6ae5ea8 Merge pull request `#41812`_ from skizunov/develop4 + * 736420eb83 TCP: Fix salt-master in bad state if remote side closed connection + +- **PR** `#41857`_: (*dmurphy18*) Modified support for deprecated netstat being removed by utilizing ss + @ *2017-06-20T18:46:27Z* + + * cf2252bcea Merge pull request `#41857`_ from dmurphy18/netstat_fix + * 017fbdbc53 Modified support for deprecated netstat being removed by utilizing ss + +- **PR** `#41837`_: (*rallytime*) Add fingerpint_hash_type option to ssh_auth state and related functions + @ *2017-06-20T18:14:53Z* + + - **ISSUE** `#40878`_: (*joewreschnig*) SSH modules spam warning about MD5 fingerprints when there aren't any + | refs: `#41837`_ `#41837`_ + - **ISSUE** `#40005`_: (*vutny*) `ssh_known_hosts.present` does not support SHA256 key fingerprints + | refs: `#40543`_ + - **PR** `#40543`_: (*rallytime*) Add the "fingerprint_hash_type" option to ssh state and module + | refs: `#41837`_ + * 12ec5f9f23 Merge pull request `#41837`_ from rallytime/`fix-40878`_ + * 48ff5d2a62 Add fingerpint_hash_type option to ssh_auth state and related functions + +- **PR** `#41839`_: (*cro*) Extend proxy to jinja + @ *2017-06-19T23:03:00Z* + + * e7fc30f482 Merge pull request `#41839`_ from cro/extend_proxy_to_jinja + * 172d3520ea Merge branch 'extend_proxy_to_jinja' of github.com:cro/salt into extend_proxy_to_jinja + + * 2e4a0633da Extend __proxy__ to jinja as `proxy` (like __salt__->salt) + + * 2ffad2af35 Extend __proxy__ to jinja as `proxy` (like __salt__->salt) + +- **PR** `#41786`_: (*whiteinge*) Runner arg parsing regressions + @ *2017-06-19T23:00:07Z* + + - **ISSUE** `#41733`_: (*sumeetisp*) Salt Rest Api call + | refs: `#41786`_ + - **ISSUE** `#40845`_: (*e-senthilkumar*) /jobs call is broken in 2016.11.4 + | refs: `#41786`_ + - **ISSUE** `#38962`_: (*gstachowiak*) Broken /jobs in salt-api in salt 2016.11.1 (Carbon) + | refs: `#39472`_ + - **PR** `#39472`_: (*whiteinge*) Update _reformat_low to not run kwarg dicts through parse_input + | refs: `#41786`_ + * 58387b127a Merge pull request `#41786`_ from whiteinge/runner-arg-parsing-regressions + * bf15c0bb5f Restore sending __current_eauth_* through to the function + + * 6be975da2c Fix regressions from not calling load_args_and_kwargs + + * 9d1cc1a176 Add test to check that runners ignore invalid kwargs + +- **PR** `#41776`_: (*gtmanfred*) npm 5.0.0 added a second line after fsevents + @ *2017-06-19T16:53:43Z* + + * be0e9abedb Merge pull request `#41776`_ from gtmanfred/2016.11 + * 733a2279ca npm 5.0.0 added a second line after fsevents + +- **PR** `#41783`_: (*rallytime*) Add a bunch of config options to the various master/minion files that are missing + @ *2017-06-19T16:42:54Z* + + - **ISSUE** `#32400`_: (*rallytime*) Document Default Config Values + | refs: `#41783`_ + * d94d4e4d19 Merge pull request `#41783`_ from rallytime/config-doc-updates + * c828ad803a Add a bunch of config options to the various master/minion files that are missing + +- **PR** `#41816`_: (*twangboy*) Upgrade psutil to version 5.2.2 + @ *2017-06-17T01:51:29Z* + + * 2c681887d3 Merge pull request `#41816`_ from twangboy/update_psutil_req + * 8b4e3ad77d Upgrade psutil to version 5.2.2 + +- **PR** `#41803`_: (*terminalmage*) Don't log an error when no top.sls is found + @ *2017-06-16T22:49:08Z* + + - **ISSUE** `#41785`_: (*UtahDave*) Using master tops without a top.sls file causes extra errors in minion log + | refs: `#41803`_ + * 3e5fe7ca4b Merge pull request `#41803`_ from terminalmage/issue41785 + * f9f4d49f05 Don't log an error when no top.sls is found + +- **PR** `#41801`_: (*terminalmage*) Don't take hostname from name param when creating docker container (2016.11 branch) + @ *2017-06-16T17:02:02Z* + + * d12bc4ee68 Merge pull request `#41801`_ from terminalmage/issue41781-2016.11 + * 8236d3e1c3 Don't take hostname from name param when creating docker container (2016.11 branch) + +- **PR** `#41768`_: (*rallytime*) Manually back-port the changes in PR `#41615`_ + @ *2017-06-15T20:41:45Z* + + - **PR** `#41615`_: (*Ch3LL*) Fix get_hwclock_aix test on MacOSX + | refs: `#41768`_ + * 87e2e72d94 Merge pull request `#41768`_ from rallytime/`bp-41615`_ + * b6cc0b6bf0 Manually backport the changes in PR `#41615`_ + +- **PR** `#41740`_: (*terminalmage*) Fix spurious error when glob/regex used in publisher_acl + @ *2017-06-15T15:14:56Z* + + * 36cb223ab2 Merge pull request `#41740`_ from terminalmage/zd1532 + * e5f3d08751 Fix spurious error when glob/regex used in publisher_acl + +- **PR** `#41749`_: (*terminalmage*) Fix bug in pkg_resource.parse_targets when version passed + @ *2017-06-15T15:05:52Z* + + * 126a36747b Merge pull request `#41749`_ from terminalmage/parse_targets + * 698806fb09 No need to manually create pkg_params dict when name and version passed + + * 7484bcc6c6 parse_targets: include version in packed return data + +- **PR** `#41753`_: (*rallytime*) Back-port `#41449`_ to 2016.11 + @ *2017-06-14T22:16:10Z* + + - **PR** `#41449`_: (*sebw*) Fix state "svn.latest" diff output in test mode + | refs: `#41753`_ + * 2c24012ded Merge pull request `#41753`_ from rallytime/`bp-41449`_ + * fae41c2875 Adjusting SVN unit test + + * eac6b151eb Improved SVN output in test mode + +- **PR** `#41750`_: (*rallytime*) [2016.11] Merge forward from 2016.3 to 2016.11 + @ *2017-06-14T22:15:41Z* + + - **PR** `#41695`_: (*xiaoanyunfei*) fix max RecursionError, Ellipsis + - **PR** `#41585`_: (*cro*) Sign_minion_messages support + * e685858269 Merge pull request `#41750`_ from rallytime/merge-2016.11 + * 89834e49c2 Merge branch '2016.3' into '2016.11' + + * c5a79a1ea6 Merge pull request `#41695`_ from xiaoanyunfei/fixRecursion + + * c54fde69a0 fix spell error + + * bc855b4711 fix swallow exception + + * c0b1f57fc0 add exception + + * aacf9f0a44 fix maximum recursion depth exceeded + + * 4b43ba3366 Merge pull request `#41585`_ from cro/sign_minion_messages + + * 628f709c3c Correct test--caching requires files on disk but the test just supplies what would have been read from disk. + + * 687872a488 Lint + + * dadf4b851c Add documentation to the example master and minion configuration files. Move minion event signing to a saner place. Enable dropping messages when signature does not verify or when minion is not adding the signature to its payloads. + + * e44673cdae Add caching of key. + + * c3917d1e91 Fat finger fix. + + * 3b9326fda7 Sign_minion_messages support + +- **PR** `#41756`_: (*Ch3LL*) Add Change Log to 2016.11.6 Release Notes + @ *2017-06-14T20:57:08Z* + + * 36cc8f1e35 Merge pull request `#41756`_ from Ch3LL/2016.11.6_release + * fa368f21ac Add Change Log to 2016.11.6 Release Notes + - **PR** `#41692`_: (*rallytime*) Add boto and boto3 version dependencies to boto_vpc state docs @ *2017-06-14T19:05:07Z* - **ISSUE** `#40155`_: (*grichmond-salt*) State module boto_vpc not working with boto 2 | refs: `#41692`_ - * edcafc6 Merge pull request `#41692`_ from rallytime/`fix-40155`_ - * 539c1b0 Add boto and boto3 version dependencies to boto_vpc state docs + * edcafc6a26 Merge pull request `#41692`_ from rallytime/`fix-40155`_ + * 539c1b0692 Add boto and boto3 version dependencies to boto_vpc state docs - **PR** `#40902`_: (*lorengordon*) Removes duplicates when merging pillar lists and adds pillar.get override for pillar_merge_lists @ *2017-06-14T18:39:09Z* - **ISSUE** `#39918`_: (*kivoli*) Enabling list merging leads to multiplying of unique list items | refs: `#40902`_ - * bdaeb55 Merge pull request `#40902`_ from lorengordon/pillar-get-merge-lists - * 6e35673 Preserves order when removing duplicates + * bdaeb55a77 Merge pull request `#40902`_ from lorengordon/pillar-get-merge-lists + * 6e35673fe3 Preserves order when removing duplicates - * 18eda70 Updates list merge tests to check for sorted, unique lists + * 18eda7084c Updates list merge tests to check for sorted, unique lists - * 74bf91f Sorts the list when removing duplicates + * 74bf91f99e Sorts the list when removing duplicates - * 26a4b1b Adds pillar.get param to control list merge/overwrite behavior + * 26a4b1b17f Adds pillar.get param to control list merge/overwrite behavior - * ed04bae Removes duplicate values when merging lists + * ed04bae94c Removes duplicate values when merging lists - **PR** `#41723`_: (*rallytime*) Support apache-libcloud work-around for issue `#32743`_ for versions older than 2.0.0 @ *2017-06-14T17:13:38Z* @@ -51,98 +223,98 @@ Changes: | refs: `#41723`_ `#41723`_ - **PR** `#40837`_: (*tonybaloney*) Upgrade apache-libcloud package dependency for 2.0 | refs: `#41723`_ `#41723`_ - * 203ec67 Merge pull request `#41723`_ from rallytime/libcloud-support - * 1e9a060 Bump version check down to 1.4.0 and use distutils.version lib + * 203ec6730f Merge pull request `#41723`_ from rallytime/libcloud-support + * 1e9a06000b Bump version check down to 1.4.0 and use distutils.version lib - * a30f654 Support apache-libcloud work-around for issue `#32743`_ for versions older than 2.0.0 + * a30f654b04 Support apache-libcloud work-around for issue `#32743`_ for versions older than 2.0.0 - **PR** `#41655`_: (*Enquier*) Allow Nova cloud module to set a specific floating ip address @ *2017-06-14T16:44:05Z* - **ISSUE** `#41654`_: (*Enquier*) Nova Cloud module doesn't work for python-novaclient 8.0.0+ | refs: `#41655`_ - * 62dbf50 Merge pull request `#41655`_ from Enquier/nova-cloud-set_ip_address - * 293bc64 Removed empty debug log + * 62dbf5083c Merge pull request `#41655`_ from Enquier/nova-cloud-set_ip_address + * 293bc64158 Removed empty debug log - * 3d9871f Cleaning up, removing debugging tests + * 3d9871fe11 Cleaning up, removing debugging tests - * c78e5fe Fixing error message + * c78e5feea9 Fixing error message - * 404dffb Debugging variable format + * 404dffb6b8 Debugging variable format - * 6fa3b97 removing string call + * 6fa3b976a5 removing string call - * 005995e modifying variable calls + * 005995e1b0 modifying variable calls - * 9e5e7a3 Testing variable changes + * 9e5e7a38ec Testing variable changes - * 05e240f Debugging Format of floating_ip variable + * 05e240f37f Debugging Format of floating_ip variable - * 366aca0 Adding Max version check for Nova since Cloud no longer operates at higher versions + * 366aca00a8 Adding Max version check for Nova since Cloud no longer operates at higher versions - * 6f66c9d Fixing response of floating_ip_show to align with other floating ip's. Spelling fix + * 6f66c9d10c Fixing response of floating_ip_show to align with other floating ip's. Spelling fix - * 58459ad Adding ability to set a Floating IP by a specific IP address + * 58459adbe8 Adding ability to set a Floating IP by a specific IP address - **PR** `#41731`_: (*terminalmage*) Clarify that archive_format is required pre-2016.11.0 @ *2017-06-14T15:05:21Z* - * 82eab84 Merge pull request `#41731`_ from terminalmage/docs - * d3f4ea1 Clarify that archive_format is required pre-2016.11.0 + * 82eab84883 Merge pull request `#41731`_ from terminalmage/docs + * d3f4ea1a84 Clarify that archive_format is required pre-2016.11.0 - **PR** `#41663`_: (*skizunov*) Don't invoke lspci if enable_lspci is False @ *2017-06-13T21:19:42Z* - * b6d27be Merge pull request `#41663`_ from skizunov/develop3 - * 154d6ce Don't invoke lspci if enable_lspci is False + * b6d27beac2 Merge pull request `#41663`_ from skizunov/develop3 + * 154d6ce59e Don't invoke lspci if enable_lspci is False - **PR** `#41693`_: (*rallytime*) Document available kwargs for ec2.create_volume function @ *2017-06-13T19:51:10Z* - **ISSUE** `#40446`_: (*sumeetisp*) [Documentation] include list of kwargs for ec2.create_volume in cloud driver | refs: `#41693`_ - * 46b8d5d Merge pull request `#41693`_ from rallytime/`fix-40446`_ - * 569eb2b Document available kwargs for ec2.create_volume function + * 46b8d5dc4b Merge pull request `#41693`_ from rallytime/`fix-40446`_ + * 569eb2bf7e Document available kwargs for ec2.create_volume function - **PR** `#41696`_: (*terminalmage*) Handle a few edge/corner cases with non-string input to cmd.run @ *2017-06-13T18:48:56Z* - **ISSUE** `#41691`_: (*jdonofrio728*) Can't pass integers as cmd.run environment variables | refs: `#41696`_ - * aab55d3 Merge pull request `#41696`_ from terminalmage/issue41691 - * 0623e40 Apparently some funcs are passing tuples to cmd.run_* + * aab55d304a Merge pull request `#41696`_ from terminalmage/issue41691 + * 0623e40d33 Apparently some funcs are passing tuples to cmd.run_* - * cdbfb94 Handle a few edge/corner cases with non-string input to cmd.run + * cdbfb94cfe Handle a few edge/corner cases with non-string input to cmd.run - **PR** `#41697`_: (*terminalmage*) Resubmit `#41545`_ against 2016.11 branch @ *2017-06-13T16:10:37Z* - * 97897d7 Merge pull request `#41697`_ from terminalmage/pr-41545 - * faaacf8 Use error name instead of error number + * 97897d7a7a Merge pull request `#41697`_ from terminalmage/pr-41545 + * faaacf88bf Use error name instead of error number - * 7eacda5 Make print_cli resilient on slow systems + * 7eacda5cbf Make print_cli resilient on slow systems - **PR** `#41711`_: (*rallytime*) Update deprecated version info in manage.bootstrap func for root_user @ *2017-06-13T16:04:32Z* - **ISSUE** `#40605`_: (*sumeetisp*) Salt-run manage.bootstrap | refs: `#41711`_ - * 09260d7 Merge pull request `#41711`_ from rallytime/`fix-40605`_ - * 903c2ff Update deprecated version info in manage.bootstrap fucn for root_user + * 09260d7c08 Merge pull request `#41711`_ from rallytime/`fix-40605`_ + * 903c2ffca5 Update deprecated version info in manage.bootstrap fucn for root_user - **PR** `#41658`_: (*garethgreenaway*) Fixes to the salt scheduler @ *2017-06-13T16:00:57Z* - **ISSUE** `#39668`_: (*mirceaulinic*) Master scheduled job not recorded on the event bus | refs: `#41658`_ - * d563b3e Merge pull request `#41658`_ from garethgreenaway/39668_schedule_runners_fire_events - * d688a1c Enable jobs scheduled on the master to fire their return data to the event bus + * d563b3e345 Merge pull request `#41658`_ from garethgreenaway/39668_schedule_runners_fire_events + * d688a1cd88 Enable jobs scheduled on the master to fire their return data to the event bus - **PR** `#41706`_: (*twangboy*) Add missing batch files @ *2017-06-13T15:32:53Z* - * 3c3b934 Merge pull request `#41706`_ from twangboy/batch_files - * 0d4be02 Add batch files for master + * 3c3b9343b7 Merge pull request `#41706`_ from twangboy/batch_files + * 0d4be0220b Add batch files for master - **PR** `#41710`_: (*rallytime*) [2016.11] Merge forward from 2016.3 to 2016.11 @ *2017-06-13T15:11:38Z* @@ -153,24 +325,24 @@ Changes: | refs: `#41707`_ - **PR** `#41707`_: (*terminalmage*) Update version in master-tops docs - **PR** `#41689`_: (*yannj-fr*) Fix `#41688`_ : fix mkfs command linux-swap support - * 1afc4ad Merge pull request `#41710`_ from rallytime/merge-2016.11 - * 5150916 Merge branch '2016.3' into '2016.11' + * 1afc4adc5a Merge pull request `#41710`_ from rallytime/merge-2016.11 + * 5150916556 Merge branch '2016.3' into '2016.11' - * 5058b0d Merge pull request `#41707`_ from terminalmage/master-tops-docs + * 5058b0de1f Merge pull request `#41707`_ from terminalmage/master-tops-docs - * 6ec9dfb Update version in master-tops docs + * 6ec9dfb7f3 Update version in master-tops docs - * 1c1964d Merge pull request `#41689`_ from yannj-fr/`fix-41688`_ + * 1c1964d807 Merge pull request `#41689`_ from yannj-fr/`fix-41688`_ - * a47eddc Fix `#41688`_ : fix mkfs command linux-swap support + * a47eddccd2 Fix `#41688`_ : fix mkfs command linux-swap support - **PR** `#41702`_: (*gtmanfred*) npm 5 and greater requires --force for cache clean @ *2017-06-12T23:21:56Z* - * 5d763b9 Merge pull request `#41702`_ from gtmanfred/2016.11 - * 8bd19fc fix version number + * 5d763b9b7f Merge pull request `#41702`_ from gtmanfred/2016.11 + * 8bd19fcc17 fix version number - * 0fa380f npm 5 and greater requires --force for cache clean + * 0fa380f75c npm 5 and greater requires --force for cache clean - **PR** `#41704`_: (*rallytime*) Back-port `#41670`_ to 2016.11 @ *2017-06-12T23:20:31Z* @@ -179,126 +351,126 @@ Changes: | refs: `#41670`_ - **PR** `#41670`_: (*yannj-fr*) fixes `#41668`_ ntfs case problem in parted module | refs: `#41704`_ - * f6519e7 Merge pull request `#41704`_ from rallytime/`bp-41670`_ - * 8afc879 fixes `#41668`_ ntfs case problem in parted module + * f6519e7f80 Merge pull request `#41704`_ from rallytime/`bp-41670`_ + * 8afc8792d1 fixes `#41668`_ ntfs case problem in parted module - **PR** `#41700`_: (*terminalmage*) roots: return actual link destination when listing symlinks @ *2017-06-12T22:07:03Z* - **ISSUE** `#39939`_: (*martinschipper*) Relative symlinks are changed with file.recurse 2016.11.3 | refs: `#41700`_ - * 0b89377 Merge pull request `#41700`_ from terminalmage/issue39939 - * bdbb265 roots: return actual link destination when listing symlinks + * 0b89377dce Merge pull request `#41700`_ from terminalmage/issue39939 + * bdbb265a0b roots: return actual link destination when listing symlinks - **PR** `#41699`_: (*rallytime*) Remove note about version incompatibility with salt-cloud @ *2017-06-12T19:44:28Z* - * 7cf47f9 Merge pull request `#41699`_ from rallytime/troubleshooting-doc-update - * c91ca5f Remove note about version incompatibility with salt-cloud + * 7cf47f9651 Merge pull request `#41699`_ from rallytime/troubleshooting-doc-update + * c91ca5f809 Remove note about version incompatibility with salt-cloud - **PR** `#41694`_: (*rallytime*) Add ipcidr options to "Allowed Values" list in LocalClient expr_form docs @ *2017-06-12T19:06:16Z* - **ISSUE** `#40410`_: (*DarrenDai*) Targeting Minions by IP Range via restful API doesn't work | refs: `#41694`_ - * d68a631 Merge pull request `#41694`_ from rallytime/`fix-40410`_ - * 6de9da1 Add ipcidr options to "Allowed Values" list in LocalClient expr_form docs + * d68a6316b8 Merge pull request `#41694`_ from rallytime/`fix-40410`_ + * 6de9da1d5d Add ipcidr options to "Allowed Values" list in LocalClient expr_form docs - **PR** `#41659`_: (*lubyou*) Use re.escape to escape paths before handing them to re.match @ *2017-06-12T18:10:53Z* - **ISSUE** `#41365`_: (*lubyou*) file.managed chokes on windows paths when source_hash is set to the URI of a file that contains source hash strings | refs: `#41659`_ - * 80d4a3a Merge pull request `#41659`_ from lubyou/41365-fix-file-managed - * d49a157 Use re.escape to escape paths, before handing them to re.match + * 80d4a3ab98 Merge pull request `#41659`_ from lubyou/41365-fix-file-managed + * d49a1579b0 Use re.escape to escape paths, before handing them to re.match - * ac240fa use correct variable + * ac240facca use correct variable - * c777eba Use re.escape to escape paths, before handing them to re.match + * c777eba2c1 Use re.escape to escape paths, before handing them to re.match - **PR** `#41661`_: (*whiteinge*) Add note about avoiding the `-i` flag for the /keys endpoint @ *2017-06-09T15:03:40Z* - * 564d5fd Merge pull request `#41661`_ from whiteinge/rest_cherrypy-keys-headers - * a66ffc9 Add note about avoiding the `-i` flag for the /keys endpoint + * 564d5fd9d3 Merge pull request `#41661`_ from whiteinge/rest_cherrypy-keys-headers + * a66ffc9d3e Add note about avoiding the `-i` flag for the /keys endpoint - **PR** `#41660`_: (*garethgreenaway*) Fix to modules/aptpkg.py for unheld @ *2017-06-09T14:53:23Z* - **ISSUE** `#41651`_: (*Sakorah*) pkg.installed fails when unholding and test=true | refs: `#41660`_ - * 38424f3 Merge pull request `#41660`_ from garethgreenaway/41651_fixing_aptpkg_held_unheld_with_test - * 30da237 Fix when test=True and packages were being set to unheld. + * 38424f3e3e Merge pull request `#41660`_ from garethgreenaway/41651_fixing_aptpkg_held_unheld_with_test + * 30da2370a4 Fix when test=True and packages were being set to unheld. - **PR** `#41656`_: (*rallytime*) Back-port `#41575`_ to 2016.11 @ *2017-06-08T22:43:23Z* - **PR** `#41575`_: (*dschaller*) Fix 41562 | refs: `#41656`_ - * a308b96 Merge pull request `#41656`_ from rallytime/`bp-41575`_ - * 4374e6b Replace "tbd" with release version information + * a308b960d8 Merge pull request `#41656`_ from rallytime/`bp-41575`_ + * 4374e6b034 Replace "tbd" with release version information - * 8141389 Lint: Add index numbers to format {} calls + * 81413896d1 Lint: Add index numbers to format {} calls - * 3845703 only list top level npm modules during {un)install + * 384570384e only list top level npm modules during {un)install - **PR** `#41456`_: (*bdrung*) Fix pkgrepo.managed always return changes for test=true @ *2017-06-08T18:21:05Z* - * e6d37b5 Merge pull request `#41456`_ from bdrung/fix-pkgrepo.managed-changes-check - * d3ce7bf Fix pkgrepo.managed always return changes for test=true + * e6d37b5f3e Merge pull request `#41456`_ from bdrung/fix-pkgrepo.managed-changes-check + * d3ce7bf05f Fix pkgrepo.managed always return changes for test=true - * 1592687 Document aptpkg architectures parameter + * 1592687294 Document aptpkg architectures parameter - **PR** `#41530`_: (*gtmanfred*) Set default for consul_pillar to None @ *2017-06-08T18:13:15Z* - **ISSUE** `#41478`_: (*jf*) security / information leak with consul pillar when subsitution values are not present | refs: `#41530`_ - * 721e5b6 Merge pull request `#41530`_ from gtmanfred/2016.11 - * 2a4633c Set default for consul_pillar to None + * 721e5b6cb9 Merge pull request `#41530`_ from gtmanfred/2016.11 + * 2a4633ce16 Set default for consul_pillar to None - **PR** `#41638`_: (*gtmanfred*) don't overwrite args if they are passed to the script @ *2017-06-08T17:48:48Z* - **ISSUE** `#41629`_: (*lubyou*) salt.states.cmd.script: Parameter "args" is overwritten if "name/id" contains spaces | refs: `#41638`_ - * 8926d1c Merge pull request `#41638`_ from gtmanfred/cmdscript - * 6c7d68b don't overwrite args if they are passed to the script + * 8926d1c731 Merge pull request `#41638`_ from gtmanfred/cmdscript + * 6c7d68b97d don't overwrite args if they are passed to the script - **PR** `#41639`_: (*dmurphy18*) Update notrim check, netstat takes minutes if large number connections @ *2017-06-07T23:03:24Z* - * ecb09b8 Merge pull request `#41639`_ from dmurphy18/minion_netstat_check - * 7ab3319 Update notrim check, netstat takes minutes if large number connections - 260K + * ecb09b8694 Merge pull request `#41639`_ from dmurphy18/minion_netstat_check + * 7ab3319090 Update notrim check, netstat takes minutes if large number connections - 260K - **PR** `#41611`_: (*garethgreenaway*) Additional fixes to states/saltmod.py @ *2017-06-07T22:58:24Z* - **ISSUE** `#38894`_: (*amendlik*) salt.runner and salt.wheel ignore test=True | refs: `#41309`_ `#41611`_ - * 2913a33 Merge pull request `#41611`_ from garethgreenaway/41309_right_return_res - * fda41ed Updating result values to be None for test cases. + * 2913a33b27 Merge pull request `#41611`_ from garethgreenaway/41309_right_return_res + * fda41ede76 Updating result values to be None for test cases. - * 003f2d9 Following the documentation, when passed the test=True argument the runner and wheel functions should return a result value of False. + * 003f2d9323 Following the documentation, when passed the test=True argument the runner and wheel functions should return a result value of False. - **PR** `#41637`_: (*gtmanfred*) never run bg for onlyif or unless cmd states @ *2017-06-07T17:37:47Z* - **ISSUE** `#41626`_: (*ruiaylin*) When onlyif and bg are used together the | refs: `#41637`_ - * 334a5fc Merge pull request `#41637`_ from gtmanfred/cmd - * 40fb6c6 never run bg for onlyif or unless cmd states + * 334a5fc2a0 Merge pull request `#41637`_ from gtmanfred/cmd + * 40fb6c6249 never run bg for onlyif or unless cmd states - **PR** `#41255`_: (*lordcirth*) linux_syctl.default_config(): only return path, don't create it @ *2017-06-07T14:13:07Z* - * 34dd9ea Merge pull request `#41255`_ from lordcirth/fix-sysctl-test-11 - * 0089be4 linux_sysctl: use dirname() as suggested + * 34dd9ea862 Merge pull request `#41255`_ from lordcirth/fix-sysctl-test-11 + * 0089be4440 linux_sysctl: use dirname() as suggested - * 262d95e linux_syctl.default_config(): only return path, don't create it + * 262d95e41d linux_syctl.default_config(): only return path, don't create it - * 277232b linux_sysctl.persist(): create config dir if needed + * 277232b3ac linux_sysctl.persist(): create config dir if needed - **PR** `#41616`_: (*rallytime*) Back-port `#41551`_ to 2016.11 @ *2017-06-06T22:44:09Z* @@ -307,32 +479,32 @@ Changes: | refs: `#41551`_ `#41551`_ - **PR** `#41551`_: (*darenjacobs*) Update __init__.py | refs: `#41616`_ - * 4cf5777 Merge pull request `#41616`_ from rallytime/`bp-41551`_ - * 53bca96 Update __init__.py + * 4cf577771b Merge pull request `#41616`_ from rallytime/`bp-41551`_ + * 53bca96328 Update __init__.py - **PR** `#41552`_: (*Enquier*) Adding logic so that update_floatingip can dissassociate floatingip's @ *2017-06-06T18:25:56Z* - * 846ca54 Merge pull request `#41552`_ from Enquier/neutron-floatingip-remove - * aeed51c Adding port=None default and documentation + * 846ca54688 Merge pull request `#41552`_ from Enquier/neutron-floatingip-remove + * aeed51c1e3 Adding port=None default and documentation - * fcce05e Adding logic so that update_floatingip can dissassociate floatingip's Previously update_floatingip would cause an error if port is set to None. + * fcce05e1e4 Adding logic so that update_floatingip can dissassociate floatingip's Previously update_floatingip would cause an error if port is set to None. - **PR** `#41569`_: (*gtmanfred*) Check all entries in result @ *2017-06-06T18:18:17Z* - * b720ecb Merge pull request `#41569`_ from gtmanfred/fix_test_result_check - * 19ea548 remove test that never passed + * b720ecb732 Merge pull request `#41569`_ from gtmanfred/fix_test_result_check + * 19ea5481b6 remove test that never passed - * e2a4d5e Check all entries in result + * e2a4d5e1e2 Check all entries in result - **PR** `#41599`_: (*garethgreenaway*) Fixes to modules/archive.py @ *2017-06-06T18:02:14Z* - **ISSUE** `#41540`_: (*UtahDave*) archive.extracted fails on second run | refs: `#41599`_ `#41599`_ - * d9546c6 Merge pull request `#41599`_ from garethgreenaway/41540_fixes_to_archive_module - * 66a136e Fixing issues raised in `#41540`_ when a zip file is created on a Windows system. The issue has two parts, first directories that end up in the archive end up in the results of aarchive.list twice as they show up as both files and directories because of the logic to handle the fact that Windows doesn't mark them as directories. This issue shows up when an extraction is run a second time since the module verified the file types and the subdirectory is not a file. The second issue is related to permissions, if Salt is told to extract permissions (which is the default) then the directory and files end up being unreadable since the permissions are not available. This change sets the permissions to what the default umask for the user running Salt is. + * d9546c6283 Merge pull request `#41599`_ from garethgreenaway/41540_fixes_to_archive_module + * 66a136e6d8 Fixing issues raised in `#41540`_ when a zip file is created on a Windows system. The issue has two parts, first directories that end up in the archive end up in the results of aarchive.list twice as they show up as both files and directories because of the logic to handle the fact that Windows doesn't mark them as directories. This issue shows up when an extraction is run a second time since the module verified the file types and the subdirectory is not a file. The second issue is related to permissions, if Salt is told to extract permissions (which is the default) then the directory and files end up being unreadable since the permissions are not available. This change sets the permissions to what the default umask for the user running Salt is. - **PR** `#41453`_: (*peter-funktionIT*) Update win_pki.py @ *2017-06-06T17:15:55Z* @@ -341,147 +513,148 @@ Changes: | refs: `#41383`_ `#41453`_ - **PR** `#41383`_: (*peter-funktionIT*) Update win_pki.py | refs: `#41453`_ - * 10ac80e Merge pull request `#41453`_ from peter-funktionIT/fix_win_pki_state_import_cert - * d146fd0 Update win_pki.py + * 10ac80ee96 Merge pull request `#41453`_ from peter-funktionIT/fix_win_pki_state_import_cert + * d146fd029c Update win_pki.py - * ef8e3ef Update win_pki.py + * ef8e3ef569 Update win_pki.py - **PR** `#41557`_: (*dmurphy18*) Add symbolic link for salt-proxy service similar to other serivce files @ *2017-06-06T17:13:52Z* - * 3335fcb Merge pull request `#41557`_ from dmurphy18/fix-proxy-service - * ffe492d Add symbolic link salt-proxy service similar to other service files + * 3335fcbc7d Merge pull request `#41557`_ from dmurphy18/fix-proxy-service + * ffe492d6a9 Add symbolic link salt-proxy service similar to other service files - **PR** `#41597`_: (*rallytime*) Back-port `#41533`_ to 2016.11 @ *2017-06-06T15:15:09Z* - **PR** `#41533`_: (*svinota*) unit tests: add pyroute2 interface dict test | refs: `#41597`_ - * 65ed230 Merge pull request `#41597`_ from rallytime/`bp-41533`_ - * 535b8e8 Update new pyroute2 unit test to conform with 2016.11 branch standards + * 65ed230f45 Merge pull request `#41597`_ from rallytime/`bp-41533`_ + * 535b8e8d8e Update new pyroute2 unit test to conform with 2016.11 branch standards - * 5c86dee unit tests: test_pyroute2 -- add skipIf + * 5c86dee73c unit tests: test_pyroute2 -- add skipIf - * 026b394 unit tests: add encoding clause into test_pyroute2 + * 026b39493f unit tests: add encoding clause into test_pyroute2 - * 9ab203d unit tests: fix absolute imports in test_pyroute2 + * 9ab203d54b unit tests: fix absolute imports in test_pyroute2 - * 1f507cf unit tests: add pyroute2 interface dict test + * 1f507cfa7a unit tests: add pyroute2 interface dict test - **PR** `#41596`_: (*rallytime*) Back-port `#41487`_ to 2016.11 @ *2017-06-06T02:44:17Z* - **PR** `#41487`_: (*svinota*) clean up `change` attribute from interface dict | refs: `#41596`_ - * bf8aed1 Merge pull request `#41596`_ from rallytime/`bp-41487`_ - * 7b497d9 clean up `change` attribute from interface dict + * bf8aed153d Merge pull request `#41596`_ from rallytime/`bp-41487`_ + * 7b497d9ec6 clean up `change` attribute from interface dict - **PR** `#41509`_: (*seanjnkns*) Add keystone V3 API support for keystone.endpoint_present|absent @ *2017-06-03T03:01:05Z* - **ISSUE** `#41435`_: (*seanjnkns*) 2016.11: Keystone.endpoint_present overwrites all interfaces | refs: `#41509`_ - * cc6c98a Merge pull request `#41509`_ from seanjnkns/fix-keystone-v3-endpoint_present - * 095e594 Fix unit tests for PR `#41509`_ + * cc6c98a8d8 Merge pull request `#41509`_ from seanjnkns/fix-keystone-v3-endpoint_present + * 095e5949a3 Fix unit tests for PR `#41509`_ - * eb7ef3c Add keystone V3 API support for keystone.endpoint_present|get, endpoint_absent|delete. + * eb7ef3c856 Add keystone V3 API support for keystone.endpoint_present|get, endpoint_absent|delete. - **PR** `#41539`_: (*gtmanfred*) allow digest to be empty in create_crl @ *2017-06-02T17:00:04Z* - **ISSUE** `#38061`_: (*Ch3LL*) x509.crl_managed ValueError when digest is not specified in the module | refs: `#41539`_ - * 0a08649 Merge pull request `#41539`_ from gtmanfred/x509 - * 0989be8 allow digest to be empty in create_crl + * 0a08649637 Merge pull request `#41539`_ from gtmanfred/x509 + * 0989be8919 allow digest to be empty in create_crl - **PR** `#41561`_: (*terminalmage*) Redact HTTP basic authentication in archive.extracted @ *2017-06-02T15:33:14Z* - **ISSUE** `#41154`_: (*mephi42*) archive.extracted outputs password embedded in archive URL | refs: `#41561`_ - * 3ae8336 Merge pull request `#41561`_ from terminalmage/issue41154 - * cbf8acb Redact HTTP basic authentication in archive.extracted + * 3ae8336895 Merge pull request `#41561`_ from terminalmage/issue41154 + * cbf8acbafc Redact HTTP basic authentication in archive.extracted - **PR** `#41436`_: (*skizunov*) TCP transport: Fix occasional errors when using salt command + | refs: `#41787`_ `#41787`_ `#41787`_ @ *2017-06-01T16:37:43Z* - * 39840bf Merge pull request `#41436`_ from skizunov/develop2 - * 07d5862 unit.transport.tcp_test: Clean up channel after use + * 39840bfe4e Merge pull request `#41436`_ from skizunov/develop2 + * 07d5862773 unit.transport.tcp_test: Clean up channel after use - * 4b6aec7 Preserve original IO Loop on cleanup + * 4b6aec7154 Preserve original IO Loop on cleanup - * 892c6d4 TCP transport: Fix occasional errors when using salt command + * 892c6d4d24 TCP transport: Fix occasional errors when using salt command - **PR** `#41337`_: (*Foxlik*) Fix `#41335`_ - list index out of range on empty line in authorized_keys @ *2017-05-31T19:59:17Z* - **ISSUE** `#41335`_: (*syphernl*) [2016.11.5] ssh_auth.present: IndexError: list index out of range | refs: `#41337`_ - * 06ed4f0 Merge pull request `#41337`_ from Foxlik/2016.11 - * 916fecb modify ssh_test.py, to check empty lines and comments in authorized_keys `#41335`_ + * 06ed4f077b Merge pull request `#41337`_ from Foxlik/2016.11 + * 916fecb64f modify ssh_test.py, to check empty lines and comments in authorized_keys `#41335`_ - * 011d6d6 Fix `#41335`_ - list index out of range on empty line in authorized_keys + * 011d6d65e7 Fix `#41335`_ - list index out of range on empty line in authorized_keys - **PR** `#41512`_: (*twangboy*) Use psutil where possible in win_status.py @ *2017-05-31T19:56:00Z* - * 1ace72d Merge pull request `#41512`_ from twangboy/fix_win_status - * 582d09b Get psutil import + * 1ace72d871 Merge pull request `#41512`_ from twangboy/fix_win_status + * 582d09b484 Get psutil import - * fd88bb2 Remove unused imports (lint) + * fd88bb277f Remove unused imports (lint) - * 41a39df Use psutil where possible + * 41a39dff00 Use psutil where possible - **PR** `#41490`_: (*t0fik*) Backport of SELinux module installation and removal @ *2017-05-31T19:38:00Z* - * 683cc5f Merge pull request `#41490`_ from jdsieci/2016.11_selinux - * e2fbada Backport of SELinux module installation and removal + * 683cc5f414 Merge pull request `#41490`_ from jdsieci/2016.11_selinux + * e2fbada1c1 Backport of SELinux module installation and removal - **PR** `#41522`_: (*jettero*) Sadly, you can't have '.'s and '$'s in dict keys in a mongodb doc. @ *2017-05-31T15:55:24Z* - * 2e7e84b Merge pull request `#41522`_ from jettero/mongodb-keys-are-stupid - * 12648f5 dang, thought I already got that. Apparently only got the bottom one. This should do it. + * 2e7e84b8f2 Merge pull request `#41522`_ from jettero/mongodb-keys-are-stupid + * 12648f5439 dang, thought I already got that. Apparently only got the bottom one. This should do it. - * 7c4a763 ugh, forgot about this lint too. This one looks especially terrible. + * 7c4a763518 ugh, forgot about this lint too. This one looks especially terrible. - * c973988 forgot about the linter pass … fixed + * c973988d8d forgot about the linter pass … fixed - * da0d9e4 Sadly, you can't have '.'s and '$'s in dict keys in a mongodb doc. + * da0d9e4045 Sadly, you can't have '.'s and '$'s in dict keys in a mongodb doc. - **PR** `#41506`_: (*gtmanfred*) check for integer types @ *2017-05-31T00:48:21Z* - **ISSUE** `#41504`_: (*mtkennerly*) Can't set REG_DWORD registry value larger than 0x7FFFFFFF | refs: `#41506`_ - * 30ad4fd Merge pull request `#41506`_ from gtmanfred/2016.11 - * 5fe2e9b check for integer types + * 30ad4fd9a0 Merge pull request `#41506`_ from gtmanfred/2016.11 + * 5fe2e9bbf5 check for integer types - **PR** `#41469`_: (*Ch3LL*) Fix keep_jobs keyerror in redis returner @ *2017-05-30T18:37:42Z* - * 06ef17d Merge pull request `#41469`_ from Ch3LL/fix_redis_error - * 8ee1251 Fix keep_jobs keyerror in redis returner + * 06ef17dec3 Merge pull request `#41469`_ from Ch3LL/fix_redis_error + * 8ee1251a3a Fix keep_jobs keyerror in redis returner - **PR** `#41473`_: (*twangboy*) Fix win_firewall execution and state modules @ *2017-05-30T18:35:24Z* - * 7a09b2b Merge pull request `#41473`_ from twangboy/fix_win_firewall - * e503b45 Fix lint error + * 7a09b2b678 Merge pull request `#41473`_ from twangboy/fix_win_firewall + * e503b455c3 Fix lint error - * d3f0f8b Fix win_firewall execution and state modules + * d3f0f8bcd2 Fix win_firewall execution and state modules - **PR** `#41499`_: (*rallytime*) [2016.11] Merge forward from 2016.3 to 2016.11 @ *2017-05-30T18:06:03Z* - **PR** `#41439`_: (*terminalmage*) base64 encode binary data sent using salt-cp - * f635cb1 Merge pull request `#41499`_ from rallytime/merge-2016.11 - * 20d893d Merge branch '2016.3' into '2016.11' + * f635cb11c4 Merge pull request `#41499`_ from rallytime/merge-2016.11 + * 20d893d397 Merge branch '2016.3' into '2016.11' - * 964b1ee Merge pull request `#41439`_ from terminalmage/salt-cp-base64 + * 964b1ee027 Merge pull request `#41439`_ from terminalmage/salt-cp-base64 - * ebf6cc7 base64 encode binary data sent using salt-cp + * ebf6cc78c7 base64 encode binary data sent using salt-cp - **PR** `#41464`_: (*rallytime*) Back-port `#39850`_ to 2016.11 @ *2017-05-26T21:22:44Z* @@ -489,24 +662,24 @@ Changes: - **ISSUE** `#35874`_: (*epcim*) keystone.endpoint_present deletes RegionOne endpoints - **PR** `#39850`_: (*epcim*) Fix endpoint handling per region | refs: `#41464`_ - * 83f1e48 Merge pull request `#41464`_ from rallytime/`bp-39850`_ - * 9b84b75 Pylint fixes + * 83f1e48241 Merge pull request `#41464`_ from rallytime/`bp-39850`_ + * 9b84b751b2 Pylint fixes - * 6db8915 Endpoint handling per region, fixes `#35874`_ - extend tests for multiple regions - region arg by default set to None - print verbose changes to be exec. + * 6db8915021 Endpoint handling per region, fixes `#35874`_ - extend tests for multiple regions - region arg by default set to None - print verbose changes to be exec. - **PR** `#41443`_: (*UtahDave*) use proper arg number @ *2017-05-26T20:36:37Z* - * 960c576 Merge pull request `#41443`_ from UtahDave/fix_args_masterpy - * dfbdc27 use proper arg number + * 960c5767fa Merge pull request `#41443`_ from UtahDave/fix_args_masterpy + * dfbdc275ca use proper arg number - **PR** `#41350`_: (*lorengordon*) Supports quoted values in /etc/sysconfig/network @ *2017-05-26T16:22:03Z* - **ISSUE** `#41341`_: (*lorengordon*) TypeError traceback in network.system with retain_settings=True | refs: `#41350`_ - * 88c28c1 Merge pull request `#41350`_ from lorengordon/issue-41341 - * f2f6da7 Supports quoted values in /etc/sysconfig/network + * 88c28c18c3 Merge pull request `#41350`_ from lorengordon/issue-41341 + * f2f6da7039 Supports quoted values in /etc/sysconfig/network - **PR** `#41398`_: (*rallytime*) [2016.11] Merge forward from 2016.3 to 2016.11 @ *2017-05-26T15:17:49Z* @@ -519,40 +692,40 @@ Changes: - **PR** `#41316`_: (*Ch3LL*) [2016.3] Bump latest release version to 2016.11.5 - **PR** `#41265`_: (*terminalmage*) yumpkg: fix latest_version() when showdupesfromrepos=1 set in /etc/yum.conf - **PR** `#41216`_: (*terminalmage*) Make salt-cp work with larger files - * 824f2d3 Merge pull request `#41398`_ from rallytime/merge-2016.11 - * 2941e9c Merge pull request `#22`_ from terminalmage/merge-2016.11 + * 824f2d3b69 Merge pull request `#41398`_ from rallytime/merge-2016.11 + * 2941e9c923 Merge pull request `#22`_ from terminalmage/merge-2016.11 - * 087a958 base64 encode binary data sent using salt-cp + * 087a958afc base64 encode binary data sent using salt-cp - * 503f925 Add missing import + * 503f925275 Add missing import - * d2d9a3d Merge branch '2016.3' into '2016.11' + * d2d9a3d29f Merge branch '2016.3' into '2016.11' - * d617c9f Merge pull request `#41265`_ from terminalmage/issue41234 + * d617c9fe72 Merge pull request `#41265`_ from terminalmage/issue41234 - * edf552f Update PKG_TARGETS for RHEL-based distros + * edf552fe9a Update PKG_TARGETS for RHEL-based distros - * 0ecc7b9 yumpkg: fix latest_version() when showdupesfromrepos=1 set in /etc/yum.conf + * 0ecc7b9b20 yumpkg: fix latest_version() when showdupesfromrepos=1 set in /etc/yum.conf - * 26bd914 Merge pull request `#41316`_ from Ch3LL/update_latest_2016.3 + * 26bd914580 Merge pull request `#41316`_ from Ch3LL/update_latest_2016.3 - * 520740d [2016.13] Bump latest release version to 2016.11.5 + * 520740d862 [2016.13] Bump latest release version to 2016.11.5 - * 18898b7 Merge pull request `#41216`_ from terminalmage/issue16592 + * 18898b7d1f Merge pull request `#41216`_ from terminalmage/issue16592 - * 0e15fdb Update salt-cp integration test to reflect recent changes + * 0e15fdbb1a Update salt-cp integration test to reflect recent changes - * 10dc695 Make salt-cp work with larger files + * 10dc695cc4 Make salt-cp work with larger files - * c078180 Make KeyErrors more specific when interpreting returns + * c078180539 Make KeyErrors more specific when interpreting returns - * fc401c9 Add generator functions for reading files + * fc401c9eb4 Add generator functions for reading files - **PR** `#41442`_: (*UtahDave*) use proper arg number @ *2017-05-26T13:42:50Z* - * ec08064 Merge pull request `#41442`_ from UtahDave/fix_args - * 0324833 use proper arg number + * ec08064b99 Merge pull request `#41442`_ from UtahDave/fix_args + * 0324833c9e use proper arg number - **PR** `#41397`_: (*Enquier*) Updating Nova/Neutron modules to support KeystoneAuth and SSLVerify @ *2017-05-25T21:16:14Z* @@ -565,121 +738,121 @@ Changes: | refs: `#41397`_ - **PR** `#38647`_: (*gtmanfred*) Allow novaclient to use keystoneauth1 sessions for authentication | refs: `#41397`_ - * 22096d9 Merge pull request `#41397`_ from Enquier/neutron-ssl-verify - * d25dcf6 Small error in nova that was preventing execution + * 22096d9213 Merge pull request `#41397`_ from Enquier/neutron-ssl-verify + * d25dcf61d5 Small error in nova that was preventing execution - * 0e7a100 Updated module docs to include changes made + * 0e7a1009ed Updated module docs to include changes made - * 05e0192 Adding missing os_auth_system + * 05e0192665 Adding missing os_auth_system - * 4e0f498 allow service_type to be specified default is now 'network' + * 4e0f4981e4 allow service_type to be specified default is now 'network' - * 991e843 Added non-profile and defaults for Neutron + * 991e84343f Added non-profile and defaults for Neutron - * c93f112 Updating Nova Module to include use_keystone Auth + * c93f112c9b Updating Nova Module to include use_keystone Auth - * 66ab1e5 Re-adding neutron dependency check + * 66ab1e5184 Re-adding neutron dependency check - * cce07ee Updating Neutron module to suport KeystoneAuth + * cce07eefc2 Updating Neutron module to suport KeystoneAuth - **PR** `#41409`_: (*garethgreenaway*) Fixes to ipc transport @ *2017-05-25T21:06:27Z* - **ISSUE** `#34460`_: (*Ch3LL*) Receive an error when using salt-api to call a runner | refs: `#41409`_ - * 14a58cf Merge pull request `#41409`_ from garethgreenaway/34460_fixes_ipc_transport - * 5613b72 Updating the exception variable to be more in line with the rest of the exception code + * 14a58cf536 Merge pull request `#41409`_ from garethgreenaway/34460_fixes_ipc_transport + * 5613b72dfe Updating the exception variable to be more in line with the rest of the exception code - * 41eee8b Fixing a potential lint issue + * 41eee8b333 Fixing a potential lint issue - * 760d561 Fixing a potential lint issue + * 760d561dfa Fixing a potential lint issue - * c11bcd0 Changing the approaching and including an except for the action socket.error exception, then logging a trace log if error number is 0 and an error log otherwise. + * c11bcd0d12 Changing the approaching and including an except for the action socket.error exception, then logging a trace log if error number is 0 and an error log otherwise. - * 3f95059 Fixing lint issues. + * 3f950596f4 Fixing lint issues. - * f3a6531 On occasion an exception will occur which results in the event not returning properly, even though the wire_bytes is correctly populated. In this situation, we log to trace and continue. `#34460`_ + * f3a6531a69 On occasion an exception will occur which results in the event not returning properly, even though the wire_bytes is correctly populated. In this situation, we log to trace and continue. `#34460`_ - **PR** `#41421`_: (*UtahDave*) Correct doc to actually blacklist a module @ *2017-05-25T21:01:46Z* - * 8244287 Merge pull request `#41421`_ from UtahDave/fix_blacklist_docs - * 5eb2757 Correct doc to actually blacklist a module + * 824428700d Merge pull request `#41421`_ from UtahDave/fix_blacklist_docs + * 5eb27571a0 Correct doc to actually blacklist a module - **PR** `#41431`_: (*terminalmage*) Fix regression in state orchestration @ *2017-05-25T18:44:53Z* - **ISSUE** `#41353`_: (*rmarchei*) Orchestrate runner needs saltenv on 2016.11.5 | refs: `#41431`_ - * b98d5e0 Merge pull request `#41431`_ from terminalmage/issue41353 - * 16eae64 Fix regression in state orchestration + * b98d5e00d4 Merge pull request `#41431`_ from terminalmage/issue41353 + * 16eae64cca Fix regression in state orchestration - **PR** `#41429`_: (*ricohouse*) Issue `#41338`_: Return false when compare config fails @ *2017-05-25T17:18:02Z* - **ISSUE** `#41338`_: (*ricohouse*) Exception not raised when running config compare and the device (Juniper) returns error | refs: `#41429`_ - * eeff3dd Merge pull request `#41429`_ from ricohouse/fix-compare-bug - * 9b61665 Issue `#41338`_: Return false when compare config fails + * eeff3dd7fb Merge pull request `#41429`_ from ricohouse/fix-compare-bug + * 9b61665c4c Issue `#41338`_: Return false when compare config fails - **PR** `#41414`_: (*Ch3LL*) Update bootstrap script verstion to latest release(v2017.05.24) @ *2017-05-24T19:51:49Z* - * 561a416 Merge pull request `#41414`_ from Ch3LL/update_bootstrap - * d8c03ee Update bootstrap script verstion to latest release(v2017.05.24) + * 561a416cf3 Merge pull request `#41414`_ from Ch3LL/update_bootstrap + * d8c03eef60 Update bootstrap script verstion to latest release(v2017.05.24) - **PR** `#41336`_: (*mcalmer*) fix setting and getting locale on SUSE systems @ *2017-05-24T17:46:08Z* - * 88fd3c0 Merge pull request `#41336`_ from mcalmer/fix-locale-on-SUSE - * f30f5c8 fix unit tests + * 88fd3c0ed9 Merge pull request `#41336`_ from mcalmer/fix-locale-on-SUSE + * f30f5c8a25 fix unit tests - * 428baa9 fix setting and getting locale on SUSE systems + * 428baa9bce fix setting and getting locale on SUSE systems - **PR** `#41393`_: (*rallytime*) Back-port `#41235`_ to 2016.11 @ *2017-05-24T16:08:56Z* - **PR** `#41235`_: (*moio*) rest_cherrypy: remove sleep call | refs: `#41393`_ - * 4265959 Merge pull request `#41393`_ from rallytime/`bp-41235`_ - * c79c0e3 rest_cherrypy: remove sleep call + * 4265959647 Merge pull request `#41393`_ from rallytime/`bp-41235`_ + * c79c0e3f43 rest_cherrypy: remove sleep call - **PR** `#41394`_: (*rallytime*) Back-port `#41243`_ to 2016.11 @ *2017-05-24T16:00:17Z* - **PR** `#41243`_: (*arif-ali*) Remove the keys that don't exist in the new change | refs: `#41394`_ - * 83f5469 Merge pull request `#41394`_ from rallytime/`bp-41243`_ - * a535130 Lint fix + * 83f54694f9 Merge pull request `#41394`_ from rallytime/`bp-41243`_ + * a5351302af Lint fix - * 05fadc0 Remove the keys that don't exist in the new change + * 05fadc0af3 Remove the keys that don't exist in the new change - **PR** `#41401`_: (*bdrung*) Add documentation key to systemd service files @ *2017-05-24T15:49:54Z* - * 3a45ac3 Merge pull request `#41401`_ from bdrung/systemd-service-documentation-key - * 3f7f308 Add documentation key to systemd service files + * 3a45ac30f0 Merge pull request `#41401`_ from bdrung/systemd-service-documentation-key + * 3f7f30895d Add documentation key to systemd service files - **PR** `#41404`_: (*bdrung*) Fix typos @ *2017-05-24T14:42:44Z* - * d34333c Merge pull request `#41404`_ from bdrung/fix-typos - * 33a7f8b Fix typos + * d34333c30b Merge pull request `#41404`_ from bdrung/fix-typos + * 33a7f8b2ec Fix typos - **PR** `#41388`_: (*bdrung*) Do not require sphinx-build for cleaning docs @ *2017-05-23T19:32:41Z* - * 3083764 Merge pull request `#41388`_ from bdrung/clean-doc-without-sphinx - * 5b79a0a Do not require sphinx-build for cleaning docs + * 3083764195 Merge pull request `#41388`_ from bdrung/clean-doc-without-sphinx + * 5b79a0a9f8 Do not require sphinx-build for cleaning docs - **PR** `#41364`_: (*automate-solutions*) Fix issue `#41362`_ invalid parameter used: KeyName.1 instead of KeyName @ *2017-05-23T17:32:10Z* - **ISSUE** `#41362`_: (*automate-solutions*) On AWS EC2: salt-cloud -f delete_keypair ec2 keyname=mykeypair doesn't delete the keypair - * 842875e Merge pull request `#41364`_ from automate-solutions/fix-issue-41362 - * cfd8eb7 Set DescribeKeyPairs back to KeyName.1 according to documentation + * 842875e590 Merge pull request `#41364`_ from automate-solutions/fix-issue-41362 + * cfd8eb7a87 Set DescribeKeyPairs back to KeyName.1 according to documentation - * 6a82ddc Fix issue `#41362`_ invalid parameter used: KeyName.1 instead of KeyName + * 6a82ddc6fc Fix issue `#41362`_ invalid parameter used: KeyName.1 instead of KeyName - **PR** `#41383`_: (*peter-funktionIT*) Update win_pki.py | refs: `#41453`_ @@ -687,66 +860,66 @@ Changes: - **ISSUE** `#40950`_: (*idokaplan*) Import certificate | refs: `#41383`_ `#41453`_ - * 92f94e6 Merge pull request `#41383`_ from peter-funktionIT/fix-win_pki-get_cert_file - * 4d9bd06 Update win_pki.py + * 92f94e66bc Merge pull request `#41383`_ from peter-funktionIT/fix-win_pki-get_cert_file + * 4d9bd06176 Update win_pki.py - **PR** `#41113`_: (*cro*) Rescue proxy_auto_tests PR from git rebase hell @ *2017-05-22T17:05:07Z* - **PR** `#39575`_: (*cro*) WIP: Proxy auto test, feedback appreciated | refs: `#41113`_ - * 1ba9568 Merge pull request `#41113`_ from cro/proxy_auto_test2 - * 19db038 Fix test--use proxy_config instead of minion_config + * 1ba95684a9 Merge pull request `#41113`_ from cro/proxy_auto_test2 + * 19db038b99 Fix test--use proxy_config instead of minion_config - * 7749cea Change default proxy minion opts so only the proxy-specific ones are listed, and the rest are taken from DEFAULT_MINION_OPTS. + * 7749ceadb6 Change default proxy minion opts so only the proxy-specific ones are listed, and the rest are taken from DEFAULT_MINION_OPTS. - * 106394c Lint. + * 106394c80c Lint. - * 3be90cc Rescue proxy_auto_tests PR from git rebase hell + * 3be90cc9f4 Rescue proxy_auto_tests PR from git rebase hell - **PR** `#41360`_: (*cro*) Sysrc on FreeBSD, YAML overeager to coerce to bool and int @ *2017-05-22T15:54:31Z* - * 375892d Merge pull request `#41360`_ from cro/sysrc_fix - * 6db31ce Fix problem with sysrc on FreeBSD, YAML overeager to coerce to bool and int. + * 375892d910 Merge pull request `#41360`_ from cro/sysrc_fix + * 6db31ce52a Fix problem with sysrc on FreeBSD, YAML overeager to coerce to bool and int. - **PR** `#41372`_: (*terminalmage*) Don't use intermediate file when listing contents of tar.xz file @ *2017-05-22T15:36:45Z* - **ISSUE** `#41190`_: (*jheidbrink*) Cannot extract tar.xz archive when it exceeds size of /tmp | refs: `#41372`_ - * 01b71c7 Merge pull request `#41372`_ from terminalmage/issue41190 - * 1f08936 Remove unused import + * 01b71c75c1 Merge pull request `#41372`_ from terminalmage/issue41190 + * 1f08936d9c Remove unused import - * 68cb897 Replace reference to fileobj + * 68cb897520 Replace reference to fileobj - * 7888744 Remove '*' from mode + * 788874408a Remove '*' from mode - * 3d4b833 Don't use intermediate file when listing contents of tar.xz file + * 3d4b833627 Don't use intermediate file when listing contents of tar.xz file - **PR** `#41373`_: (*alex-zel*) Allow HTTP authentication to ES. @ *2017-05-22T15:32:09Z* - * 5edfcf9 Merge pull request `#41373`_ from alex-zel/patch-3 - * 3192eab Allow HTTP authentication to ES. + * 5edfcf972c Merge pull request `#41373`_ from alex-zel/patch-3 + * 3192eab128 Allow HTTP authentication to ES. - **PR** `#41287`_: (*garethgreenaway*) Fix to consul cache @ *2017-05-19T18:32:56Z* - **ISSUE** `#40748`_: (*djhaskin987*) Consul backend minion cache does not work | refs: `#41287`_ - * 29bd7f4 Merge pull request `#41287`_ from garethgreenaway/40748_2016_11_consul - * 5039fe1 Removing chdir as it is no needed with this change + * 29bd7f48b7 Merge pull request `#41287`_ from garethgreenaway/40748_2016_11_consul + * 5039fe12fb Removing chdir as it is no needed with this change - * 4550c3c Updating the code that is pulling in the list of cached minions to use self.cache.list instead of relying on checking the local file system, which only works for the localfs cache method. `#40748`_ + * 4550c3ce49 Updating the code that is pulling in the list of cached minions to use self.cache.list instead of relying on checking the local file system, which only works for the localfs cache method. `#40748`_ - **PR** `#41309`_: (*garethgreenaway*) Adding test argument for runners & wheel orchestration modules @ *2017-05-19T18:26:09Z* - **ISSUE** `#38894`_: (*amendlik*) salt.runner and salt.wheel ignore test=True | refs: `#41309`_ `#41611`_ - * 672aaa8 Merge pull request `#41309`_ from garethgreenaway/38894_allowing_test_argument - * e1a88e8 Allowing test=True to be passed for salt.runner and salt.wheel when used with orchestration + * 672aaa88d3 Merge pull request `#41309`_ from garethgreenaway/38894_allowing_test_argument + * e1a88e8bf7 Allowing test=True to be passed for salt.runner and salt.wheel when used with orchestration - **PR** `#41319`_: (*lomeroe*) backport `#41307`_ to 2016.11, properly pack version numbers into single @ *2017-05-19T18:25:00Z* @@ -755,76 +928,76 @@ Changes: | refs: `#41319`_ `#41307`_ - **PR** `#41307`_: (*lomeroe*) properly pack/unpack the verison numbers into a number | refs: `#41319`_ - * 140b042 Merge pull request `#41319`_ from lomeroe/bp_41307 - * 4f0aa57 backport 41307 to 2016.11, properly pack version numbers into single number + * 140b0427e1 Merge pull request `#41319`_ from lomeroe/bp_41307 + * 4f0aa577a5 backport 41307 to 2016.11, properly pack version numbers into single number - **PR** `#41327`_: (*Ch3LL*) Add 2016.11.6 Release Notes @ *2017-05-19T18:05:09Z* - * 6bdb7cc Merge pull request `#41327`_ from Ch3LL/add_2016.11.6_release - * e5fc0ae Add 2016.11.6 Release Notes + * 6bdb7cca7d Merge pull request `#41327`_ from Ch3LL/add_2016.11.6_release + * e5fc0aeb9c Add 2016.11.6 Release Notes - **PR** `#41329`_: (*lorengordon*) Corrects versionadded for win_network.get_route @ *2017-05-19T17:47:57Z* - * 1faffd3 Merge pull request `#41329`_ from lorengordon/doc-fix - * 3c47124 Corrects versionadded for win_network.get_route + * 1faffd3932 Merge pull request `#41329`_ from lorengordon/doc-fix + * 3c471247f0 Corrects versionadded for win_network.get_route - **PR** `#41322`_: (*Ch3LL*) Add patched packages warning to 2016.11.5 release notes @ *2017-05-18T21:53:26Z* - * 6ca6559 Merge pull request `#41322`_ from Ch3LL/fix_release_2016.11.5_notes - * 9a1bf42 fix url refs in rst + * 6ca65592da Merge pull request `#41322`_ from Ch3LL/fix_release_2016.11.5_notes + * 9a1bf4205f fix url refs in rst - * cde008f Add patched packages warning to 2016.11.5 release notes + * cde008ff77 Add patched packages warning to 2016.11.5 release notes - **PR** `#41208`_: (*pkazmierczak*) Fix: zypper handling of multiple version packages @ *2017-05-18T15:44:26Z* - * 9f359d8 Merge pull request `#41208`_ from pkazmierczak/pkazmierczak-zypper-multiple-ver-pkgs - * d411a91 Reverted back to cascading with statements for python 2.6 compat + * 9f359d841f Merge pull request `#41208`_ from pkazmierczak/pkazmierczak-zypper-multiple-ver-pkgs + * d411a91676 Reverted back to cascading with statements for python 2.6 compat - * 7204013 Compacted with statements in the unit test. + * 7204013653 Compacted with statements in the unit test. - * 6c4c080 Added unit tests and copied the behavior to .upgrade method, too. + * 6c4c08042c Added unit tests and copied the behavior to .upgrade method, too. - * 5f95200 Fix: zypper handling of multiple version packages + * 5f952007f6 Fix: zypper handling of multiple version packages - **PR** `#41317`_: (*Ch3LL*) [2016.11] Bump latest release version to 2016.11.5 @ *2017-05-18T15:34:13Z* - * bcef99a Merge pull request `#41317`_ from Ch3LL/update_latest_2016.11 - * cdb072c [2016.11] Bump latest release version to 2016.11.5 + * bcef99adb6 Merge pull request `#41317`_ from Ch3LL/update_latest_2016.11 + * cdb072c207 [2016.11] Bump latest release version to 2016.11.5 - **PR** `#41232`_: (*axmetishe*) Add basic auth for SPM @ *2017-05-17T19:08:56Z* - * b8ddd7e Merge pull request `#41232`_ from axmetishe/2016.11 - * 76104f23 Add basic auth for SPM + * b8ddd7ee08 Merge pull request `#41232`_ from axmetishe/2016.11 + * 76104f23b4 Add basic auth for SPM - **PR** `#41236`_: (*BenoitKnecht*) states: cron: show correct changes when using `special` @ *2017-05-17T18:51:58Z* - * 7bdb66d Merge pull request `#41236`_ from BenoitKnecht/2016.11 - * 33211d0 states: cron: show correct changes when using `special` + * 7bdb66d969 Merge pull request `#41236`_ from BenoitKnecht/2016.11 + * 33211d032e states: cron: show correct changes when using `special` - **PR** `#41269`_: (*isbm*) Bugfix: Unable to use "127" as hostname for the Minion ID @ *2017-05-17T18:31:15Z* - * 1c1e092 Merge pull request `#41269`_ from isbm/isbm-minion-id-127-name - * 5168ef8 Add unit test for hostname can be started from 127 + * 1c1e092f56 Merge pull request `#41269`_ from isbm/isbm-minion-id-127-name + * 5168ef8959 Add unit test for hostname can be started from 127 - * 0d03541 Harden to 127. IP part + * 0d0354198b Harden to 127. IP part - * d9c8324 Unit test for accepting hosts names as 127 + * d9c8324a6b Unit test for accepting hosts names as 127 - * 65b03c6 Bugfix: unable to use 127 as hostname + * 65b03c667b Bugfix: unable to use 127 as hostname - **PR** `#41289`_: (*garethgreenaway*) Fixing consul cache @ *2017-05-17T16:54:12Z* - * d0fa31d Merge pull request `#41289`_ from garethgreenaway/2016_11_5_fix_consul_cache_ls - * 780a28c Swapping the order in the func_alias so the ls function is available. + * d0fa31d4ca Merge pull request `#41289`_ from garethgreenaway/2016_11_5_fix_consul_cache_ls + * 780a28c9a0 Swapping the order in the func_alias so the ls function is available. - **PR** `#41303`_: (*lomeroe*) backport `#41301`_ -- properly convert packed string to decimal values @ *2017-05-17T16:32:22Z* @@ -833,8 +1006,8 @@ Changes: | refs: `#41301`_ `#41303`_ - **PR** `#41301`_: (*lomeroe*) properly convert packed string to decimal values | refs: `#41303`_ - * 6566648 Merge pull request `#41303`_ from lomeroe/`bp-41301`_ - * f4b93f9 properly convert packed string to decimal values + * 6566648948 Merge pull request `#41303`_ from lomeroe/`bp-41301`_ + * f4b93f9d9a properly convert packed string to decimal values - **PR** `#41283`_: (*terminalmage*) Backport `#41251`_ to 2016.11 @ *2017-05-16T18:01:17Z* @@ -846,16 +1019,16 @@ Changes: - **PR** `#41251`_: (*abednarik*) Update apt module regarding upgrade against hold packages. - **PR** `#30777`_: (*abednarik*) Fix update apt hold pkgs | refs: `#41251`_ - * 4459861 Merge pull request `#41283`_ from terminalmage/`bp-41251`_ - * ed03ca5 Update apt module regarding upgrade against hold packages. + * 44598617be Merge pull request `#41283`_ from terminalmage/`bp-41251`_ + * ed03ca534f Update apt module regarding upgrade against hold packages. - **PR** `#41181`_: (*gtmanfred*) add resolving extra flags to yum upgrade @ *2017-05-16T04:07:47Z* - * d8e9676 Merge pull request `#41181`_ from gtmanfred/2016.11 - * 2ca7171 use six and clean_kwargs + * d8e9676fcf Merge pull request `#41181`_ from gtmanfred/2016.11 + * 2ca71713b1 use six and clean_kwargs - * c9bf09a add resolving extra flags to yum upgrade + * c9bf09a5a1 add resolving extra flags to yum upgrade - **PR** `#41220`_: (*rallytime*) Back-port `#40246`_ to 2016.11 @ *2017-05-15T17:59:38Z* @@ -864,20 +1037,20 @@ Changes: | refs: `#40246`_ `#40246`_ - **PR** `#40246`_: (*tonybaloney*) Fix libcloud_dns state module bug | refs: `#41220`_ - * 7594223 Merge pull request `#41220`_ from rallytime/`bp-40246`_ - * 79f1bb2 Remove unused/duplicate imports leftover from merge-conflict resolution + * 75942235f0 Merge pull request `#41220`_ from rallytime/`bp-40246`_ + * 79f1bb2bba Remove unused/duplicate imports leftover from merge-conflict resolution - * 2f61068 remove unused imports + * 2f610680e5 remove unused imports - * 9b7de2e fix unit tests + * 9b7de2e7d7 fix unit tests - * 49d9455 linting + * 49d94559ab linting - * 4b260a4 linting + * 4b260a4594 linting - * 41d1ada fix up tests + * 41d1adab5f fix up tests - * b3822e0 add fixes for incorrectly importing modules directly instead of using __salt__ + * b3822e03fc add fixes for incorrectly importing modules directly instead of using __salt__ - **PR** `#41244`_: (*cachedout*) Fix ipv6 nameserver grains @ *2017-05-15T17:55:39Z* @@ -888,30 +1061,30 @@ Changes: | refs: `#40934`_ - **PR** `#40934`_: (*gtmanfred*) Only display IPvX warning if role is master | refs: `#41244`_ `#41244`_ - * 53d5b3e Merge pull request `#41244`_ from cachedout/fix_ipv6_nameserver_grains - * f745db1 Lint + * 53d5b3e816 Merge pull request `#41244`_ from cachedout/fix_ipv6_nameserver_grains + * f745db1a43 Lint - * 6e1ab69 Partial revert of `#40934`_ + * 6e1ab69710 Partial revert of `#40934`_ - * 88f49f9 Revert "Only display IPvX warning if role is master" + * 88f49f9146 Revert "Only display IPvX warning if role is master" - **PR** `#41242`_: (*pprkut*) Fix changing a mysql user to unix socket authentication. @ *2017-05-15T17:00:06Z* - * 895fe58 Merge pull request `#41242`_ from M2Mobi/mysql_socket_auth - * 7d83597 Fix changing a mysql user to unix socket authentication. + * 895fe582eb Merge pull request `#41242`_ from M2Mobi/mysql_socket_auth + * 7d8359766d Fix changing a mysql user to unix socket authentication. - **PR** `#41101`_: (*terminalmage*) Fix "latest" keyword for version specification when used with aggregation @ *2017-05-15T16:52:35Z* - **ISSUE** `#40940`_: (*djhaskin987*) When `state_aggregate` is set to `True`, the `latest` keyword doesn't work with pkg.installed | refs: `#41101`_ - * 50d8fde Merge pull request `#41101`_ from terminalmage/issue40940 - * 7fe6421 Add rtag check to integration test for pkg.refresh_db + * 50d8fde123 Merge pull request `#41101`_ from terminalmage/issue40940 + * 7fe64219ae Add rtag check to integration test for pkg.refresh_db - * 88a08aa Add comments to explain what removing the rtag file actually does + * 88a08aa3bf Add comments to explain what removing the rtag file actually does - * 92011db Fix "latest" keyword for version specification when used with aggregation + * 92011dbe5f Fix "latest" keyword for version specification when used with aggregation - **PR** `#41146`_: (*terminalmage*) gitfs: Backport performance fixes for getting tree objects @ *2017-05-12T17:35:47Z* @@ -920,72 +1093,72 @@ Changes: | refs: `#41144`_ - **PR** `#41144`_: (*terminalmage*) gitfs: Add two new options to affect saltenv mapping | refs: `#41146`_ - * 049712b Merge pull request `#41146`_ from terminalmage/backport-get_tree-performance-improvement - * f9d6734 gitfs: Backport performance fixes for getting tree objects + * 049712ba53 Merge pull request `#41146`_ from terminalmage/backport-get_tree-performance-improvement + * f9d6734afe gitfs: Backport performance fixes for getting tree objects - **PR** `#41161`_: (*The-Loeki*) gpg renderer: fix gpg_keydir always reverting to default @ *2017-05-12T17:19:07Z* - **ISSUE** `#41135`_: (*shallot*) gpg renderer doesn't seem to work with salt-ssh, tries to execute gpg on the minion? | refs: `#41161`_ - * 4215a0b Merge pull request `#41161`_ from The-Loeki/2016.11 - * 24946fe gpg renderer: fix gpg_keydir always reverting to default + * 4215a0b99d Merge pull request `#41161`_ from The-Loeki/2016.11 + * 24946fef18 gpg renderer: fix gpg_keydir always reverting to default - **PR** `#41163`_: (*onlyanegg*) Elasticsearch - pass hosts and profile to index_exists() @ *2017-05-12T17:18:06Z* - **ISSUE** `#41162`_: (*onlyanegg*) Elasticsearch module functions should pass hosts and profile to index_exists() | refs: `#41163`_ - * 5b10fc5 Merge pull request `#41163`_ from onlyanegg/elasticsearch-pass_profile_to_index_exists - * 7f512c7 Pass hosts and profile to index_exists() method + * 5b10fc58ba Merge pull request `#41163`_ from onlyanegg/elasticsearch-pass_profile_to_index_exists + * 7f512c701b Pass hosts and profile to index_exists() method - **PR** `#41186`_: (*jmarinaro*) Fix package name collisions in chocolatey state @ *2017-05-12T17:01:31Z* - **ISSUE** `#41185`_: (*jmarinaro*) package name collisions in chocolatey state | refs: `#41186`_ - * d433cf8 Merge pull request `#41186`_ from jmarinaro/fix-chocolatey-package-collision - * 229f3bf apply changes to uninstalled function + * d433cf850d Merge pull request `#41186`_ from jmarinaro/fix-chocolatey-package-collision + * 229f3bf9f3 apply changes to uninstalled function - * ffd4c7e Fix package name collisions in chocolatey state + * ffd4c7ef04 Fix package name collisions in chocolatey state - **PR** `#41189`_: (*github-abcde*) utils/minions.py: Fixed case where data is an empty dict resulting in… @ *2017-05-12T16:32:25Z* - * bb5ef41 Merge pull request `#41189`_ from github-abcde/utils-minions-fix - * 853dc54 utils/minions.py: Fixed case where data is an empty dict resulting in errors. + * bb5ef41ce0 Merge pull request `#41189`_ from github-abcde/utils-minions-fix + * 853dc5406c utils/minions.py: Fixed case where data is an empty dict resulting in errors. - **PR** `#41104`_: (*Ch3LL*) Add test to query results of /jobs call in api @ *2017-05-10T20:11:08Z* - * b136b15 Merge pull request `#41104`_ from Ch3LL/add_jobs_test - * dac1658 add test to query results of /jobs call in api + * b136b15330 Merge pull request `#41104`_ from Ch3LL/add_jobs_test + * dac16583b7 add test to query results of /jobs call in api - **PR** `#41170`_: (*lomeroe*) Backport `#41081`_ to 2016.11 @ *2017-05-10T19:58:52Z* - **PR** `#41081`_: (*lomeroe*) Update win_dns_client to use reg.read_value and set_value | refs: `#41170`_ - * ca18b4d Merge pull request `#41170`_ from lomeroe/`bp-41081`_ - * 2af89f2 update mock data + * ca18b4df93 Merge pull request `#41170`_ from lomeroe/`bp-41081`_ + * 2af89f2165 update mock data - * b7fa115 update win_dns_client tests with correct module names + * b7fa115a59 update win_dns_client tests with correct module names - * 4d05a22 Update win_dns_client to use reg.read_value and set_value + * 4d05a22675 Update win_dns_client to use reg.read_value and set_value - **PR** `#41173`_: (*twangboy*) Add silent action to MsgBox for Path Actions @ *2017-05-10T19:57:06Z* - * d7ec37b Merge pull request `#41173`_ from twangboy/fix_installer - * 24b11ff Add release notes + * d7ec37b003 Merge pull request `#41173`_ from twangboy/fix_installer + * 24b11ffdc2 Add release notes - * 96918dc Add silent action to MsgBox for Path Actions + * 96918dcfa6 Add silent action to MsgBox for Path Actions - **PR** `#41158`_: (*Ch3LL*) 2016.11.5 release notes: add additional commits @ *2017-05-09T22:41:40Z* - * 88e93b7 Merge pull request `#41158`_ from Ch3LL/update_2016.11.5 - * 28371aa 2016.11.5 release notes: add additional commits + * 88e93b7fe5 Merge pull request `#41158`_ from Ch3LL/update_2016.11.5 + * 28371aa035 2016.11.5 release notes: add additional commits - **PR** `#41148`_: (*rallytime*) [2016.11] Merge forward from 2016.3 to 2016.11 @ *2017-05-09T20:23:28Z* @@ -993,137 +1166,137 @@ Changes: - **PR** `#41123`_: (*terminalmage*) Add note on lack of support for VSTS in older libssh2 releases. - **PR** `#41122`_: (*terminalmage*) gitfs: refresh env cache during update in masterless - **PR** `#41090`_: (*bbinet*) rdurations should be floats so that they can be summed when profiling - * d2ae7de Merge pull request `#41148`_ from rallytime/merge-2016.11 - * aba35e2 Merge branch '2016.3' into '2016.11' + * d2ae7deff2 Merge pull request `#41148`_ from rallytime/merge-2016.11 + * aba35e20dd Merge branch '2016.3' into '2016.11' - * 2969153 Merge pull request `#41122`_ from terminalmage/masterless-env_cache-fix + * 2969153097 Merge pull request `#41122`_ from terminalmage/masterless-env_cache-fix - * be732f0 gitfs: refresh env cache during update in masterless + * be732f0577 gitfs: refresh env cache during update in masterless - * b8f0a4f Merge pull request `#41123`_ from terminalmage/gitfs-vsts-note + * b8f0a4f108 Merge pull request `#41123`_ from terminalmage/gitfs-vsts-note - * f6a1695 Add note on lack of support for VSTS in older libssh2 releases. + * f6a16956a0 Add note on lack of support for VSTS in older libssh2 releases. - * 8f79b6f Merge pull request `#41090`_ from bbinet/rdurations_float + * 8f79b6f537 Merge pull request `#41090`_ from bbinet/rdurations_float - * fd48a63 rdurations should be floats so that they can be summed when profiling + * fd48a63653 rdurations should be floats so that they can be summed when profiling - **PR** `#41147`_: (*rallytime*) Back-port `#39676`_ to 2016.11 @ *2017-05-09T18:40:44Z* - **PR** `#39676`_: (*F30*) Fix comments about the "hash_type" option | refs: `#41147`_ - * 2156395 Merge pull request `#41147`_ from rallytime/`bp-39676`_ - * 5b55fb2 Fix comments about the "hash_type" option + * 2156395b2e Merge pull request `#41147`_ from rallytime/`bp-39676`_ + * 5b55fb2452 Fix comments about the "hash_type" option - **PR** `#40852`_: (*isbm*) Isbm fix coregrains constants bsc`#1032931`_ @ *2017-05-09T18:35:46Z* - **ISSUE** `#1032931`_: (**) - * a2f359f Merge pull request `#40852`_ from isbm/isbm-fix-coregrains-constants-bsc`#1032931`_ - * f3b12a3 Do not use multiple variables in "with" statement as of lint issues + * a2f359fa13 Merge pull request `#40852`_ from isbm/isbm-fix-coregrains-constants-bsc`#1032931`_ + * f3b12a3f5b Do not use multiple variables in "with" statement as of lint issues - * 35a8d99 Disable the test for a while + * 35a8d99934 Disable the test for a while - * 76cb1b7 Rewrite test case for using no patch decorators + * 76cb1b7150 Rewrite test case for using no patch decorators - * f71af0b Fix lint issues + * f71af0b625 Fix lint issues - * 0e6abb3 Add UT on set_hw_clock on Gentoo + * 0e6abb3e37 Add UT on set_hw_clock on Gentoo - * a2b1d46 Add UT for set_hwclock on Debian + * a2b1d4638c Add UT for set_hwclock on Debian - * 5356a08 Bugfix: use correct grain name for SUSE platform + * 5356a0821a Bugfix: use correct grain name for SUSE platform - * 88e8184 Add UT set_hwclock on SUSE + * 88e8184702 Add UT set_hwclock on SUSE - * 0cd590f Fix UT names + * 0cd590f927 Fix UT names - * bee94ad Add UT for set_hwclock on RedHat + * bee94ade63 Add UT for set_hwclock on RedHat - * dfe2610 Add UT for set_hwclock on Arch + * dfe2610d05 Add UT for set_hwclock on Arch - * d000a8a Add UT for set_hwclock on solaris + * d000a8a6f5 Add UT for set_hwclock on solaris - * d2614ae Fix docstrings + * d2614aedaa Fix docstrings - * 6d78219 Add UT for set_hwclock on AIX + * 6d782191dc Add UT for set_hwclock on AIX - * d303e0d Add UT for AIX on get_hwclock + * d303e0dd8a Add UT for AIX on get_hwclock - * 86f2d83 Add UT on Solaris + * 86f2d83781 Add UT on Solaris - * c3cafed Add UT for Debian on get_hwclock + * c3cafed6d5 Add UT for Debian on get_hwclock - * d337c09 Add UT for RedHat/SUSE platforms on get_hwclock + * d337c09357 Add UT for RedHat/SUSE platforms on get_hwclock - * 501a59c Bugfix: use correct grain for SUSE and RedHat platform + * 501a59ca7e Bugfix: use correct grain for SUSE and RedHat platform - * f25dc5c Add UT for get_hwclock on SUSE platform + * f25dc5c56c Add UT for get_hwclock on SUSE platform - * 08e00c8 Remove dead code + * 08e00c865c Remove dead code - * 1216a0b Add UT for get_hwclock on UTC/localtime + * 1216a0bf12 Add UT for get_hwclock on UTC/localtime - * 39332c7 Remove duplicate code + * 39332c71d3 Remove duplicate code - * 58676c5 Add UT for Debian on set_zone + * 58676c568d Add UT for Debian on set_zone - * 1b9ce37 Add UT for gentoo on set_zone + * 1b9ce37b1b Add UT for gentoo on set_zone - * cf7f766 Bugfix: use correct os_family grain value for SUSE series + * cf7f766a68 Bugfix: use correct os_family grain value for SUSE series - * 6ed9be9 Adjust UT to use correct grain for SUSE series + * 6ed9be985e Adjust UT to use correct grain for SUSE series - * ce4c836 Add UT for set_zone on SUSE series + * ce4c836a60 Add UT for set_zone on SUSE series - * 155a498 Doc fix + * 155a498b49 Doc fix - * a40876c Remove unnecessary mock patch + * a40876cdac Remove unnecessary mock patch - * ffab2db Fix doc for RH UT + * ffab2db213 Fix doc for RH UT - * 72388f7 Add UT for RedHat's set_zone + * 72388f7ae2 Add UT for RedHat's set_zone - * 11595d3 Refactor with setup/teardown + * 11595d3a42 Refactor with setup/teardown - * ce6a06d Bugfix: use correct grain constant for platform + * ce6a06de98 Bugfix: use correct grain constant for platform - * 28072c9 Adjust the test so it is using the right grain for SUSE systems + * 28072c9e41 Adjust the test so it is using the right grain for SUSE systems - * 7a0e4be Add unit test for get_zone and various platforms + * 7a0e4be4f8 Add unit test for get_zone and various platforms - **PR** `#41111`_: (*terminalmage*) Allow "ssl_verify: False" to work with pygit2 @ *2017-05-09T17:56:12Z* - **ISSUE** `#41105`_: (*terminalmage*) ssl_verify gitfs/git_pillar option does not work with pygit2 | refs: `#41111`_ - * 6fa41dc Merge pull request `#41111`_ from terminalmage/issue41105 - * 8c6410e Add notices about ssl_verify only working in 0.23.2 and newer + * 6fa41dc89d Merge pull request `#41111`_ from terminalmage/issue41105 + * 8c6410e3cd Add notices about ssl_verify only working in 0.23.2 and newer - * 98ce829 Support ssl_verify in pygit2 + * 98ce829729 Support ssl_verify in pygit2 - * f73c4b7 Add http(s) auth config docs for GitPython + * f73c4b7167 Add http(s) auth config docs for GitPython - **PR** `#41008`_: (*cro*) Look in /opt/*/lib instead of just /opt/local/lib on Illumos distros. @ *2017-05-09T16:56:00Z* - * 81add1b Merge pull request `#41008`_ from cro/rsax_smos - * a4f7aa1 Look for libcrypto in both /opt/tools and /opt/local on Illumos-based distros. + * 81add1b944 Merge pull request `#41008`_ from cro/rsax_smos + * a4f7aa145e Look for libcrypto in both /opt/tools and /opt/local on Illumos-based distros. - **PR** `#41124`_: (*gtmanfred*) add user_data to digitalocean @ *2017-05-09T16:47:42Z* - * c649725 Merge pull request `#41124`_ from gtmanfred/do - * 2370d93 add user_data to digital ocean + * c649725e9b Merge pull request `#41124`_ from gtmanfred/do + * 2370d9316b add user_data to digital ocean - **PR** `#41127`_: (*tmeneau*) Fix incorrect service.running state response when enable=None and init script returns 0 @ *2017-05-09T16:43:35Z* - **ISSUE** `#41125`_: (*tmeneau*) service.running returns True if enable=None and init script returns 0 | refs: `#41127`_ - * d0a3fcf Merge pull request `#41127`_ from xetus-oss/`fix-41125`_-service-running - * d876656 fix incorrect service.running success response + * d0a3fcf33a Merge pull request `#41127`_ from xetus-oss/`fix-41125`_-service-running + * d8766562c9 fix incorrect service.running success response .. _`#1032931`: https://github.com/saltstack/salt/issues/1032931 @@ -1131,6 +1304,7 @@ Changes: .. _`#22`: https://github.com/saltstack/salt/issues/22 .. _`#30733`: https://github.com/saltstack/salt/issues/30733 .. _`#30777`: https://github.com/saltstack/salt/pull/30777 +.. _`#32400`: https://github.com/saltstack/salt/issues/32400 .. _`#32743`: https://github.com/saltstack/salt/issues/32743 .. _`#34460`: https://github.com/saltstack/salt/issues/34460 .. _`#34775`: https://github.com/saltstack/salt/issues/34775 @@ -1142,22 +1316,28 @@ Changes: .. _`#38061`: https://github.com/saltstack/salt/issues/38061 .. _`#38647`: https://github.com/saltstack/salt/pull/38647 .. _`#38894`: https://github.com/saltstack/salt/issues/38894 +.. _`#38962`: https://github.com/saltstack/salt/issues/38962 +.. _`#39472`: https://github.com/saltstack/salt/pull/39472 .. _`#39575`: https://github.com/saltstack/salt/pull/39575 .. _`#39668`: https://github.com/saltstack/salt/issues/39668 .. _`#39676`: https://github.com/saltstack/salt/pull/39676 .. _`#39850`: https://github.com/saltstack/salt/pull/39850 .. _`#39918`: https://github.com/saltstack/salt/issues/39918 .. _`#39939`: https://github.com/saltstack/salt/issues/39939 +.. _`#40005`: https://github.com/saltstack/salt/issues/40005 .. _`#40155`: https://github.com/saltstack/salt/issues/40155 .. _`#40177`: https://github.com/saltstack/salt/issues/40177 .. _`#40246`: https://github.com/saltstack/salt/pull/40246 .. _`#40410`: https://github.com/saltstack/salt/issues/40410 .. _`#40446`: https://github.com/saltstack/salt/issues/40446 +.. _`#40543`: https://github.com/saltstack/salt/pull/40543 .. _`#40605`: https://github.com/saltstack/salt/issues/40605 .. _`#40748`: https://github.com/saltstack/salt/issues/40748 .. _`#40752`: https://github.com/saltstack/salt/pull/40752 .. _`#40837`: https://github.com/saltstack/salt/pull/40837 +.. _`#40845`: https://github.com/saltstack/salt/issues/40845 .. _`#40852`: https://github.com/saltstack/salt/pull/40852 +.. _`#40878`: https://github.com/saltstack/salt/issues/40878 .. _`#40902`: https://github.com/saltstack/salt/pull/40902 .. _`#40912`: https://github.com/saltstack/salt/issues/40912 .. _`#40934`: https://github.com/saltstack/salt/pull/40934 @@ -1255,6 +1435,7 @@ Changes: .. _`#41439`: https://github.com/saltstack/salt/pull/41439 .. _`#41442`: https://github.com/saltstack/salt/pull/41442 .. _`#41443`: https://github.com/saltstack/salt/pull/41443 +.. _`#41449`: https://github.com/saltstack/salt/pull/41449 .. _`#41453`: https://github.com/saltstack/salt/pull/41453 .. _`#41456`: https://github.com/saltstack/salt/pull/41456 .. _`#41464`: https://github.com/saltstack/salt/pull/41464 @@ -1280,10 +1461,12 @@ Changes: .. _`#41561`: https://github.com/saltstack/salt/pull/41561 .. _`#41569`: https://github.com/saltstack/salt/pull/41569 .. _`#41575`: https://github.com/saltstack/salt/pull/41575 +.. _`#41585`: https://github.com/saltstack/salt/pull/41585 .. _`#41596`: https://github.com/saltstack/salt/pull/41596 .. _`#41597`: https://github.com/saltstack/salt/pull/41597 .. _`#41599`: https://github.com/saltstack/salt/pull/41599 .. _`#41611`: https://github.com/saltstack/salt/pull/41611 +.. _`#41615`: https://github.com/saltstack/salt/pull/41615 .. _`#41616`: https://github.com/saltstack/salt/pull/41616 .. _`#41626`: https://github.com/saltstack/salt/issues/41626 .. _`#41629`: https://github.com/saltstack/salt/issues/41629 @@ -1307,6 +1490,7 @@ Changes: .. _`#41692`: https://github.com/saltstack/salt/pull/41692 .. _`#41693`: https://github.com/saltstack/salt/pull/41693 .. _`#41694`: https://github.com/saltstack/salt/pull/41694 +.. _`#41695`: https://github.com/saltstack/salt/pull/41695 .. _`#41696`: https://github.com/saltstack/salt/pull/41696 .. _`#41697`: https://github.com/saltstack/salt/pull/41697 .. _`#41699`: https://github.com/saltstack/salt/pull/41699 @@ -1319,6 +1503,27 @@ Changes: .. _`#41711`: https://github.com/saltstack/salt/pull/41711 .. _`#41723`: https://github.com/saltstack/salt/pull/41723 .. _`#41731`: https://github.com/saltstack/salt/pull/41731 +.. _`#41733`: https://github.com/saltstack/salt/issues/41733 +.. _`#41740`: https://github.com/saltstack/salt/pull/41740 +.. _`#41749`: https://github.com/saltstack/salt/pull/41749 +.. _`#41750`: https://github.com/saltstack/salt/pull/41750 +.. _`#41753`: https://github.com/saltstack/salt/pull/41753 +.. _`#41756`: https://github.com/saltstack/salt/pull/41756 +.. _`#41768`: https://github.com/saltstack/salt/pull/41768 +.. _`#41776`: https://github.com/saltstack/salt/pull/41776 +.. _`#41778`: https://github.com/saltstack/salt/issues/41778 +.. _`#41783`: https://github.com/saltstack/salt/pull/41783 +.. _`#41785`: https://github.com/saltstack/salt/issues/41785 +.. _`#41786`: https://github.com/saltstack/salt/pull/41786 +.. _`#41787`: https://github.com/saltstack/salt/pull/41787 +.. _`#41801`: https://github.com/saltstack/salt/pull/41801 +.. _`#41803`: https://github.com/saltstack/salt/pull/41803 +.. _`#41812`: https://github.com/saltstack/salt/pull/41812 +.. _`#41816`: https://github.com/saltstack/salt/pull/41816 +.. _`#41837`: https://github.com/saltstack/salt/pull/41837 +.. _`#41839`: https://github.com/saltstack/salt/pull/41839 +.. _`#41857`: https://github.com/saltstack/salt/pull/41857 +.. _`#41861`: https://github.com/saltstack/salt/pull/41861 .. _`bp-39676`: https://github.com/saltstack/salt/pull/39676 .. _`bp-39850`: https://github.com/saltstack/salt/pull/39850 .. _`bp-40246`: https://github.com/saltstack/salt/pull/40246 @@ -1327,15 +1532,17 @@ Changes: .. _`bp-41243`: https://github.com/saltstack/salt/pull/41243 .. _`bp-41251`: https://github.com/saltstack/salt/pull/41251 .. _`bp-41301`: https://github.com/saltstack/salt/pull/41301 +.. _`bp-41449`: https://github.com/saltstack/salt/pull/41449 .. _`bp-41487`: https://github.com/saltstack/salt/pull/41487 .. _`bp-41533`: https://github.com/saltstack/salt/pull/41533 .. _`bp-41551`: https://github.com/saltstack/salt/pull/41551 .. _`bp-41575`: https://github.com/saltstack/salt/pull/41575 +.. _`bp-41615`: https://github.com/saltstack/salt/pull/41615 .. _`bp-41670`: https://github.com/saltstack/salt/pull/41670 .. _`fix-40155`: https://github.com/saltstack/salt/issues/40155 .. _`fix-40410`: https://github.com/saltstack/salt/issues/40410 .. _`fix-40446`: https://github.com/saltstack/salt/issues/40446 .. _`fix-40605`: https://github.com/saltstack/salt/issues/40605 +.. _`fix-40878`: https://github.com/saltstack/salt/issues/40878 .. _`fix-41125`: https://github.com/saltstack/salt/issues/41125 .. _`fix-41688`: https://github.com/saltstack/salt/issues/41688 - diff --git a/doc/topics/releases/2016.11.7.rst b/doc/topics/releases/2016.11.7.rst new file mode 100644 index 00000000000..2ad821ff22c --- /dev/null +++ b/doc/topics/releases/2016.11.7.rst @@ -0,0 +1,15 @@ +============================ +Salt 2016.11.7 Release Notes +============================ + +Version 2016.11.7 is a bugfix release for :ref:`2016.11.0 `. + +Changes for v2016.11.6..v2016.11.7 +---------------------------------- + +Security Fix +============ + +CVE-2017-12791 Maliciously crafted minion IDs can cause unwanted directory traversals on the Salt-master + +Correct a flaw in minion id validation which could allow certain minions to authenticate to a master despite not having the correct credentials. To exploit the vulnerability, an attacker must create a salt-minion with an ID containing characters that will cause a directory traversal. Credit for discovering the security flaw goes to: Vernhk@qq.com diff --git a/doc/topics/releases/2016.3.0.rst b/doc/topics/releases/2016.3.0.rst index 01e1a5751c5..b28fed2c662 100644 --- a/doc/topics/releases/2016.3.0.rst +++ b/doc/topics/releases/2016.3.0.rst @@ -175,6 +175,10 @@ they are being loaded for the correct proxytype, example below: return False +.. note:: + ``salt.utils.is_proxy()`` has been renamed to + ``salt.utils.platform.is_proxy`` as of the Oxygen release. + The try/except block above exists because grains are processed very early in the proxy minion startup process, sometimes earlier than the proxy key in the ``__opts__`` dictionary is populated. diff --git a/doc/topics/releases/2016.3.7.rst b/doc/topics/releases/2016.3.7.rst index 3c47df7eda6..0fca807661b 100644 --- a/doc/topics/releases/2016.3.7.rst +++ b/doc/topics/releases/2016.3.7.rst @@ -4,23 +4,12 @@ Salt 2016.3.7 Release Notes Version 2016.3.7 is a bugfix release for :ref:`2016.3.0 `. -New master configuration option `allow_minion_key_revoke`, defaults to True. This option -controls whether a minion can request that the master revoke its key. When True, a minion -can request a key revocation and the master will comply. If it is False, the key will not -be revoked by the msater. +Changes for v2016.3.6..v2016.3.7 +-------------------------------- -New master configuration option `require_minion_sign_messages` -This requires that minions cryptographically sign the messages they -publish to the master. If minions are not signing, then log this information -at loglevel 'INFO' and drop the message without acting on it. +Security Fix +============ -New master configuration option `drop_messages_signature_fail` -Drop messages from minions when their signatures do not validate. -Note that when this option is False but `require_minion_sign_messages` is True -minions MUST sign their messages but the validity of their signatures -is ignored. +CVE-2017-12791 Maliciously crafted minion IDs can cause unwanted directory traversals on the Salt-master -New minion configuration option `minion_sign_messages` -Causes the minion to cryptographically sign the payload of messages it places -on the event bus for the master. The payloads are signed with the minion's -private key so the master can verify the signature with its public key. +Correct a flaw in minion id validation which could allow certain minions to authenticate to a master despite not having the correct credentials. To exploit the vulnerability, an attacker must create a salt-minion with an ID containing characters that will cause a directory traversal. Credit for discovering the security flaw goes to: Vernhk@qq.com diff --git a/doc/topics/releases/2016.3.8.rst b/doc/topics/releases/2016.3.8.rst new file mode 100644 index 00000000000..c5f0c01da80 --- /dev/null +++ b/doc/topics/releases/2016.3.8.rst @@ -0,0 +1,29 @@ +=========================== +Salt 2016.3.8 Release Notes +=========================== + +Version 2016.3.8 is a bugfix release for :ref:`2016.3.0 `. + +Changes for v2016.3.7..v2016.3.8 +-------------------------------- + +New master configuration option `allow_minion_key_revoke`, defaults to True. This option +controls whether a minion can request that the master revoke its key. When True, a minion +can request a key revocation and the master will comply. If it is False, the key will not +be revoked by the msater. + +New master configuration option `require_minion_sign_messages` +This requires that minions cryptographically sign the messages they +publish to the master. If minions are not signing, then log this information +at loglevel 'INFO' and drop the message without acting on it. + +New master configuration option `drop_messages_signature_fail` +Drop messages from minions when their signatures do not validate. +Note that when this option is False but `require_minion_sign_messages` is True +minions MUST sign their messages but the validity of their signatures +is ignored. + +New minion configuration option `minion_sign_messages` +Causes the minion to cryptographically sign the payload of messages it places +on the event bus for the master. The payloads are signed with the minion's +private key so the master can verify the signature with its public key. diff --git a/doc/topics/releases/2017.7.0.rst b/doc/topics/releases/2017.7.0.rst index 5f0d1ae2c50..a0e257b3473 100644 --- a/doc/topics/releases/2017.7.0.rst +++ b/doc/topics/releases/2017.7.0.rst @@ -4,6 +4,45 @@ Salt 2017.7.0 Release Notes - Codename Nitrogen =============================================== + +======== +Python 3 +======== + +The 2017.7 Salt Release adds initial Python 3 support. + +The default Python version of Salt will remain Python 2, although Python 3 packages will be supplied for users who want to help test this new feature. + +====================== +Python 2.6 Deprecation +====================== + +Salt will no longer support Python 2.6. We will provide python2.7 packages on our repo_ for RedHat and CentOS 6 to ensure users can still run Salt on these platforms. + +.. _repo: https://repo.saltstack.com/ + +============ +Known Issues +============ +The following salt-cloud drivers have known issues running with Python 3. These drivers will not work with Python 3, and Python 2.7 should be used instead: + +- Joyent + +- When running under Python 3, users who require Unicode support should ensure that a locale is set on their machines. + Users using the `C` locale are advised to switch to a UTF-aware locale to ensure proper functionality with Salt with Python 3. + + +Remember to update the Salt Master first +======================================== +Salt's policy has always been that when upgrading, the minion should never be +on a newer version than the master. Specifically with this update, because of +changes in the fileclient, the 2017.7 minion requires a 2017.7 master. + +Backwards compatiblity is still maintained, so older minions can still be used. + +More information can be found in the :ref:`Salt FAQ` + + States Added for Management of systemd Unit Masking =================================================== @@ -83,13 +122,12 @@ State Module Changes # After run_something: module.run: - mymodule.something: + - mymodule.something: - name: some name - first_arg: one - second_arg: two - do_stuff: True - Since a lot of users are already using :py:func:`module.run ` states, this new behavior must currently be explicitly turned on, to allow users to take their time updating their SLS @@ -97,6 +135,36 @@ State Module Changes the next feature release of Salt (Oxygen) and the old usage will no longer be supported at that time. + Another feature of the new :py:func:`module.run ` is that + it allows calling many functions in a single batch, such as: + + .. code-block:: yaml + + run_something: + module.run: + - mymodule.function_without_parameters: + - mymodule.another_function: + - myparam + - my_other_param + + In a rare case that you have a function that needs to be called several times but + with the different parameters, an additional feature of "tagging" is to the + rescue. In order to tag a function, use a colon delimeter. For example: + + .. code-block:: yaml + + run_something: + module.run: + - mymodule.same_function:1: + - mymodule.same_function:2: + - myparam + - my_other_param + - mymodule.same_function:3: + - foo: bar + + The example above will run `mymodule.same_function` three times with the + different parameters. + To enable the new behavior for :py:func:`module.run `, add the following to the minion config file: @@ -104,6 +172,7 @@ State Module Changes use_superseded: - module.run + - The default for the ``fingerprint_hash_type`` option used in the ``present`` function in the :mod:`ssh ` state changed from ``md5`` to ``sha256``. @@ -161,6 +230,7 @@ Wildcard Versions in :py:func:`pkg.installed ` States - The :py:func:`pkg.installed ` state now supports wildcards in package versions, for the following platforms: + - SUSE/openSUSE Leap/Thumbleweed - Debian/Ubuntu - RHEL/CentOS - Arch Linux @@ -542,6 +612,38 @@ Using the new ``roster_order`` configuration syntax it's now possible to compose of grains, pillar and mine data and even Salt SDB URLs. The new release is also fully IPv4 and IPv6 enabled and even has support for CIDR ranges. +Salt-SSH Default Options +======================== + +Defaults for rosters can now be set, so that they don't have to be set on every +entry in a roster or specified from the commandline. + +The new option is :ref:`roster_defaults` and is specified in +the master config file. + +.. code-block:: yaml + + roster_defaults: + user: daniel + sudo: True + priv: /root/.ssh/id_rsa + tty: True + +Blacklist or Whitelist Extmod Sync +================================== + +The modules that are synced to minions can now be limited. + +The following configuration options have been added for the master: + +- :conf_master:`extmod_whitelist` +- :conf_master:`extmod_blacklist` + +and for the minion: + +- :conf_minion:`extmod_whitelist` +- :conf_minion:`extmod_blacklist` + Additional Features =================== @@ -604,6 +706,7 @@ Execution modules - :mod:`salt.modules.grafana4 ` - :mod:`salt.modules.heat ` - :mod:`salt.modules.icinga2 ` +- :mod:`salt.modules.kubernetes ` - :mod:`salt.modules.logmod ` - :mod:`salt.modules.mattermost ` - :mod:`salt.modules.namecheap_dns ` @@ -682,6 +785,7 @@ States - :mod:`salt.states.icinga2 ` - :mod:`salt.states.influxdb_continuous_query ` - :mod:`salt.states.influxdb_retention_policy ` +- :mod:`salt.states.kubernetes ` - :mod:`salt.states.logadm ` - :mod:`salt.states.logrotate ` - :mod:`salt.states.msteams ` @@ -871,3 +975,13 @@ The ``glusterfs`` state had the following function removed: The ``openvswitch_port`` state had the following change: - The ``type`` option was removed from the ``present`` function. Please use ``tunnel_type`` instead. + +Build Notes +=========== + +Windows Installer Packages +-------------------------- + +Windows Installer packages have been patched with the following PR: 42347_ + +.. _42347: https://github.com/saltstack/salt/pull/42347 diff --git a/doc/topics/releases/2017.7.1.rst b/doc/topics/releases/2017.7.1.rst new file mode 100644 index 00000000000..7cc616c94bc --- /dev/null +++ b/doc/topics/releases/2017.7.1.rst @@ -0,0 +1,190 @@ +============================ +Salt 2017.7.1 Release Notes +============================ + +Version 2017.7.1 is a bugfix release for :ref:`2017.7.0 `. + +Security Fix +============ + +CVE-2017-12791 Maliciously crafted minion IDs can cause unwanted directory traversals on the Salt-master + +Correct a flaw in minion id validation which could allow certain minions to authenticate to a master despite not having the correct credentials. To exploit the vulnerability, an attacker must create a salt-minion with an ID containing characters that will cause a directory traversal. Credit for discovering the security flaw goes to: Vernhk@qq.com + +Changes for v2017.7.0..v2017.7.1 +-------------------------------- + +Extended changelog courtesy of Todd Stansell (https://github.com/tjstansell/salt-changelogs): + +*Generated at: 2017-07-26T01:09:40Z* + +Statistics: + +- Total Merges: **11** +- Total Issue references: **9** +- Total PR references: **22** + +Changes: + + +- **PR** `#42548`_: (*gtmanfred*) pass in empty kwarg for reactor + @ *2017-07-26T00:41:20Z* + + - **ISSUE** `#460`_: (*whiteinge*) Add a topic and a ref for modules/states/returners/renderers/runners + | refs: `#42548`_ + * 711b742c54 Merge pull request `#42548`_ from gtmanfred/2017.7.1 + * 0257c1dc32 pass in empty kwarg for reactor + + * b948e980d2 update chunk, not kwarg in chunk + +- **PR** `#42522`_: (*gtmanfred*) pacman wildcard is only for repository installs + @ *2017-07-24T20:51:05Z* + + - **ISSUE** `#42519`_: (*xuhcc*) Error when installing package from file under Arch Linux + | refs: `#42522`_ + * 50c1635dcc Merge pull request `#42522`_ from gtmanfred/2017.7.1 + * 7787fb9e1b pacman wildcard is only for repository installs + +- **PR** `#42508`_: (*rallytime*) Back-port `#42474`_ to 2017.7.1 + @ *2017-07-24T20:49:51Z* + + - **PR** `#42474`_: (*whiteinge*) Cmd arg kwarg parsing test + | refs: `#42508`_ + - **PR** `#39646`_: (*terminalmage*) Handle deprecation of passing string args to load_args_and_kwargs + | refs: `#42474`_ + * 05c07ac049 Merge pull request `#42508`_ from rallytime/`bp-42474`_ + * 76fb074433 Add a test.arg variant that cleans the pub kwargs by default + + * 624f63648e Lint fixes + + * d246a5fc61 Add back support for string kwargs + + * 854e098aa0 Add LocalClient.cmd test for arg/kwarg parsing + +- **PR** `#42472`_: (*rallytime*) Back-port `#42435`_ to 2017.7.1 + @ *2017-07-24T15:11:13Z* + + - **ISSUE** `#42427`_: (*grichmond-salt*) Issue Passing Variables created from load_json as Inline Pillar Between States + | refs: `#42435`_ + - **PR** `#42435`_: (*terminalmage*) Modify our custom YAML loader to treat unicode literals as unicode strings + | refs: `#42472`_ + * 95fe2558e4 Merge pull request `#42472`_ from rallytime/`bp-42435`_ + * 5c47af5b98 Modify our custom YAML loader to treat unicode literals as unicode strings + +- **PR** `#42473`_: (*rallytime*) Back-port `#42436`_ to 2017.7.1 + @ *2017-07-24T15:10:29Z* + + - **ISSUE** `#42374`_: (*tyhunt99*) [2017.7.0] salt-run mange.versions throws exception if minion is offline or unresponsive + | refs: `#42436`_ + - **PR** `#42436`_: (*garethgreenaway*) Fixes to versions function in manage runner + | refs: `#42473`_ + * 5b99d45f54 Merge pull request `#42473`_ from rallytime/`bp-42436`_ + * 82ed919803 Updating the versions function inside the manage runner to account for when a minion is offline and we are unable to determine it's version. + +- **PR** `#42471`_: (*rallytime*) Back-port `#42399`_ to 2017.7.1 + @ *2017-07-24T15:09:50Z* + + - **ISSUE** `#42381`_: (*zebooka*) Git.detached broken in 2017.7.0 + | refs: `#42399`_ + - **ISSUE** `#38878`_: (*tomlaredo*) [Naming consistency] git.latest "rev" option VS git.detached "ref" option + | refs: `#38898`_ + - **PR** `#42399`_: (*rallytime*) Update old "ref" references to "rev" in git.detached state + | refs: `#42471`_ + - **PR** `#38898`_: (*terminalmage*) git.detached: rename ref to rev for consistency + | refs: `#42399`_ + * 3d1a2d3f9f Merge pull request `#42471`_ from rallytime/`bp-42399`_ + * b9a4669e5a Update old "ref" references to "rev" in git.detached state + +- **PR** `#42470`_: (*rallytime*) Back-port `#42031`_ to 2017.7.1 + @ *2017-07-24T15:09:30Z* + + - **ISSUE** `#42400`_: (*Enquier*) Conflict in execution of passing pillar data to orch/reactor event executions 2017.7.0 + | refs: `#42031`_ + - **PR** `#42031`_: (*skizunov*) Fix: Reactor emits critical error + | refs: `#42470`_ + * 09766bccbc Merge pull request `#42470`_ from rallytime/`bp-42031`_ + * 0a0c6287a4 Fix: Reactor emits critical error + +- **PR** `#42469`_: (*rallytime*) Back-port `#42027`_ to 2017.7.1 + @ *2017-07-21T22:41:02Z* + + - **ISSUE** `#41949`_: (*jrporcaro*) Event returner doesn't work with Windows Master + | refs: `#42027`_ + - **PR** `#42027`_: (*gtmanfred*) import salt.minion for EventReturn for Windows + | refs: `#42469`_ + * d7b172a15b Merge pull request `#42469`_ from rallytime/`bp-42027`_ + * ed612b4ee7 import salt.minion for EventReturn for Windows + +- **PR** `#42466`_: (*rallytime*) Back-port `#42452`_ to 2017.7.1 + @ *2017-07-21T19:41:24Z* + + - **PR** `#42452`_: (*Ch3LL*) update windows urls to new py2/py3 naming scheme + | refs: `#42466`_ + * 8777b1a825 Merge pull request `#42466`_ from rallytime/`bp-42452`_ + * c10196f68c update windows urls to new py2/py3 naming scheme + +- **PR** `#42439`_: (*rallytime*) Back-port `#42409`_ to 2017.7.1 + @ *2017-07-21T17:38:10Z* + + - **PR** `#42409`_: (*twangboy*) Add Scripts to build Py3 on Mac + | refs: `#42439`_ + * fceaaf41d0 Merge pull request `#42439`_ from rallytime/`bp-42409`_ + * 8176964b41 Remove build and dist, sign pkgs + + * 2c14d92a07 Fix hard coded pip path + + * 82fdd7c2e1 Add support for Py3 + + * 2478447246 Update Python and other reqs + +- **PR** `#42441`_: (*rallytime*) Back-port `#42433`_ to 2017.7.1 + @ *2017-07-21T17:37:01Z* + + - **ISSUE** `#42403`_: (*astronouth7303*) [2017.7] Pillar empty when state is applied from orchestrate + | refs: `#42433`_ + - **PR** `#42433`_: (*terminalmage*) Only force saltenv/pillarenv to be a string when not None + | refs: `#42441`_ + * 660400560b Merge pull request `#42441`_ from rallytime/`bp-42433`_ + * 17f347123a Only force saltenv/pillarenv to be a string when not None + + +.. _`#38878`: https://github.com/saltstack/salt/issues/38878 +.. _`#38898`: https://github.com/saltstack/salt/pull/38898 +.. _`#39646`: https://github.com/saltstack/salt/pull/39646 +.. _`#41949`: https://github.com/saltstack/salt/issues/41949 +.. _`#42027`: https://github.com/saltstack/salt/pull/42027 +.. _`#42031`: https://github.com/saltstack/salt/pull/42031 +.. _`#42374`: https://github.com/saltstack/salt/issues/42374 +.. _`#42381`: https://github.com/saltstack/salt/issues/42381 +.. _`#42399`: https://github.com/saltstack/salt/pull/42399 +.. _`#42400`: https://github.com/saltstack/salt/issues/42400 +.. _`#42403`: https://github.com/saltstack/salt/issues/42403 +.. _`#42409`: https://github.com/saltstack/salt/pull/42409 +.. _`#42427`: https://github.com/saltstack/salt/issues/42427 +.. _`#42433`: https://github.com/saltstack/salt/pull/42433 +.. _`#42435`: https://github.com/saltstack/salt/pull/42435 +.. _`#42436`: https://github.com/saltstack/salt/pull/42436 +.. _`#42439`: https://github.com/saltstack/salt/pull/42439 +.. _`#42441`: https://github.com/saltstack/salt/pull/42441 +.. _`#42452`: https://github.com/saltstack/salt/pull/42452 +.. _`#42466`: https://github.com/saltstack/salt/pull/42466 +.. _`#42469`: https://github.com/saltstack/salt/pull/42469 +.. _`#42470`: https://github.com/saltstack/salt/pull/42470 +.. _`#42471`: https://github.com/saltstack/salt/pull/42471 +.. _`#42472`: https://github.com/saltstack/salt/pull/42472 +.. _`#42473`: https://github.com/saltstack/salt/pull/42473 +.. _`#42474`: https://github.com/saltstack/salt/pull/42474 +.. _`#42508`: https://github.com/saltstack/salt/pull/42508 +.. _`#42519`: https://github.com/saltstack/salt/issues/42519 +.. _`#42522`: https://github.com/saltstack/salt/pull/42522 +.. _`#42548`: https://github.com/saltstack/salt/pull/42548 +.. _`#460`: https://github.com/saltstack/salt/issues/460 +.. _`bp-42027`: https://github.com/saltstack/salt/pull/42027 +.. _`bp-42031`: https://github.com/saltstack/salt/pull/42031 +.. _`bp-42399`: https://github.com/saltstack/salt/pull/42399 +.. _`bp-42409`: https://github.com/saltstack/salt/pull/42409 +.. _`bp-42433`: https://github.com/saltstack/salt/pull/42433 +.. _`bp-42435`: https://github.com/saltstack/salt/pull/42435 +.. _`bp-42436`: https://github.com/saltstack/salt/pull/42436 +.. _`bp-42452`: https://github.com/saltstack/salt/pull/42452 +.. _`bp-42474`: https://github.com/saltstack/salt/pull/42474 diff --git a/doc/topics/releases/oxygen.rst b/doc/topics/releases/oxygen.rst index b5765dc1348..4c651bfce95 100644 --- a/doc/topics/releases/oxygen.rst +++ b/doc/topics/releases/oxygen.rst @@ -13,18 +13,655 @@ packages on minions which use :mod:`yum/dnf ` or :mod:`apt ` state and in the ``pkg.install`` remote execution function. -Configuration Option Deprecations +:ref:`Master Tops ` Changes +----------------------------------------------- + +When both :ref:`Master Tops ` and a :ref:`Top File +` produce SLS matches for a given minion, the matches were being +merged in an unpredictable manner which did not preserve ordering. This has +been changed. The top file matches now execute in the expected order, followed +by any master tops matches that are not matched via a top file. + +To make master tops matches execute first, followed by top file matches, set +the new :conf_minion:`master_tops_first` minion config option to ``True``. + +LDAP via External Authentication Changes +---------------------------------------- +In this release of Salt, if LDAP Bind Credentials are supplied, then +these credentials will be used for all LDAP access except the first +authentication when a job is submitted. The first authentication will +use the user's credentials as passed on the CLI. This behavior is to +accommodate certain two-factor authentication schemes where the authentication +token can only be used once. + +In previous releases the bind credentials would only be used to determine +the LDAP user's existence and group membership. The user's LDAP credentials +were used from then on. + +Stormpath External Authentication Removed +----------------------------------------- + +Per Stormpath's announcement, their API will be shutting down on 8/17/2017 at +noon PST so the Stormpath external authentication module has been removed. + +https://stormpath.com/oktaplusstormpath + +New NaCl Renderer +----------------- + +A new renderer has been added for encrypted data. + +New support for Cisco UCS Chassis --------------------------------- +The salt proxy minion now allows for control of Cisco USC chassis. See +the `cimc` modules for details. + +New salt-ssh roster +------------------- + +A new roster has been added that allows users to pull in a list of hosts +for salt-ssh targeting from a ~/.ssh configuration. For full details, +please see the `sshconfig` roster. + +New GitFS Features +------------------ + +Two new features which affect how GitFS maps branches/tags to fileserver +environments (i.e. ``saltenvs``) have been added: + +1. It is now possible to completely turn off Salt's default mapping logic + (aside from the mapping of the ``base`` saltenv). This can be triggered + using the new :conf_master:`gitfs_disable_saltenv_mapping` config option. + + .. note:: + When this is disabled, only the ``base`` saltenv and any configured + using :ref:`per-saltenv configuration parameters + ` will be available. + +2. The types of refs which Salt will use as saltenvs can now be controlled. In + previous releases, branches and tags were both mapped as environments, and + individual commit SHAs could be specified as saltenvs in states (and when + caching files using :py:func:`cp.cache_file `). + Using the new :conf_master:`gitfs_ref_types` config option, the types of + refs which are used as saltenvs can be restricted. This makes it possible to + ignore all tags and use branches only, and also to keep SHAs from being made + available as saltenvs. + +Salt Cloud Features +------------------- + +Pre-Flight Commands +=================== + +Support has been added for specified "preflight commands" to run on a VM before +the deploy script is run. These must be defined as a list in a cloud configuration +file. For example: + +.. code-block:: yaml + + my-cloud-profile: + provider: linode-config + image: Ubuntu 16.04 LTS + size: Linode 2048 + preflight_cmds: + - whoami + - echo 'hello world!' + +These commands will run in sequence **before** the bootstrap script is executed. + +Newer PyWinRM Versions +---------------------- + +Versions of ``pywinrm>=0.2.1`` are finally able to disable validation of self +signed certificates. :ref:`Here` for more information. + +DigitalOcean +------------ + +The DigitalOcean driver has been renamed to conform to the companies name. The +new driver name is ``digitalocean``. The old name ``digital_ocean`` and a +short one ``do`` will still be supported through virtual aliases, this is mostly +cosmetic. + +Solaris Logical Domains In Virtual Grain +---------------------------------------- + +Support has been added to the ``virtual`` grain for detecting Solaris LDOMs +running on T-Series SPARC hardware. The ``virtual_subtype`` grain is +populated as a list of domain roles. + +Lists of comments in state returns +---------------------------------- + +State functions can now return a list of strings for the ``comment`` field, +as opposed to only a single string. +This is meant to ease writing states with multiple or multi-part comments. + +Beacon configuration changes +---------------------------- + +In order to remain consistent and to align with other Salt components such as states, +support for configuring beacons using dictionary based configuration has been deprecated +in favor of list based configuration. All beacons have a validation function which will +check the configuration for the correct format and only load if the validation passes. + +- ``avahi_announce`` beacon + + Old behavior: + ``` + beacons: + avahi_announce: + run_once: True + servicetype: _demo._tcp + port: 1234 + txt: + ProdName: grains.productname + SerialNo: grains.serialnumber + Comments: 'this is a test' + ``` + + New behavior: + ``` + beacons: + avahi_announce: + - run_once: True + - servicetype: _demo._tcp + - port: 1234 + - txt: + ProdName: grains.productname + SerialNo: grains.serialnumber + Comments: 'this is a test' + ``` + + - ``bonjour_announce`` beacon + + Old behavior: + ``` + beacons: + bonjour_announce: + run_once: True + servicetype: _demo._tcp + port: 1234 + txt: + ProdName: grains.productname + SerialNo: grains.serialnumber + Comments: 'this is a test' + ``` + + New behavior: + ``` + beacons: + bonjour_announce: + - run_once: True + - servicetype: _demo._tcp + - port: 1234 + - txt: + ProdName: grains.productname + SerialNo: grains.serialnumber + Comments: 'this is a test' + ``` + +- ``btmp`` beacon + + Old behavior: + ``` + beacons: + btmp: {} + ``` + + New behavior: + ``` + beacons: + btmp: [] + + ``` + +- ``glxinfo`` beacon + + Old behavior: + ``` + beacons: + glxinfo: + user: frank + screen_event: True + ``` + + New behavior: + ``` + beacons: + glxinfo: + - user: frank + - screen_event: True + ``` + +- ``haproxy`` beacon + + Old behavior: + ``` + beacons: + haproxy: + - www-backend: + threshold: 45 + servers: + - web1 + - web2 + - interval: 120 + ``` + + New behavior: + ``` + beacons: + haproxy: + - backends: + www-backend: + threshold: 45 + servers: + - web1 + - web2 + - interval: 120 + ``` + +- ``inotify`` beacon + + Old behavior: + ``` + beacons: + inotify: + /path/to/file/or/dir: + mask: + - open + - create + - close_write + recurse: True + auto_add: True + exclude: + - /path/to/file/or/dir/exclude1 + - /path/to/file/or/dir/exclude2 + - /path/to/file/or/dir/regex[a-m]*$: + regex: True + coalesce: True + ``` + + New behavior: + ``` + beacons: + inotify: + - files: + /path/to/file/or/dir: + mask: + - open + - create + - close_write + recurse: True + auto_add: True + exclude: + - /path/to/file/or/dir/exclude1 + - /path/to/file/or/dir/exclude2 + - /path/to/file/or/dir/regex[a-m]*$: + regex: True + - coalesce: True +``` + +- ``journald`` beacon + + Old behavior: + ``` + beacons: + journald: + sshd: + SYSLOG_IDENTIFIER: sshd + PRIORITY: 6 + ``` + + New behavior: + ``` + beacons: + journald: + - services: + sshd: + SYSLOG_IDENTIFIER: sshd + PRIORITY: 6 + ``` + +- ``load`` beacon + + Old behavior: + ``` + beacons: + load: + 1m: + - 0.0 + - 2.0 + 5m: + - 0.0 + - 1.5 + 15m: + - 0.1 + - 1.0 + emitatstartup: True + onchangeonly: False + ``` + + New behavior: + ``` + beacons: + load: + - averages: + 1m: + - 0.0 + - 2.0 + 5m: + - 0.0 + - 1.5 + 15m: + - 0.1 + - 1.0 + - emitatstartup: True + - onchangeonly: False + ``` + +- ``log`` beacon + + Old behavior: + ``` + beacons: + log: + file: + : + regex: + ``` + + New behavior: + ``` + beacons: + log: + - file: + - tags: + : + regex: + ``` + +- ``network_info`` beacon + + Old behavior: + ``` + beacons: + network_info: + - eth0: + type: equal + bytes_sent: 100000 + bytes_recv: 100000 + packets_sent: 100000 + packets_recv: 100000 + errin: 100 + errout: 100 + dropin: 100 + dropout: 100 + ``` + + New behavior: + ``` + beacons: + network_info: + - interfaces: + eth0: + type: equal + bytes_sent: 100000 + bytes_recv: 100000 + packets_sent: 100000 + packets_recv: 100000 + errin: 100 + errout: 100 + dropin: 100 + dropout: 100 + ``` + +- ``network_settings`` beacon + + Old behavior: + ``` + beacons: + network_settings: + eth0: + ipaddr: + promiscuity: + onvalue: 1 + eth1: + linkmode: + ``` + + New behavior: + ``` + beacons: + network_settings: + - interfaces: + - eth0: + ipaddr: + promiscuity: + onvalue: 1 + - eth1: + linkmode: + ``` + +- ``proxy_example`` beacon + + Old behavior: + ``` + beacons: + proxy_example: + endpoint: beacon + ``` + + New behavior: + ``` + beacons: + proxy_example: + - endpoint: beacon + ``` + +- ``ps`` beacon + + Old behavior: + ``` + beacons: + ps: + - salt-master: running + - mysql: stopped + ``` + + New behavior: + ``` + beacons: + ps: + - processes: + salt-master: running + mysql: stopped + ``` + +- ``salt_proxy`` beacon + + Old behavior: + ``` + beacons: + salt_proxy: + - p8000: {} + - p8001: {} + ``` + + New behavior: + ``` + beacons: + salt_proxy: + - proxies: + p8000: {} + p8001: {} + ``` + +- ``sensehat`` beacon + + Old behavior: + ``` + beacons: + sensehat: + humidity: 70% + temperature: [20, 40] + temperature_from_pressure: 40 + pressure: 1500 + ``` + + New behavior: + ``` + beacons: + sensehat: + - sensors: + humidity: 70% + temperature: [20, 40] + temperature_from_pressure: 40 + pressure: 1500 + ``` + +- ``service`` beacon + + Old behavior: + ``` + beacons: + service: + salt-master: + mysql: + + ``` + + New behavior: + ``` + beacons: + service: + - services: + nginx: + onchangeonly: True + delay: 30 + uncleanshutdown: /run/nginx.pid + ``` + +- ``sh`` beacon + + Old behavior: + ``` + beacons: + sh: {} + ``` + + New behavior: + ``` + beacons: + sh: [] + ``` + +- ``status`` beacon + + Old behavior: + ``` + beacons: + status: {} + ``` + + New behavior: + ``` + beacons: + status: [] + ``` + +- ``telegram_bot_msg`` beacon + + Old behavior: + ``` + beacons: + telegram_bot_msg: + token: "" + accept_from: + - "" + interval: 10 + ``` + + New behavior: + ``` + beacons: + telegram_bot_msg: + - token: "" + - accept_from: + - "" + - interval: 10 + ``` + +- ``twilio_txt_msg`` beacon + + Old behavior: + ``` + beacons: + twilio_txt_msg: + account_sid: "" + auth_token: "" + twilio_number: "+15555555555" + interval: 10 + ``` + + New behavior: + ``` + beacons: + twilio_txt_msg: + - account_sid: "" + - auth_token: "" + - twilio_number: "+15555555555" + - interval: 10 + ``` + +- ``wtmp`` beacon + + Old behavior: + ``` + beacons: + wtmp: {} + ``` + + New behavior: + ``` + beacons: + wtmp: [] + ``` + +Deprecations +------------ + +Configuration Option Deprecations +================================= + - The ``requests_lib`` configuration option has been removed. Please use ``backend`` instead. +Profitbricks Cloud Updated Dependency +===================================== + +The minimum version of the ``profitbrick`` python package for the ``profitbricks`` +cloud driver has changed from 3.0.0 to 3.1.0. + +Azure Cloud Updated Dependency +------------------------------ + +The azure sdk used for the ``azurearm`` cloud driver now depends on ``azure-cli>=2.0.12`` + Module Deprecations -------------------- +=================== The ``blockdev`` execution module has been removed. Its functions were merged with the ``disk`` module. Please use the ``disk`` execution module instead. +The ``lxc`` execution module had the following changes: + +- The ``dnsservers`` option to the ``cloud_init_interface`` function no longer + defaults to ``4.4.4.4`` and ``8.8.8.8``. +- The ``dns_via_dhcp`` option to the ``cloud_init_interface`` function defaults + to ``True`` now instead of ``False``. + The ``win_psget`` module had the following changes: - The ``psversion`` function was removed. Please use ``cmd.shell_info`` instead. @@ -52,8 +689,16 @@ The ``win_service`` module had the following changes: - The ``type`` option was removed from the ``create`` function. Please use ``service_type`` instead. +Runner Deprecations +=================== + +The ``manage`` runner had the following changes: + +- The ``root_user`` kwarg was removed from the ``bootstrap`` function. Please + use ``salt-ssh`` roster entries for the host instead. + State Deprecations ------------------- +================== The ``archive`` state had the following changes: @@ -76,9 +721,47 @@ The ``file`` state had the following changes: - The ``show_diff`` option was removed. Please use ``show_changes`` instead. Grain Deprecations ------------------- +================== For ``smartos`` some grains have been deprecated. These grains will be removed in Neon. - The ``hypervisor_uuid`` has been replaced with ``mdata:sdc:server_uuid`` grain. - The ``datacenter`` has been replaced with ``mdata:sdc:datacenter_name`` grain. + +Minion Blackout +--------------- + +During a blackout, minions will not execute any remote execution commands, +except for :mod:`saltutil.refresh_pillar `. +Previously, support was added so that blackouts are enabled using a special +pillar key, ``minion_blackout`` set to ``True`` and an optional pillar key +``minion_blackout_whitelist`` to specify additional functions that are permitted +during blackout. This release adds support for using this feature in the grains +as well, by using special grains keys ``minion_blackout`` and +``minion_blackout_whitelist``. + +Pillar Deprecations +------------------- + +The legacy configuration for ``git_pillar`` has been removed. Please use the new +configuration for ``git_pillar``, which is documented in the external pillar module +for :mod:`git_pillar `. + +Utils Deprecations +================== + +The ``salt.utils.cloud.py`` file had the following change: + +- The ``fire_event`` function now requires a ``sock_dir`` argument. It was previously + optional. + +Other Miscellaneous Deprecations +================================ + +The ``version.py`` file had the following changes: + +- The ``rc_info`` function was removed. Please use ``pre_info`` instead. + +Warnings for moving away from the ``env`` option were removed. ``saltenv`` should be +used instead. The removal of these warnings does not have a behavior change. Only +the warning text was removed. diff --git a/doc/topics/releases/releasecandidate.rst b/doc/topics/releases/releasecandidate.rst index 0b042b86daa..60918d6ac3d 100644 --- a/doc/topics/releases/releasecandidate.rst +++ b/doc/topics/releases/releasecandidate.rst @@ -8,7 +8,7 @@ Installing/Testing a Salt Release Candidate It's time for a new feature release of Salt! Follow the instructions below to install the latest release candidate of Salt, and try :ref:`all the shiny new -features `! Be sure to report any bugs you find on `Github +features `! Be sure to report any bugs you find on `Github `_. Installing Using Packages @@ -32,32 +32,12 @@ Builds for a few platforms are available as part of the RC at https://repo.salts Available builds: -- Amazon Linux -- Debian 8 -- macOS -- RHEL 7 -- SmartOS (see below) -- Ubuntu 16.04 +- Ubuntu16 +- Redhat7 - Windows .. FreeBSD -SmartOS -------- -Release candidate builds for SmartOS are available at http://pkg.blackdot.be/extras/salt-2016.11rc/. - -On a base64 2015Q4-x86_64 based native zone the package can be installed by the following: - -.. code-block:: bash - - pfexec pkg_add -U https://pkg.blackdot.be/extras/salt-2016.11rc/salt-2016.11.0rc2_2015Q4_x86_64.tgz - -When using the 2016Q2-tools release on the global zone by the following: - -.. code-block:: bash - - pfexec pkg_add -U https://pkg.blackdot.be/extras/salt-2016.11rc/salt-2016.11.0rc2_2016Q2_TOOLS.tgz - Installing Using Bootstrap ========================== @@ -67,14 +47,14 @@ You can install a release candidate of Salt using `Salt Bootstrap .. code-block:: bash curl -o install_salt.sh -L https://bootstrap.saltstack.com - sudo sh install_salt.sh -P git v2016.11.0rc2 + sudo sh install_salt.sh -P git v2017.7.0rc1 If you want to also install a master using Salt Bootstrap, use the ``-M`` flag: .. code-block:: bash curl -o install_salt.sh -L https://bootstrap.saltstack.com - sudo sh install_salt.sh -P -M git v2016.11.0rc2 + sudo sh install_salt.sh -P -M git v2017.7.0rc1 If you want to install only a master and not a minion using Salt Bootstrap, use the ``-M`` and ``-N`` flags: @@ -82,13 +62,13 @@ the ``-M`` and ``-N`` flags: .. code-block:: bash curl -o install_salt.sh -L https://bootstrap.saltstack.com - sudo sh install_salt.sh -P -M -N git v2016.11.0rc2 + sudo sh install_salt.sh -P -M -N git v2017.7.0rc1 Installing Using PyPI ===================== Installing from the `source archive -`_ on +`_ on `PyPI `_ is fairly straightforward. .. note:: @@ -126,4 +106,4 @@ Then install salt using the following command: .. code-block:: bash - sudo pip install salt==2016.11.0rc2 + sudo pip install salt==2017.7.0rc1 diff --git a/doc/topics/spm/dev.rst b/doc/topics/spm/dev.rst index 11ad83364c1..ece953b4f06 100644 --- a/doc/topics/spm/dev.rst +++ b/doc/topics/spm/dev.rst @@ -256,7 +256,7 @@ This function will not generally be more complex than: .. code-block:: python def hash_file(path, hashobj, conn=None): - with salt.utils.fopen(path, 'r') as f: + with salt.utils.files.fopen(path, 'r') as f: hashobj.update(f.read()) return hashobj.hexdigest() diff --git a/doc/topics/spm/spm_formula.rst b/doc/topics/spm/spm_formula.rst index 445ee204f75..47ed5e0c60b 100644 --- a/doc/topics/spm/spm_formula.rst +++ b/doc/topics/spm/spm_formula.rst @@ -11,7 +11,7 @@ describes the package. An example of this file is: name: apache os: RedHat, Debian, Ubuntu, SUSE, FreeBSD - os_family: RedHat, Debian, SUSE, FreeBSD + os_family: RedHat, Debian, Suse, FreeBSD version: 201506 release: 2 summary: Formula for installing Apache diff --git a/doc/topics/ssh/index.rst b/doc/topics/ssh/index.rst index 8920e9e648a..95a2bdecdbf 100644 --- a/doc/topics/ssh/index.rst +++ b/doc/topics/ssh/index.rst @@ -64,7 +64,8 @@ Deploy ssh key for salt-ssh =========================== By default, salt-ssh will generate key pairs for ssh, the default path will be -/etc/salt/pki/master/ssh/salt-ssh.rsa +``/etc/salt/pki/master/ssh/salt-ssh.rsa``. The key generation happens when you run +``salt-ssh`` for the first time. You can use ssh-copy-id, (the OpenSSH key deployment tool) to deploy keys to your servers. diff --git a/doc/topics/ssh/roster.rst b/doc/topics/ssh/roster.rst index 489d6d9ff64..e969479e4d2 100644 --- a/doc/topics/ssh/roster.rst +++ b/doc/topics/ssh/roster.rst @@ -61,6 +61,8 @@ The information which can be stored in a roster ``target`` is the following: cmd_umask: # umask to enforce for the salt-call command. Should be in # octal (so for 0o077 in YAML you would do 0077, or 63) +.. _roster_defaults: + Target Defaults --------------- @@ -71,10 +73,10 @@ not need to be passed with commandline arguments. .. code-block:: yaml roster_defaults: - user: daniel - sudo: True - priv: /root/.ssh/id_rsa - tty: True + user: daniel + sudo: True + priv: /root/.ssh/id_rsa + tty: True thin_dir -------- diff --git a/doc/topics/troubleshooting/yaml_idiosyncrasies.rst b/doc/topics/troubleshooting/yaml_idiosyncrasies.rst index e1c63834596..f5a9fed916a 100644 --- a/doc/topics/troubleshooting/yaml_idiosyncrasies.rst +++ b/doc/topics/troubleshooting/yaml_idiosyncrasies.rst @@ -28,6 +28,7 @@ hit `Enter`. Also, you can convert tabs to 2 spaces by these commands in Vim: Indentation =========== + The suggested syntax for YAML files is to use 2 spaces for indentation, but YAML will follow whatever indentation system that the individual file uses. Indentation of two spaces works very well for SLS files given the @@ -112,8 +113,24 @@ PyYAML will load these values as boolean ``True`` or ``False``. Un-capitalized versions will also be loaded as booleans (``true``, ``false``, ``yes``, ``no``, ``on``, and ``off``). This can be especially problematic when constructing Pillar data. Make sure that your Pillars which need to use the string versions -of these values are enclosed in quotes. Pillars will be parsed twice by salt, -so you'll need to wrap your values in multiple quotes, for example '"false"'. +of these values are enclosed in quotes. Pillars will be parsed twice by salt, +so you'll need to wrap your values in multiple quotes, including double quotation +marks (``" "``) and single quotation marks (``' '``). Note that spaces are included +in the quotation type examples for clarity. + +Multiple quoting examples looks like this: + +.. code-block:: yaml + + - '"false"' + - "'True'" + - "'YES'" + - '"No"' + +.. note:: + + When using multiple quotes in this manner, they must be different. Using ``"" ""`` + or ``'' ''`` won't work in this case (spaces are included in examples for clarity). The '%' Sign ============ @@ -248,8 +265,10 @@ Alternatively, they can be defined the "old way", or with multiple - require: - user: fred -YAML support only plain ASCII -============================= +.. _yaml_plain_ascii: + +YAML supports only plain ASCII +============================== According to YAML specification, only ASCII characters can be used. diff --git a/doc/topics/tutorials/firewall.rst b/doc/topics/tutorials/firewall.rst index e04f52e2e24..4ed954ee270 100644 --- a/doc/topics/tutorials/firewall.rst +++ b/doc/topics/tutorials/firewall.rst @@ -91,8 +91,6 @@ firewall. yast2 firewall -.. _linux-iptables: - Windows ======= @@ -137,6 +135,8 @@ following command from the command line or a run prompt: netsh advfirewall firewall add rule name="Salt" dir=in action=allow protocol=TCP localport=4505-4506 +.. _linux-iptables: + iptables ======== diff --git a/doc/topics/tutorials/gitfs.rst b/doc/topics/tutorials/gitfs.rst index fb23173dfb1..cc0b1df9f8c 100644 --- a/doc/topics/tutorials/gitfs.rst +++ b/doc/topics/tutorials/gitfs.rst @@ -166,13 +166,15 @@ Ubuntu 14.04 LTS and Debian Wheezy (7.x) also have a compatible version packaged # apt-get install python-git -If your master is running an older version (such as Ubuntu 12.04 LTS or Debian -Squeeze), then you will need to install GitPython using either pip_ or -easy_install (it is recommended to use pip). Version 0.3.2.RC1 is now marked as -the stable release in PyPI, so it should be a simple matter of running ``pip -install GitPython`` (or ``easy_install GitPython``) as root. +GitPython_ requires the ``git`` CLI utility to work. If installed from a system +package, then git should already be installed, but if installed via pip_ then +it may still be necessary to install git separately. For MacOS users, +GitPython_ comes bundled in with the Salt installer, but git must still be +installed for it to work properly. Git can be installed in several ways, +including by installing XCode_. -.. _`pip`: http://www.pip-installer.org/ +.. _pip: http://www.pip-installer.org/ +.. _XCode: https://developer.apple.com/xcode/ .. warning:: @@ -330,6 +332,8 @@ configured gitfs remotes): * :conf_master:`gitfs_privkey` (**pygit2 only**, new in 2014.7.0) * :conf_master:`gitfs_passphrase` (**pygit2 only**, new in 2014.7.0) * :conf_master:`gitfs_refspecs` (new in 2017.7.0) +* :conf_master:`gitfs_disable_saltenv_mapping` (new in Oxygen) +* :conf_master:`gitfs_ref_types` (new in Oxygen) .. note:: pygit2 only supports disabling SSL verification in versions 0.23.2 and @@ -355,11 +359,17 @@ tremendous amount of customization. Here's some example usage: - root: other/salt - mountpoint: salt://other/bar - base: salt-base + - ref_types: + - branch - http://foo.com/baz.git: - root: salt/states - user: joe - password: mysupersecretpassword - insecure_auth: True + - disable_saltenv_mapping: True + - saltenv: + - foo: + - ref: foo - http://foo.com/quux.git: - all_saltenvs: master @@ -391,18 +401,24 @@ In the example configuration above, the following is true: will only serve files from the ``salt/states`` directory (and its subdirectories). -3. The first and fourth remotes will have files located under the root of the +3. The third remote will only serve files from branches, and not from tags or + SHAs. + +4. The fourth remote will only have two saltenvs available: ``base`` (pointed + at ``develop``), and ``foo`` (pointed at ``foo``). + +5. The first and fourth remotes will have files located under the root of the Salt fileserver namespace (``salt://``). The files from the second remote will be located under ``salt://bar``, while the files from the third remote will be located under ``salt://other/bar``. -4. The second and third remotes reference the same repository and unique names +6. The second and third remotes reference the same repository and unique names need to be declared for duplicate gitfs remotes. -5. The fourth remote overrides the default behavior of :ref:`not authenticating +7. The fourth remote overrides the default behavior of :ref:`not authenticating to insecure (non-HTTPS) remotes `. -6. Because ``all_saltenvs`` is configured for the fifth remote, files from the +8. Because ``all_saltenvs`` is configured for the fifth remote, files from the branch/tag ``master`` will appear in every fileserver environment. .. note:: @@ -1094,15 +1110,8 @@ Using Git as an External Pillar Source The git external pillar (a.k.a. git_pillar) has been rewritten for the 2015.8.0 release. This rewrite brings with it pygit2_ support (allowing for access to authenticated repositories), as well as more granular support for per-remote -configuration. - -To make use of the new features, changes to the git ext_pillar configuration -must be made. The new configuration schema is detailed :ref:`here -`. - -For Salt releases before 2015.8.0, click :ref:`here ` -for documentation. - +configuration. This configuration schema is detailed :ref:`here +`. .. _faq-gitfs-bug: diff --git a/doc/topics/tutorials/http.rst b/doc/topics/tutorials/http.rst index 1eaee620719..e6b20c62a24 100644 --- a/doc/topics/tutorials/http.rst +++ b/doc/topics/tutorials/http.rst @@ -110,7 +110,7 @@ To pass through a file that contains jinja + yaml templating (the default): method='POST', data_file='/srv/salt/somefile.jinja', data_render=True, - template_data={'key1': 'value1', 'key2': 'value2'} + template_dict={'key1': 'value1', 'key2': 'value2'} ) To pass through a file that contains mako templating: @@ -123,7 +123,7 @@ To pass through a file that contains mako templating: data_file='/srv/salt/somefile.mako', data_render=True, data_renderer='mako', - template_data={'key1': 'value1', 'key2': 'value2'} + template_dict={'key1': 'value1', 'key2': 'value2'} ) Because this function uses Salt's own rendering system, any Salt renderer can @@ -140,7 +140,7 @@ However, this can be changed to ``master`` if necessary. method='POST', data_file='/srv/salt/somefile.jinja', data_render=True, - template_data={'key1': 'value1', 'key2': 'value2'}, + template_dict={'key1': 'value1', 'key2': 'value2'}, opts=__opts__ ) @@ -149,7 +149,7 @@ However, this can be changed to ``master`` if necessary. method='POST', data_file='/srv/salt/somefile.jinja', data_render=True, - template_data={'key1': 'value1', 'key2': 'value2'}, + template_dict={'key1': 'value1', 'key2': 'value2'}, node='master' ) @@ -170,11 +170,11 @@ a Python dict. header_file='/srv/salt/headers.jinja', header_render=True, header_renderer='jinja', - template_data={'key1': 'value1', 'key2': 'value2'} + template_dict={'key1': 'value1', 'key2': 'value2'} ) Because much of the data that would be templated between headers and data may be -the same, the ``template_data`` is the same for both. Correcting possible +the same, the ``template_dict`` is the same for both. Correcting possible variable name collisions is up to the user. Authentication diff --git a/doc/topics/tutorials/index.rst b/doc/topics/tutorials/index.rst index cba3acd739d..df668bc395d 100644 --- a/doc/topics/tutorials/index.rst +++ b/doc/topics/tutorials/index.rst @@ -28,9 +28,8 @@ Tutorials Index * :ref:`States tutorial, part 3 - Templating, Includes, Extends ` * :ref:`States tutorial, part 4 ` * :ref:`How to Convert Jinja Logic to an Execution Module ` -* :ref:`Using Salt with Stormpath ` * :ref:`Syslog-ng usage ` * :ref:`The macOS (Maverick) Developer Step By Step Guide To Salt Installation ` * :ref:`SaltStack Walk-through ` * :ref:`Writing Salt Tests ` -* :ref:`Multi-cloud orchestration with Apache Libcloud ` \ No newline at end of file +* :ref:`Multi-cloud orchestration with Apache Libcloud ` diff --git a/doc/topics/tutorials/libcloud.rst b/doc/topics/tutorials/libcloud.rst index 793c37aaa8a..a66c2e4e767 100644 --- a/doc/topics/tutorials/libcloud.rst +++ b/doc/topics/tutorials/libcloud.rst @@ -13,7 +13,7 @@ Using Apache Libcloud for declarative and procedural multi-cloud orchestration Apache Libcloud is a Python library which hides differences between different cloud provider APIs and allows you to manage different cloud resources through a unified and easy to use API. Apache Libcloud supports over -60 cloud platforms, including Amazon, Microsoft Azure, Digital Ocean, Google Cloud Platform and OpenStack. +60 cloud platforms, including Amazon, Microsoft Azure, DigitalOcean, Google Cloud Platform and OpenStack. Execution and state modules are available for Compute, DNS, Storage and Load Balancer drivers from Apache Libcloud in SaltStack. diff --git a/doc/topics/tutorials/pillar.rst b/doc/topics/tutorials/pillar.rst index e0c97f26bd9..3ec2c1ddea7 100644 --- a/doc/topics/tutorials/pillar.rst +++ b/doc/topics/tutorials/pillar.rst @@ -75,7 +75,7 @@ The default location for the pillar is in /srv/pillar. .. note:: - The pillar location can be configured via the `pillar_roots` option inside + The pillar location can be configured via the ``pillar_roots`` option inside the master configuration file. It must not be in a subdirectory of the state tree or file_roots. If the pillar is under file_roots, any pillar targeting can be bypassed by minions. @@ -242,7 +242,7 @@ set in the minion's pillar, then the default of ``httpd`` will be used. .. note:: Under the hood, pillar is just a Python dict, so Python dict methods such - as `get` and `items` can be used. + as ``get`` and ``items`` can be used. Pillar Makes Simple States Grow Easily ====================================== @@ -303,6 +303,18 @@ Where the vimrc source location can now be changed via pillar: Ensuring that the right vimrc is sent out to the correct minions. +The pillar top file must include a reference to the new sls pillar file: + +``/srv/pillar/top.sls``: + +.. code-block:: yaml + + base: + '*': + - pkg + - edit.vim + + Setting Pillar Data on the Command Line ======================================= diff --git a/doc/topics/tutorials/stormpath.rst b/doc/topics/tutorials/stormpath.rst deleted file mode 100644 index 34003d5f996..00000000000 --- a/doc/topics/tutorials/stormpath.rst +++ /dev/null @@ -1,198 +0,0 @@ -.. _tutorial-stormpath: - -========================= -Using Salt with Stormpath -========================= - -`Stormpath `_ is a user management and authentication -service. This tutorial covers using SaltStack to manage and take advantage of -Stormpath's features. - -External Authentication ------------------------ -Stormpath can be used for Salt's external authentication system. In order to do -this, the master should be configured with an ``apiid``, ``apikey``, and the ID -of the ``application`` that is associated with the users to be authenticated: - -.. code-block:: yaml - - stormpath: - apiid: 367DFSF4FRJ8767FSF4G34FGH - apikey: FEFREF43t3FEFRe/f323fwer4FWF3445gferWRWEer1 - application: 786786FREFrefreg435fr1 - -.. note:: - These values can be found in the `Stormpath dashboard - `_`. - -Users that are to be authenticated should be set up under the ``stormpath`` -dict under ``external_auth``: - -.. code-block:: yaml - - external_auth: - stormpath: - larry: - - .* - - '@runner' - - '@wheel' - -Keep in mind that while Stormpath defaults the username associated with the -account to the email address, it is better to use a username without an ``@`` -sign in it. - - -Configuring Stormpath Modules ------------------------------ -Stormpath accounts can be managed via either an execution or state module. In -order to use either, a minion must be configured with an API ID and key. - -.. code-block:: yaml - - stormpath: - apiid: 367DFSF4FRJ8767FSF4G34FGH - apikey: FEFREF43t3FEFRe/f323fwer4FWF3445gferWRWEer1 - directory: efreg435fr1786786FREFr - application: 786786FREFrefreg435fr1 - -Some functions in the ``stormpath`` modules can make use of other options. The -following options are also available. - -directory -````````` -The ID of the directory that is to be used with this minion. Many functions -require an ID to be specified to do their work. However, if the ID of a -``directory`` is specified, then Salt can often look up the resource in -question. - -application -``````````` -The ID of the application that is to be used with this minion. Many functions -require an ID to be specified to do their work. However, if the ID of a -``application`` is specified, then Salt can often look up the resource in -question. - - -Managing Stormpath Accounts ---------------------------- -With the ``stormpath`` configuration in place, Salt can be used to configure -accounts (which may be thought of as users) on the Stormpath service. The -following functions are available. - -stormpath.create_account -```````````````````````` -Create an account on the Stormpath service. This requires a ``directory_id`` as -the first argument; it will not be retrieved from the minion configuration. An -``email`` address, ``password``, first name (``givenName``) and last name -(``surname``) are also required. For the full list of other parameters that may -be specified, see: - -http://docs.stormpath.com/rest/product-guide/#account-resource - -When executed with no errors, this function will return the information about -the account, from Stormpath. - -.. code-block:: bash - - salt myminion stormpath.create_account shemp@example.com letmein Shemp Howard - - -stormpath.list_accounts -``````````````````````` -Show all accounts on the Stormpath service. This will return all accounts, -regardless of directory, application, or group. - -.. code-block:: bash - - salt myminion stormpath.list_accounts - ''' - -stormpath.show_account -`````````````````````` -Show the details for a specific Stormpath account. An ``account_id`` is normally -required. However, if am ``email`` is provided instead, along with either a -``directory_id``, ``application_id``, or ``group_id``, then Salt will search the -specified resource to try and locate the ``account_id``. - -.. code-block:: bash - - salt myminion stormpath.show_account - salt myminion stormpath.show_account email= directory_id= - - -stormpath.update_account -```````````````````````` -Update one or more items for this account. Specifying an empty value will clear -it for that account. This function may be used in one of two ways. In order to -update only one key/value pair, specify them in order: - -.. code-block:: bash - - salt myminion stormpath.update_account givenName shemp - salt myminion stormpath.update_account middleName '' - -In order to specify multiple items, they need to be passed in as a dict. From -the command line, it is best to do this as a JSON string: - -.. code-block:: bash - - salt myminion stormpath.update_account items='{"givenName": "Shemp"} - salt myminion stormpath.update_account items='{"middlename": ""} - -When executed with no errors, this function will return the information about -the account, from Stormpath. - - -stormpath.delete_account -```````````````````````` -Delete an account from Stormpath. - -.. code-block:: bash - - salt myminion stormpath.delete_account - - -stormpath.list_directories -`````````````````````````` -Show all directories associated with this tenant. - -.. code-block:: bash - - salt myminion stormpath.list_directories - - -Using Stormpath States ----------------------- -Stormpath resources may be managed using the state system. The following states -are available. - -stormpath_account.present -````````````````````````` -Ensure that an account exists on the Stormpath service. All options that are -available with the ``stormpath.create_account`` function are available here. -If an account needs to be created, then this function will require the same -fields that ``stormpath.create_account`` requires, including the ``password``. -However, if a password changes for an existing account, it will NOT be updated -by this state. - -.. code-block:: yaml - - curly@example.com: - stormpath_account.present: - - directory_id: efreg435fr1786786FREFr - - password: badpass - - firstName: Curly - - surname: Howard - - nickname: curly - -It is advisable to always set a ``nickname`` that is not also an email address, -so that it can be used by Salt's external authentication module. - -stormpath_account.absent -```````````````````````` -Ensure that an account does not exist on Stormpath. As with -``stormpath_account.present``, the ``name`` supplied to this state is the -``email`` address associated with this account. Salt will use this, with or -without the ``directory`` ID that is configured for the minion. However, lookups -will be much faster with a directory ID specified. - diff --git a/doc/topics/utils/index.rst b/doc/topics/utils/index.rst index 48728174f53..34d48144ebd 100644 --- a/doc/topics/utils/index.rst +++ b/doc/topics/utils/index.rst @@ -54,7 +54,7 @@ types like so: salt '*' mymodule.observe_the_awesomeness ''' - print __utils__['foo.bar']() + return __utils__['foo.bar']() Utility modules, like any other kind of Salt extension, support using a :ref:`__virtual__ function ` to conditionally load them, @@ -81,11 +81,60 @@ the ``foo`` utility module with a ``__virtual__`` function. def bar(): return 'baz' +.. versionadded:: Oxygen + Instantiating objects from classes declared in util modules works with + Master side modules, such as Runners, Outputters, etc. + +Also you could even write your utility modules in object oriented fashion: + +.. code-block:: python + + # -*- coding: utf-8 -*- + ''' + My OOP-style utils module + ------------------------- + + This module contains common functions for use in my other custom types. + ''' + + class Foo(object): + + def __init__(self): + pass + + def bar(self): + return 'baz' + +And import them into other custom modules: + +.. code-block:: python + + # -*- coding: utf-8 -*- + ''' + My awesome execution module + --------------------------- + ''' + + import mymodule + + def observe_the_awesomeness(): + ''' + Prints information from my utility module + + CLI Example: + + .. code-block:: bash + + salt '*' mymodule.observe_the_awesomeness + ''' + foo = mymodule.Foo() + return foo.bar() + These are, of course, contrived examples, but they should serve to show some of the possibilities opened up by writing utility modules. Keep in mind though -that States still have access to all of the execution modules, so it is not +that states still have access to all of the execution modules, so it is not necessary to write a utility module to make a function available to both a -state and an execution module. One good use case for utililty modules is one +state and an execution module. One good use case for utility modules is one where it is necessary to invoke the same function from a custom :ref:`outputter `/returner, as well as an execution module. diff --git a/doc/topics/windows/windows-package-manager.rst b/doc/topics/windows/windows-package-manager.rst index 1b2989dd0d3..20ed60baf68 100644 --- a/doc/topics/windows/windows-package-manager.rst +++ b/doc/topics/windows/windows-package-manager.rst @@ -141,7 +141,7 @@ packages: - 2015.8.0 and later minions: https://github.com/saltstack/salt-winrepo-ng - Earlier releases: https://github.com/saltstack/salt-winrepo -By default, these repositories are mirrored to ``/srv/salt/win/repo_ng`` +By default, these repositories are mirrored to ``/srv/salt/win/repo-ng`` and ``/srv/salt/win/repo``. This location can be changed in the master config file by setting the @@ -481,11 +481,17 @@ Alternatively the ``uninstaller`` can also simply repeat the URL of the msi file :param bool allusers: This parameter is specific to `.msi` installations. It tells `msiexec` to install the software for all users. The default is True. -:param bool cache_dir: If true, the entire directory where the installer resides - will be recursively cached. This is useful for installers that depend on - other files in the same directory for installation. +:param bool cache_dir: If true when installer URL begins with salt://, the + entire directory where the installer resides will be recursively cached. + This is useful for installers that depend on other files in the same + directory for installation. -.. note:: Only applies to salt: installer URLs. +:param str cache_file: + When installer URL begins with salt://, this indicates single file to copy + down for use with the installer. Copied to the same location as the + installer. Use this over ``cache_dir`` if there are many files in the + directory and you only need a specific file and don't want to cache + additional files that may reside in the installer directory. Here's an example for a software package that has dependent files: diff --git a/pkg/osx/build.sh b/pkg/osx/build.sh index e304248328a..7850d48cd83 100755 --- a/pkg/osx/build.sh +++ b/pkg/osx/build.sh @@ -19,14 +19,16 @@ # $1 : : the version of salt to build # (a git tag, not a branch) # (defaults to git-repo state) -# $2 : : the staging area for the package +# $2 : : The version of Python to use in the +# build. Default is 2 +# $3 : : the staging area for the package # defaults to /tmp/salt_pkg # # Example: -# The following will build Salt v2015.8.3 and stage all files -# in /tmp/custom_pkg: +# The following will build Salt v2015.8.3 with Python 2 and +# stage all files in /tmp/custom_pkg: # -# ./build.sh v2015.8.3 /tmp/custom_pkg +# ./build.sh v2015.8.3 2 /tmp/custom_pkg # ############################################################################ echo -n -e "\033]0;Build: Variables\007" @@ -41,9 +43,15 @@ else fi if [ "$2" == "" ]; then + PYVER=2 +else + PYVER=$2 +fi + +if [ "$3" == "" ]; then PKGDIR=/tmp/salt_pkg else - PKGDIR=$2 + PKGDIR=$3 fi ############################################################################ @@ -51,6 +59,12 @@ fi ############################################################################ SRCDIR=`git rev-parse --show-toplevel` PKGRESOURCES=$SRCDIR/pkg/osx +if [ "$PYVER" == "2" ]; then + PYTHON=/opt/salt/bin/python +else + PYTHON=/opt/salt/bin/python3 +fi +CPUARCH=`uname -m` ############################################################################ # Make sure this is the Salt Repository @@ -66,16 +80,23 @@ fi # Create the Build Environment ############################################################################ echo -n -e "\033]0;Build: Build Environment\007" -sudo $PKGRESOURCES/build_env.sh +sudo $PKGRESOURCES/build_env.sh $PYVER ############################################################################ # Install Salt ############################################################################ echo -n -e "\033]0;Build: Install Salt\007" -sudo /opt/salt/bin/python $SRCDIR/setup.py install +sudo rm -rf $SRCDIR/build +sudo rm -rf $SRCDIR/dist +sudo $PYTHON $SRCDIR/setup.py build -e "$PYTHON -E -s" install ############################################################################ # Build Package ############################################################################ echo -n -e "\033]0;Build: Package Salt\007" -sudo $PKGRESOURCES/build_pkg.sh $VERSION $PKGDIR +sudo $PKGRESOURCES/build_pkg.sh $VERSION $PYVER $PKGDIR + +############################################################################ +# Sign Package +############################################################################ +sudo $PKGRESOURCES/build_sig.sh salt-$VERSION-py$PYVER-$CPUARCH.pkg salt-$VERSION-py$PYVER-$CPUARCH-signed.pkg diff --git a/pkg/osx/build_env.sh b/pkg/osx/build_env.sh index f886bd59ad7..5d7f1bac5a3 100755 --- a/pkg/osx/build_env.sh +++ b/pkg/osx/build_env.sh @@ -6,18 +6,21 @@ # Authors: CR Oldham, Shane Lee # Date: December 2015 # -# Description: This script sets up a build environment for salt on macOS. +# Description: This script sets up a build environment for Salt on macOS. # # Requirements: # - XCode Command Line Tools (xcode-select --install) # # Usage: -# This script is not passed any parameters +# This script can be passed 1 parameter +# $1 : : the version of Python to use for the +# build environment. Default is 2 # # Example: -# The following will set up a build environment for salt on macOS +# The following will set up a Python 3 build environment for Salt +# on macOS # -# ./dev_env.sh +# ./dev_env.sh 3 # ############################################################################ @@ -31,6 +34,15 @@ quit_on_error() { exit -1 } +############################################################################ +# Check passed parameters, set defaults +############################################################################ +if [ "$1" == "" ]; then + PYVER=2 +else + PYVER=$1 +fi + ############################################################################ # Parameters Required for the script to function properly ############################################################################ @@ -45,6 +57,15 @@ SHADIR=$SCRIPTDIR/shasums PKG_CONFIG_PATH=/opt/salt/lib/pkgconfig CFLAGS="-I/opt/salt/include" LDFLAGS="-L/opt/salt/lib" +if [ "$PYVER" == "2" ]; then + PYDIR=/opt/salt/lib/python2.7 + PYTHON=/opt/salt/bin/python + PIP=/opt/salt/bin/pip +else + PYDIR=/opt/salt/lib/python3.5 + PYTHON=/opt/salt/bin/python3 + PIP=/opt/salt/bin/pip3 +fi ############################################################################ # Determine Which XCode is being used (XCode or XCode Command Line Tools) @@ -121,8 +142,8 @@ BUILDDIR=$SCRIPTDIR/build ############################################################################ echo -n -e "\033]0;Build_Env: pkg-config\007" -PKGURL="http://pkgconfig.freedesktop.org/releases/pkg-config-0.29.tar.gz" -PKGDIR="pkg-config-0.29" +PKGURL="http://pkgconfig.freedesktop.org/releases/pkg-config-0.29.2.tar.gz" +PKGDIR="pkg-config-0.29.2" download $PKGURL @@ -140,8 +161,8 @@ sudo -H $MAKE install ############################################################################ echo -n -e "\033]0;Build_Env: libsodium\007" -PKGURL="https://download.libsodium.org/libsodium/releases/libsodium-1.0.12.tar.gz" -PKGDIR="libsodium-1.0.12" +PKGURL="https://download.libsodium.org/libsodium/releases/libsodium-1.0.13.tar.gz" +PKGDIR="libsodium-1.0.13" download $PKGURL @@ -159,8 +180,8 @@ sudo -H $MAKE install ############################################################################ echo -n -e "\033]0;Build_Env: zeromq\007" -PKGURL="http://download.zeromq.org/zeromq-4.1.3.tar.gz" -PKGDIR="zeromq-4.1.3" +PKGURL="http://download.zeromq.org/zeromq-4.1.4.tar.gz" +PKGDIR="zeromq-4.1.4" download $PKGURL @@ -178,13 +199,13 @@ sudo -H $MAKE install ############################################################################ echo -n -e "\033]0;Build_Env: OpenSSL\007" -PKGURL="http://openssl.org/source/openssl-1.0.2f.tar.gz" -PKGDIR="openssl-1.0.2f" +PKGURL="http://openssl.org/source/openssl-1.0.2l.tar.gz" +PKGDIR="openssl-1.0.2l" download $PKGURL echo "################################################################################" -echo "Building OpenSSL 1.0.2f" +echo "Building OpenSSL" echo "################################################################################" cd $PKGDIR ./Configure darwin64-x86_64-cc --prefix=/opt/salt --openssldir=/opt/salt/openssl @@ -197,13 +218,18 @@ sudo -H $MAKE install ############################################################################ echo -n -e "\033]0;Build_Env: Python\007" -PKGURL="https://www.python.org/ftp/python/2.7.12/Python-2.7.12.tar.xz" -PKGDIR="Python-2.7.12" +if [ "$PYVER" == "2" ]; then + PKGURL="https://www.python.org/ftp/python/2.7.13/Python-2.7.13.tar.xz" + PKGDIR="Python-2.7.13" +else + PKGURL="https://www.python.org/ftp/python/3.5.3/Python-3.5.3.tar.xz" + PKGDIR="Python-3.5.3" +fi download $PKGURL echo "################################################################################" -echo "Building Python 2.7.12" +echo "Building Python" echo "################################################################################" echo "Note there are some test failures" cd $PKGDIR @@ -215,7 +241,7 @@ sudo -H $MAKE install ############################################################################ # upgrade pip ############################################################################ -sudo -H /opt/salt/bin/pip install --upgrade pip +sudo -H $PIP install --upgrade pip ############################################################################ # Download and install salt python dependencies @@ -227,23 +253,23 @@ cd $BUILDDIR echo "################################################################################" echo "Installing Salt Dependencies with pip (normal)" echo "################################################################################" -sudo -H /opt/salt/bin/pip install \ - -r $SRCDIR/pkg/osx/req.txt \ - --no-cache-dir +sudo -H $PIP install \ + -r $SRCDIR/pkg/osx/req.txt \ + --no-cache-dir echo "################################################################################" echo "Installing Salt Dependencies with pip (build_ext)" echo "################################################################################" -sudo -H /opt/salt/bin/pip install \ - -r $SRCDIR/pkg/osx/req_ext.txt \ - --global-option=build_ext \ - --global-option="-I/opt/salt/include" \ - --no-cache-dir +sudo -H $PIP install \ + -r $SRCDIR/pkg/osx/req_ext.txt \ + --global-option=build_ext \ + --global-option="-I/opt/salt/include" \ + --no-cache-dir echo "--------------------------------------------------------------------------------" echo "Create Symlink to certifi for openssl" echo "--------------------------------------------------------------------------------" -sudo ln -s /opt/salt/lib/python2.7/site-packages/certifi/cacert.pem /opt/salt/openssl/cert.pem +sudo ln -s $PYDIR/site-packages/certifi/cacert.pem /opt/salt/openssl/cert.pem echo -n -e "\033]0;Build_Env: Finished\007" diff --git a/pkg/osx/build_pkg.sh b/pkg/osx/build_pkg.sh index 86ad9aa450c..80f5c4f7342 100755 --- a/pkg/osx/build_pkg.sh +++ b/pkg/osx/build_pkg.sh @@ -15,13 +15,16 @@ # This script can be passed 2 parameters # $1 : : the version name to give the package (overrides # version of the git repo) (Defaults to the git repo version) -# $2 : : the staging area for the package defaults to +# $2 : : the version of python that was built (defaults +# to 2) +# $3 : : the staging area for the package defaults to # /tmp/salt_pkg # # Example: -# The following will build Salt and stage all files in /tmp/salt_pkg: +# The following will build Salt version 2017.7.0 with Python 3 and +# stage all files in /tmp/salt_pkg: # -# ./build.sh +# ./build.sh 2017.7.0 3 # ############################################################################ @@ -45,11 +48,18 @@ else VERSION=$1 fi -# Get/Set temp directory +# Get/Set Python Version if [ "$2" == "" ]; then + PYVER=2 +else + PYVER=$2 +fi + +# Get/Set temp directory +if [ "$3" == "" ]; then PKGDIR=/tmp/salt_pkg else - PKGDIR=$2 + PKGDIR=$3 fi CPUARCH=`uname -m` @@ -114,7 +124,11 @@ sudo rm -rdf $PKGDIR/opt/salt/lib/engines sudo rm -rdf $PKGDIR/opt/salt/share/aclocal sudo rm -rdf $PKGDIR/opt/salt/share/doc sudo rm -rdf $PKGDIR/opt/salt/share/man/man1/pkg-config.1 -sudo rm -rdf $PKGDIR/opt/salt/lib/python2.7/test +if [ "$PYVER" == "2" ]; then + sudo rm -rdf $PKGDIR/opt/salt/lib/python2.7/test +else + sudo rm -rdf $PKGDIR/opt/salt/lib/python3.5/test +fi echo -n -e "\033]0;Build_Pkg: Remove compiled python files\007" sudo find $PKGDIR/opt/salt -name '*.pyc' -type f -delete @@ -133,15 +147,30 @@ cp $SRCDIR/conf/master $PKGDIR/etc/salt/master.dist ############################################################################ echo -n -e "\033]0;Build_Pkg: Add Version to .xml\007" +if [ "$PYVER" == "2" ]; then + TITLE="Salt $VERSION" + DESC="Salt $VERSION with Python 2" +else + TITLE="Salt $VERSION (Python 3)" + DESC="Salt $VERSION with Python 3" +fi + cd $PKGRESOURCES cp distribution.xml.dist distribution.xml -SEDSTR="s/@VERSION@/$VERSION/" -echo $SEDSTR -sed -i '' $SEDSTR distribution.xml +SEDSTR="s/@TITLE@/$TITLE/g" +sed -E -i '' "$SEDSTR" distribution.xml -SEDSTR="s/@CPUARCH@/$CPUARCH/" -echo $SEDSTR -sed -i '' $SEDSTR distribution.xml +SEDSTR="s/@DESC@/$DESC/g" +sed -E -i '' "$SEDSTR" distribution.xml + +SEDSTR="s/@VERSION@/$VERSION/g" +sed -E -i '' "$SEDSTR" distribution.xml + +SEDSTR="s/@PYVER@/$PYVER/g" +sed -E -i '' "$SEDSTR" distribution.xml + +SEDSTR="s/@CPUARCH@/$CPUARCH/g" +sed -i '' "$SEDSTR" distribution.xml ############################################################################ # Build the Package @@ -152,10 +181,10 @@ pkgbuild --root=$PKGDIR \ --scripts=pkg-scripts \ --identifier=com.saltstack.salt \ --version=$VERSION \ - --ownership=recommended salt-src-$VERSION-$CPUARCH.pkg + --ownership=recommended salt-src-$VERSION-py$PYVER-$CPUARCH.pkg productbuild --resources=pkg-resources \ --distribution=distribution.xml \ - --package-path=salt-src-$VERSION-$CPUARCH.pkg \ - --version=$VERSION salt-$VERSION-$CPUARCH.pkg + --package-path=salt-src-$VERSION-py$PYVER-$CPUARCH.pkg \ + --version=$VERSION salt-$VERSION-py$PYVER-$CPUARCH.pkg diff --git a/pkg/osx/distribution.xml.dist b/pkg/osx/distribution.xml.dist index 083fef44f65..d31063f5f4b 100644 --- a/pkg/osx/distribution.xml.dist +++ b/pkg/osx/distribution.xml.dist @@ -1,6 +1,6 @@ - Salt @VERSION@ + @TITLE@ com.saltstack.salt @@ -25,7 +25,7 @@ salt-src-@VERSION@-@CPUARCH@.pkg + auth="root">salt-src-@VERSION@-py@PYVER@-@CPUARCH@.pkg @@ -34,8 +34,8 @@ diff --git a/pkg/osx/pkg-scripts/postinstall b/pkg/osx/pkg-scripts/postinstall index 89404e2158e..f521666a6f8 100755 --- a/pkg/osx/pkg-scripts/postinstall +++ b/pkg/osx/pkg-scripts/postinstall @@ -15,91 +15,119 @@ # This script is run as a part of the macOS Salt Installation # ############################################################################### -echo "Post install started on:" > /tmp/postinstall.txt -date >> /tmp/postinstall.txt + +############################################################################### +# Define Variables +############################################################################### +# Get Minor Version +OSX_VERSION=$(sw_vers | grep ProductVersion | cut -f 2 -d: | tr -d '[:space:]') +MINOR=$(echo ${OSX_VERSION} | cut -f 2 -d.) +# Path Variables +INSTALL_DIR="/opt/salt" +BIN_DIR="$INSTALL_DIR/bin" +CONFIG_DIR="/etc/salt" +TEMP_DIR="/tmp" +SBIN_DIR="/usr/local/sbin" + +############################################################################### +# Set up logging and error handling +############################################################################### +echo "Post install script started on:" > "$TEMP_DIR/postinstall.txt" +date "+%Y/%m/%d %H:%m:%S" >> "$TEMP_DIR/postinstall.txt" trap 'quit_on_error $LINENO $BASH_COMMAND' ERR quit_on_error() { - echo "$(basename $0) caught error on line : $1 command was: $2" >> /tmp/postinstall.txt + echo "$(basename $0) caught error on line : $1 command was: $2" >> "$TEMP_DIR/postinstall.txt" exit -1 } ############################################################################### # Check for existing minion config, copy if it doesn't exist ############################################################################### -if [ ! -f /etc/salt/minion ]; then - echo "Config copy: Started..." >> /tmp/postinstall.txt - cp /etc/salt/minion.dist /etc/salt/minion - echo "Config copy: Successful" >> /tmp/postinstall.txt +if [ ! -f "$CONFIG_DIR/minion" ]; then + echo "Config: Copy Started..." >> "$TEMP_DIR/postinstall.txt" + cp "$CONFIG_DIR/minion.dist" "$CONFIG_DIR/minion" + echo "Config: Copied Successfully" >> "$TEMP_DIR/postinstall.txt" fi ############################################################################### # Create symlink to salt-config.sh ############################################################################### -# echo "Symlink: Creating symlink for salt-config..." >> /tmp/postinstall.txt -if [ ! -d "/usr/local/sbin" ]; then - mkdir /usr/local/sbin +if [ ! -d "$SBIN_DIR" ]; then + echo "Symlink: Creating $SBIN_DIR..." >> "$TEMP_DIR/postinstall.txt" + mkdir "$SBIN_DIR" + echo "Symlink: Created Successfully" >> "$TEMP_DIR/postinstall.txt" fi -ln -sf /opt/salt/bin/salt-config.sh /usr/local/sbin/salt-config +echo "Symlink: Creating symlink for salt-config..." >> "$TEMP_DIR/postinstall.txt" +ln -sf "$BIN_DIR/salt-config.sh" "$SBIN_DIR/salt-config" +echo "Symlink: Created Successfully" >> "$TEMP_DIR/postinstall.txt" ############################################################################### # Add salt to paths.d ############################################################################### -# echo "Path: Adding salt to the path..." >> /tmp/postinstall.txt if [ ! -d "/etc/paths.d" ]; then + echo "Path: Creating paths.d directory..." >> "$TEMP_DIR/postinstall.txt" mkdir /etc/paths.d + echo "Path: Created Successfully" >> "$TEMP_DIR/postinstall.txt" fi -sh -c 'echo "/opt/salt/bin" > /etc/paths.d/salt' -sh -c 'echo "/usr/local/sbin" >> /etc/paths.d/salt' +echo "Path: Adding salt to the path..." >> "$TEMP_DIR/postinstall.txt" +sh -c "echo \"$BIN_DIR\" > /etc/paths.d/salt" +sh -c "echo \"$SBIN_DIR\" >> /etc/paths.d/salt" +echo "Path: Added Successfully" >> "$TEMP_DIR/postinstall.txt" ############################################################################### # Register Salt as a service ############################################################################### setup_services_maverick() { - echo "Using old (< 10.10) launchctl interface" >> /tmp/postinstall.txt + echo "Service: Using old (< 10.10) launchctl interface" >> "$TEMP_DIR/postinstall.txt" if /bin/launchctl list "com.saltstack.salt.minion" &> /dev/null; then - echo "Stop running service..." >> /tmp/postinstall.txt + echo "Service: Stopping salt-minion..." >> "$TEMP_DIR/postinstall.txt" launchctl unload -w /Library/LaunchDaemons/com.saltstack.salt.minion.plist + echo "Service: Stopped Successfully" >> "$TEMP_DIR/postinstall.txt" fi; + echo "Service: Starting salt-minion..." >> "$TEMP_DIR/postinstall.txt" launchctl load -w /Library/LaunchDaemons/com.saltstack.salt.minion.plist || return 1 + echo "Service: Started Successfully" >> "$TEMP_DIR/postinstall.txt" - echo "Service start: Successful" >> /tmp/postinstall.txt - - echo "Service disable: Disabling Master, Syndic, and API" >> /tmp/postinstall.txt - + echo "Service: Disabling Master, Syndic, and API services..." >> "$TEMP_DIR/postinstall.txt" launchctl unload -w /Library/LaunchDaemons/com.saltstack.salt.api.plist launchctl unload -w /Library/LaunchDaemons/com.saltstack.salt.master.plist launchctl unload -w /Library/LaunchDaemons/com.saltstack.salt.syndic.plist + echo "Service: Disabled Successfully" >> "$TEMP_DIR/postinstall.txt" return 0 } setup_services_yosemite_and_later() { - echo "Using new (>= 10.10) launchctl interface" >> /tmp/postinstall.txt + echo "Service: Using new (>= 10.10) launchctl interface" >> "$TEMP_DIR/postinstall.txt" + echo "Service: Enabling salt-minion..." >> "$TEMP_DIR/postinstall.txt" launchctl enable system/com.saltstack.salt.minion - echo "Service start: Bootstrapping service..." >> /tmp/postinstall.txt + echo "Service: Enabled Successfully" >> "$TEMP_DIR/postinstall.txt" + + echo "Service: Bootstrapping salt-minion..." >> "$TEMP_DIR/postinstall.txt" launchctl bootstrap system /Library/LaunchDaemons/com.saltstack.salt.minion.plist + echo "Service: Bootstrapped Successfully" >> "$TEMP_DIR/postinstall.txt" if /bin/launchctl list "com.saltstack.salt.minion" &> /dev/null; then - echo "Service is running" >> /tmp/postinstall.txt + echo "Service: Service Running" >> "$TEMP_DIR/postinstall.txt" else - echo "Service start: Kickstarting service..." >> /tmp/postinstall.txt + echo "Service: Kickstarting Service..." >> "$TEMP_DIR/postinstall.txt" launchctl kickstart -kp system/com.saltstack.salt.minion + echo "Service: Kickstarted Successfully" >> "$TEMP_DIR/postinstall.txt" fi - echo "Service start: Successful" >> /tmp/postinstall.txt - - echo "Service disable: Disabling Master, Syndic, and API" >> /tmp/postinstall.txt + echo "Service: Started Successfully" >> "$TEMP_DIR/postinstall.txt" + echo "Service: Disabling Master, Syndic, and API services" >> "$TEMP_DIR/postinstall.txt" launchctl disable system/com.saltstack.salt.master launchctl disable system/com.saltstack.salt.syndic launchctl disable system/com.saltstack.salt.api + echo "Service: Disabled Successfully" >> "$TEMP_DIR/postinstall.txt" + + return 0 } -OSX_VERSION=$(sw_vers | grep ProductVersion | cut -f 2 -d: | tr -d '[:space:]') -MINOR=$(echo ${OSX_VERSION} | cut -f 2 -d.) - -echo "Service start: Enabling service..." >> /tmp/postinstall.txt +echo "Service: Configuring..." >> "$TEMP_DIR/postinstall.txt" case $MINOR in 9 ) setup_services_maverick; @@ -108,7 +136,9 @@ case $MINOR in setup_services_yosemite_and_later; ;; esac +echo "Service: Configured Successfully" >> "$TEMP_DIR/postinstall.txt" -echo "Post install completed successfully" >> /tmp/postinstall.txt +echo "Post install completed successfully on:" >> "$TEMP_DIR/postinstall.txt" +date "+%Y/%m/%d %H:%m:%S" >> "$TEMP_DIR/postinstall.txt" exit 0 diff --git a/pkg/osx/pkg-scripts/preinstall b/pkg/osx/pkg-scripts/preinstall index e9a12c7c9ae..3eb4235107a 100755 --- a/pkg/osx/pkg-scripts/preinstall +++ b/pkg/osx/pkg-scripts/preinstall @@ -6,7 +6,8 @@ # Date: December 2015 # # Description: This script stops the salt minion service before attempting to -# install Salt on macOS +# install Salt on macOS. It also removes the /opt/salt/bin +# directory, symlink to salt-config, and salt from paths.d. # # Requirements: # - None @@ -15,12 +16,29 @@ # This script is run as a part of the macOS Salt Installation # ############################################################################### -echo "Preinstall started on:" > /tmp/preinstall.txt -date >> /tmp/preinstall.txt + +############################################################################### +# Define Variables +############################################################################### +# Get Minor Version +OSX_VERSION=$(sw_vers | grep ProductVersion | cut -f 2 -d: | tr -d '[:space:]') +MINOR=$(echo ${OSX_VERSION} | cut -f 2 -d.) +# Path Variables +INSTALL_DIR="/opt/salt" +BIN_DIR="$INSTALL_DIR/bin" +CONFIG_DIR="/etc/salt" +TEMP_DIR="/tmp" +SBIN_DIR="/usr/local/sbin" + +############################################################################### +# Set up logging and error handling +############################################################################### +echo "Preinstall started on:" > "$TEMP_DIR/preinstall.txt" +date "+%Y/%m/%d %H:%m:%S" >> "$TEMP_DIR/preinstall.txt" trap 'quit_on_error $LINENO $BASH_COMMAND' ERR quit_on_error() { - echo "$(basename $0) caught error on line : $1 command was: $2" >> /tmp/preinstall.txt + echo "$(basename $0) caught error on line : $1 command was: $2" >> "$TEMP_DIR/preinstall.txt" exit -1 } @@ -31,24 +49,58 @@ MINOR=$(echo ${OSX_VERSION} | cut -f 2 -d.) # Stop the service ############################################################################### stop_service_maverick() { - echo "Using old (< 10.10) launchctl interface" >> /tmp/preinstall.txt + echo "Service: Using old (< 10.10) launchctl interface" >> "$TEMP_DIR/preinstall.txt" if /bin/launchctl list "com.saltstack.salt.minion" &> /dev/null; then - echo "Stop service: Started..." >> /tmp/preinstall.txt + echo "Service: Unloading minion..." >> "$TEMP_DIR/preinstall.txt" launchctl unload -w /Library/LaunchDaemons/com.saltstack.salt.minion.plist - echo "Stop service: Successful" >> /tmp/preinstall.txt + echo "Service: Unloaded Successfully" >> "$TEMP_DIR/preinstall.txt" + fi + if /bin/launchctl list "com.saltstack.salt.master" &> /dev/null; then + echo "Service: Unloading master..." >> "$TEMP_DIR/preinstall.txt" + launchctl unload -w /Library/LaunchDaemons/com.saltstack.salt.master.plist + echo "Service: Unloaded Successfully" >> "$TEMP_DIR/preinstall.txt" + fi + if /bin/launchctl list "com.saltstack.salt.syndic" &> /dev/null; then + echo "Service: Unloading syndic..." >> "$TEMP_DIR/preinstall.txt" + launchctl unload -w /Library/LaunchDaemons/com.saltstack.salt.syndic.plist + echo "Service: Unloaded Successfully" >> "$TEMP_DIR/preinstall.txt" + fi + if /bin/launchctl list "com.saltstack.salt.api" &> /dev/null; then + echo "Service: Unloading api..." >> "$TEMP_DIR/preinstall.txt" + launchctl unload -w /Library/LaunchDaemons/com.saltstack.salt.api.plist + echo "Service: Unloaded Successfully" >> "$TEMP_DIR/preinstall.txt" fi } stop_service_yosemite_and_later() { - echo "Using new (>= 10.10) launchctl interface" >> /tmp/preinstall.txt + echo "Service: Using new (>= 10.10) launchctl interface" >> "$TEMP_DIR/preinstall.txt" if /bin/launchctl list "com.saltstack.salt.minion" &> /dev/null; then - echo "Stop service: Started..." >> /tmp/preinstall.txt + echo "Service: Stopping minion..." >> "$TEMP_DIR/preinstall.txt" launchctl disable system/com.saltstack.salt.minion launchctl bootout system /Library/LaunchDaemons/com.saltstack.salt.minion.plist - echo "Stop service: Successful" >> /tmp/preinstall.txt + echo "Service: Stopped Successfully" >> "$TEMP_DIR/preinstall.txt" + fi + if /bin/launchctl list "com.saltstack.salt.master" &> /dev/null; then + echo "Service: Stopping master..." >> "$TEMP_DIR/preinstall.txt" + launchctl disable system/com.saltstack.salt.master + launchctl bootout system /Library/LaunchDaemons/com.saltstack.salt.master.plist + echo "Service: Stopped Successfully" >> "$TEMP_DIR/preinstall.txt" + fi + if /bin/launchctl list "com.saltstack.salt.syndic" &> /dev/null; then + echo "Service: Stopping syndic..." >> "$TEMP_DIR/preinstall.txt" + launchctl disable system/com.saltstack.salt.syndic + launchctl bootout system /Library/LaunchDaemons/com.saltstack.salt.syndic.plist + echo "Service: Stopped Successfully" >> "$TEMP_DIR/preinstall.txt" + fi + if /bin/launchctl list "com.saltstack.salt.api" &> /dev/null; then + echo "Service: Stopping api..." >> "$TEMP_DIR/preinstall.txt" + launchctl disable system/com.saltstack.salt.api + launchctl bootout system /Library/LaunchDaemons/com.saltstack.salt.api.plist + echo "Service: Stopped Successfully" >> "$TEMP_DIR/preinstall.txt" fi } +echo "Service: Configuring..." >> "$TEMP_DIR/preinstall.txt" case $MINOR in 9 ) stop_service_maverick; @@ -57,6 +109,36 @@ case $MINOR in stop_service_yosemite_and_later; ;; esac -echo "Preinstall Completed Successfully" >> /tmp/preinstall.txt +echo "Service: Configured Successfully" >> "$TEMP_DIR/preinstall.txt" + +############################################################################### +# Remove the Symlink to salt-config.sh +############################################################################### +if [ -L "$SBIN_DIR/salt-config" ]; then + echo "Cleanup: Removing Symlink $BIN_DIR/salt-config" >> "$TEMP_DIR/preinstall.txt" + rm "$SBIN_DIR/salt-config" + echo "Cleanup: Removed Successfully" >> "$TEMP_DIR/preinstall.txt" +fi + +############################################################################### +# Remove the $INSTALL_DIR directory +############################################################################### +if [ -d "$INSTALL_DIR" ]; then + echo "Cleanup: Removing $INSTALL_DIR" >> "$TEMP_DIR/preinstall.txt" + rm -rf "$INSTALL_DIR" + echo "Cleanup: Removed Successfully" >> "$TEMP_DIR/preinstall.txt" +fi + +############################################################################### +# Remove the salt from the paths.d +############################################################################### +if [ ! -f "/etc/paths.d/salt" ]; then + echo "Path: Removing salt from the path..." >> "$TEMP_DIR/preinstall.txt" + rm "/etc/paths.d/salt" + echo "Path: Removed Successfully" >> "$TEMP_DIR/preinstall.txt" +fi + +echo "Preinstall Completed Successfully on:" >> "$TEMP_DIR/preinstall.txt" +date "+%Y/%m/%d %H:%m:%S" >> "$TEMP_DIR/preinstall.txt" exit 0 diff --git a/pkg/osx/req.txt b/pkg/osx/req.txt index 3bee76f5b1e..338454f456b 100644 --- a/pkg/osx/req.txt +++ b/pkg/osx/req.txt @@ -1,34 +1,31 @@ -apache-libcloud==0.20.1 +apache-libcloud==2.1.0 backports.ssl_match_hostname==3.5.0.1 -backports_abc==0.4 +backports_abc==0.5 certifi -cffi==1.5.0 -CherryPy==4.0.0 -click==6.2 -enum34==1.1.2 +cffi==1.10.0 +CherryPy==11.0.0 +click==6.7 +enum34==1.1.6 gitdb==0.6.4 -GitPython==1.0.1 -idna==2.0 -ioflo==1.5.0 -ipaddress==1.0.16 -Jinja2==2.9.4 -libnacl==1.4.4 +GitPython==2.1.1 +idna==2.5 +ipaddress==1.0.18 +Jinja2==2.9.6 linode-python==1.1.1 -Mako==1.0.3 -MarkupSafe==0.23 -msgpack-python==0.4.7 -pyasn1==0.1.9 -pycparser==2.14 +Mako==1.0.7 +MarkupSafe==1.0 +msgpack-python==0.4.8 +pyasn1==0.2.3 +pycparser==2.18 pycrypto==2.6.1 -python-dateutil==2.4.2 -python-gnupg==0.3.8 -PyYAML==3.11 -pyzmq==15.2.0 -raet==0.6.5 -requests==2.9.1 +python-dateutil==2.6.1 +python-gnupg==0.4.1 +PyYAML==3.12 +pyzmq==16.0.2 +requests==2.18.1 singledispatch==3.4.0.3 six==1.10.0 smmap==0.9.0 timelib==0.2.4 -tornado==4.3 -vultr==0.1.2 +tornado==4.5.1 +vultr==1.0rc1 diff --git a/pkg/osx/req_ext.txt b/pkg/osx/req_ext.txt index b31fff2b1d1..fac429a9429 100644 --- a/pkg/osx/req_ext.txt +++ b/pkg/osx/req_ext.txt @@ -1,2 +1,2 @@ -cryptography==1.2.2 -pyOpenSSL==0.15.1 +cryptography==2.0 +pyOpenSSL==17.1.0 diff --git a/pkg/osx/shasums/Python-2.7.12.tar.xz.sha512 b/pkg/osx/shasums/Python-2.7.12.tar.xz.sha512 deleted file mode 100644 index 513cf3de5ab..00000000000 --- a/pkg/osx/shasums/Python-2.7.12.tar.xz.sha512 +++ /dev/null @@ -1 +0,0 @@ -6ddbbce47cc49597433d98ca05c2f62f07ed1070807b645602a8e9e9b996adc6fa66fa20a33cd7d23d4e7e925e25071d7301d288149fbe4e8c5f06d5438dda1f ./Python-2.7.12.tar.xz diff --git a/pkg/osx/shasums/Python-2.7.13.tar.xz.sha512 b/pkg/osx/shasums/Python-2.7.13.tar.xz.sha512 new file mode 100644 index 00000000000..a57f8db9b9b --- /dev/null +++ b/pkg/osx/shasums/Python-2.7.13.tar.xz.sha512 @@ -0,0 +1 @@ +f37c9a28ce129d01e63c84d7db627a06402854578f62d17927334ea21ede318e04bbf66e890e3f47c85333e6b19f6e5581fb3f3e27efd24be27017d1b6529c4b ./Python-2.7.13.tar.xz diff --git a/pkg/osx/shasums/Python-3.5.3.tar.xz.sha512 b/pkg/osx/shasums/Python-3.5.3.tar.xz.sha512 new file mode 100644 index 00000000000..d4ab61d5f71 --- /dev/null +++ b/pkg/osx/shasums/Python-3.5.3.tar.xz.sha512 @@ -0,0 +1 @@ +bbcc20e315c63dbc8901d7e7bfa29d4dbdad9335720757d8d679730319fd1d9fcfdb55cf62d620c9b052134170f162c28d653a8af60923185b8932524d827864 ./Python-3.5.3.tar.xz diff --git a/pkg/osx/shasums/libsodium-1.0.12.tar.gz.sha512 b/pkg/osx/shasums/libsodium-1.0.12.tar.gz.sha512 deleted file mode 100644 index 948a5da6b65..00000000000 --- a/pkg/osx/shasums/libsodium-1.0.12.tar.gz.sha512 +++ /dev/null @@ -1 +0,0 @@ -1e63960da42bcc90945463ae1f5b1355849881dce5bba6d293391f8d6f0932063a5bfd433a071cb184af90ebeab469acc34710587116922144d61f3d7661901b ./libsodium-1.0.12.tar.gz diff --git a/pkg/osx/shasums/libsodium-1.0.13.tar.gz.sha512 b/pkg/osx/shasums/libsodium-1.0.13.tar.gz.sha512 new file mode 100644 index 00000000000..1b5270adbfb --- /dev/null +++ b/pkg/osx/shasums/libsodium-1.0.13.tar.gz.sha512 @@ -0,0 +1 @@ +c619b12fdf0b2e59174b6e383a62d5499ebcd720fdbb2c1a41a98a46c285df075202423454b294fefee185432441e943805397d7656f7cd7837de425da623929 ./libsodium-1.0.13.tar.gz diff --git a/pkg/osx/shasums/openssl-1.0.2f.tar.gz.sha512 b/pkg/osx/shasums/openssl-1.0.2f.tar.gz.sha512 deleted file mode 100644 index b107e52b8a0..00000000000 --- a/pkg/osx/shasums/openssl-1.0.2f.tar.gz.sha512 +++ /dev/null @@ -1 +0,0 @@ -50abf6dc94cafd06e7fd20770808bdc675c88daa369e4f752bd584ab17f72a57357c1ca1eca3c83e6745b5a3c9c73c99dce70adaa904d73f6df4c75bc7138351 ./openssl-1.0.2f.tar.gz diff --git a/pkg/osx/shasums/openssl-1.0.2l.tar.gz.sha512 b/pkg/osx/shasums/openssl-1.0.2l.tar.gz.sha512 new file mode 100644 index 00000000000..9dac0d205f6 --- /dev/null +++ b/pkg/osx/shasums/openssl-1.0.2l.tar.gz.sha512 @@ -0,0 +1 @@ +047d964508ad6025c79caabd8965efd2416dc026a56183d0ef4de7a0a6769ce8e0b4608a3f8393d326f6d03b26a2b067e6e0c750f35b20be190e595e8290c0e3 ./openssl-1.0.2l.tar.gz diff --git a/pkg/osx/shasums/pkg-config-0.29.2.tar.gz.sha512 b/pkg/osx/shasums/pkg-config-0.29.2.tar.gz.sha512 new file mode 100644 index 00000000000..beb4354b5ab --- /dev/null +++ b/pkg/osx/shasums/pkg-config-0.29.2.tar.gz.sha512 @@ -0,0 +1 @@ +4861ec6428fead416f5cbbbb0bbad10b9152967e481d4b0ff2eb396a9f297f552984c9bb72f6864a37dcd8fca1d9ccceda3ef18d8f121938dbe4fdf2b870fe75 ./pkg-config-0.29.2.tar.gz diff --git a/pkg/osx/shasums/pkg-config-0.29.tar.gz.sha512 b/pkg/osx/shasums/pkg-config-0.29.tar.gz.sha512 deleted file mode 100644 index 8e19abaabb2..00000000000 --- a/pkg/osx/shasums/pkg-config-0.29.tar.gz.sha512 +++ /dev/null @@ -1 +0,0 @@ -c2857cd67801c0db5d204912453ff6bdc7da3ea61f8b1c6b38983d48dffb958725e7723f909abbc057c7b34a85c27290eec6943808312a75909306076064aa63 ./pkg-config-0.29.tar.gz diff --git a/pkg/osx/shasums/zeromq-4.1.3.tar.gz.sha512 b/pkg/osx/shasums/zeromq-4.1.3.tar.gz.sha512 deleted file mode 100644 index ea85a1581d0..00000000000 --- a/pkg/osx/shasums/zeromq-4.1.3.tar.gz.sha512 +++ /dev/null @@ -1 +0,0 @@ -2c993d18ea44e1cba890e024176af65b85b842ca4f8a22d319be4ace8388ab8828dd706b065f02754025bf271b1d7aa878c3f6655878248f7826452cb2a6134c ./zeromq-4.1.3.tar.gz diff --git a/pkg/osx/shasums/zeromq-4.1.4.tar.gz.sha512 b/pkg/osx/shasums/zeromq-4.1.4.tar.gz.sha512 new file mode 100644 index 00000000000..4f8932f8298 --- /dev/null +++ b/pkg/osx/shasums/zeromq-4.1.4.tar.gz.sha512 @@ -0,0 +1 @@ +8a8cf4f52ad78dddfff104bfba0f80bbc12566920906a0fafb9fc340aa92f5577c2923cb2e5346c69835cd2ea1609647a8893c2883cd22c1f0340a720511460c ./zeromq-4.1.4.tar.gz diff --git a/pkg/suse/salt-api.service b/pkg/suse/salt-api.service index 130d7da5b62..6634b74a7d2 100644 --- a/pkg/suse/salt-api.service +++ b/pkg/suse/salt-api.service @@ -1,11 +1,11 @@ [Unit] Description=The Salt API +Documentation=man:salt-api(1) file:///usr/share/doc/salt/html/contents.html https://docs.saltstack.com/en/latest/contents.html After=network.target [Service] User=salt Type=simple -Environment=SHELL=/bin/bash LimitNOFILE=8192 ExecStart=/usr/bin/salt-api TimeoutStopSec=3 diff --git a/pkg/suse/salt-common.logrotate b/pkg/suse/salt-common.logrotate new file mode 100644 index 00000000000..0d99d1b801c --- /dev/null +++ b/pkg/suse/salt-common.logrotate @@ -0,0 +1,25 @@ +/var/log/salt/master { + su salt salt + weekly + missingok + rotate 7 + compress + notifempty +} + +/var/log/salt/minion { + weekly + missingok + rotate 7 + compress + notifempty +} + +/var/log/salt/key { + su salt salt + weekly + missingok + rotate 7 + compress + notifempty +} diff --git a/pkg/suse/salt-master.service b/pkg/suse/salt-master.service new file mode 100644 index 00000000000..c0ea4606d87 --- /dev/null +++ b/pkg/suse/salt-master.service @@ -0,0 +1,13 @@ +[Unit] +Description=The Salt Master Server +Documentation=man:salt-master(1) file:///usr/share/doc/salt/html/contents.html https://docs.saltstack.com/en/latest/contents.html +After=network.target + +[Service] +LimitNOFILE=16384 +Type=simple +ExecStart=/usr/bin/salt-master +TasksMax=infinity + +[Install] +WantedBy=multi-user.target diff --git a/pkg/suse/salt-minion b/pkg/suse/salt-minion old mode 100644 new mode 100755 index 0f04a13e176..2e418094ed4 --- a/pkg/suse/salt-minion +++ b/pkg/suse/salt-minion @@ -51,8 +51,23 @@ SERVICE=salt-minion PROCESS=salt-minion RETVAL=0 +WATCHDOG_CRON="/etc/cron.d/salt-minion" + +set_watchdog() { + if [ ! -f $WATCHDOG_CRON ]; then + echo -e '* * * * * root /usr/bin/salt-daemon-watcher --with-init\n' > $WATCHDOG_CRON + # Kick the watcher for 1 minute immediately, because cron will wake up only afterwards + /usr/bin/salt-daemon-watcher --with-init & disown + fi +} + +remove_watchdog() { + rm $WATCHDOG_CRON 2>/dev/null || true + kill -9 $(ps uax | grep [s]alt-daemon-watcher | awk '{print $2}') 2>/dev/null +} start() { + set_watchdog; echo -n $"Starting salt-minion daemon: " if [ -f $SUSE_RELEASE ]; then startproc -p /var/run/$SERVICE.pid $SALTMINION -d $MINION_ARGS @@ -80,6 +95,10 @@ start() { } stop() { + IS_RESTARTING=$1 + if [ -z $IS_RESTARTING ]; then + remove_watchdog; + fi echo -n $"Stopping salt-minion daemon: " if [ -f $SUSE_RELEASE ]; then killproc -TERM $SALTMINION @@ -101,8 +120,8 @@ stop() { } restart() { - stop - start + stop 1; + start; } # See how we were called. diff --git a/pkg/suse/salt-minion.service.rhel7 b/pkg/suse/salt-minion.service.rhel7 new file mode 100644 index 00000000000..69172677140 --- /dev/null +++ b/pkg/suse/salt-minion.service.rhel7 @@ -0,0 +1,14 @@ +[Unit] +Description=The Salt Minion +After=network.target + +[Service] +Type=simple +LimitNOFILE=8192 +ExecStart=/usr/bin/salt-minion +KillMode=process +Restart=on-failure +RestartSec=15 + +[Install] +WantedBy=multi-user.target diff --git a/pkg/windows/build.bat b/pkg/windows/build.bat index 9dde8b2e72d..59fafde137c 100644 --- a/pkg/windows/build.bat +++ b/pkg/windows/build.bat @@ -89,7 +89,7 @@ if Defined x ( if %Python%==2 ( Set "PyDir=C:\Python27" ) else ( - Set "PyDir=C:\Program Files\Python35" + Set "PyDir=C:\Python35" ) Set "PATH=%PATH%;%PyDir%;%PyDir%\Scripts" @@ -110,6 +110,13 @@ if not %errorLevel%==0 ( ) @echo. +:: Remove build and dist directories +@echo %0 :: Remove build and dist directories... +@echo --------------------------------------------------------------------- +rd /s /q "%SrcDir%\build" +rd /s /q "%SrcDir%\dist" +@echo. + :: Install Current Version of salt @echo %0 :: Install Current Version of salt... @echo --------------------------------------------------------------------- diff --git a/pkg/windows/build_env_2.ps1 b/pkg/windows/build_env_2.ps1 index 98a922ca3d7..b1865178126 100644 --- a/pkg/windows/build_env_2.ps1 +++ b/pkg/windows/build_env_2.ps1 @@ -175,7 +175,7 @@ If (Test-Path "$($ini['Settings']['Python2Dir'])\python.exe") { DownloadFileWithProgress $url $file Write-Output " - $script_name :: Installing $($ini[$bitPrograms]['Python2']) . . ." - $p = Start-Process msiexec -ArgumentList "/i $file /qb ADDLOCAL=DefaultFeature,SharedCRT,Extensions,pip_feature,PrependPath TARGETDIR=$($ini['Settings']['Python2Dir'])" -Wait -NoNewWindow -PassThru + $p = Start-Process msiexec -ArgumentList "/i $file /qb ADDLOCAL=DefaultFeature,SharedCRT,Extensions,pip_feature,PrependPath TARGETDIR=`"$($ini['Settings']['Python2Dir'])`"" -Wait -NoNewWindow -PassThru } #------------------------------------------------------------------------------ @@ -191,7 +191,7 @@ If (!($Path.ToLower().Contains("$($ini['Settings']['Scripts2Dir'])".ToLower()))) #============================================================================== # Update PIP and SetupTools -# caching depends on environmant variable SALT_PIP_LOCAL_CACHE +# caching depends on environment variable SALT_PIP_LOCAL_CACHE #============================================================================== Write-Output " ----------------------------------------------------------------" Write-Output " - $script_name :: Updating PIP and SetupTools . . ." @@ -212,7 +212,7 @@ if ( ! [bool]$Env:SALT_PIP_LOCAL_CACHE) { #============================================================================== # Install pypi resources using pip -# caching depends on environmant variable SALT_REQ_LOCAL_CACHE +# caching depends on environment variable SALT_REQ_LOCAL_CACHE #============================================================================== Write-Output " ----------------------------------------------------------------" Write-Output " - $script_name :: Installing pypi resources using pip . . ." @@ -230,6 +230,24 @@ if ( ! [bool]$Env:SALT_REQ_LOCAL_CACHE) { Start_Process_and_test_exitcode "$($ini['Settings']['Python2Dir'])\python.exe" "-m pip install --no-index --find-links=$Env:SALT_REQ_LOCAL_CACHE -r $($script_path)\req_2.txt" "pip install" } +#============================================================================== +# Move PyWin32 DLL's to site-packages\win32 +#============================================================================== +Write-Output " - $script_name :: Moving PyWin32 DLLs . . ." +Move-Item "$($ini['Settings']['SitePkgs2Dir'])\pywin32_system32\*.dll" "$($ini['Settings']['SitePkgs2Dir'])\win32" -Force + +# Remove pywin32_system32 directory +Write-Output " - $script_name :: Removing pywin32_system32 Directory . . ." +Remove-Item "$($ini['Settings']['SitePkgs2Dir'])\pywin32_system32" + +# Remove pythonwin directory +Write-Output " - $script_name :: Removing pythonwin Directory . . ." +Remove-Item "$($ini['Settings']['SitePkgs2Dir'])\pythonwin" -Force -Recurse + +# Remove PyWin32 PostInstall and testall Scripts +Write-Output " - $script_name :: Removing PyWin32 scripts . . ." +Remove-Item "$($ini['Settings']['Scripts2Dir'])\pywin32_*" -Force -Recurse + #============================================================================== # Install PyYAML with CLoader # This has to be a compiled binary to get the CLoader diff --git a/pkg/windows/build_env_3.ps1 b/pkg/windows/build_env_3.ps1 index 33f95871ae4..0dcbafd996c 100644 --- a/pkg/windows/build_env_3.ps1 +++ b/pkg/windows/build_env_3.ps1 @@ -175,7 +175,7 @@ If (Test-Path "$($ini['Settings']['Python3Dir'])\python.exe") { DownloadFileWithProgress $url $file Write-Output " - $script_name :: Installing $($ini[$bitPrograms]['Python3']) . . ." - $p = Start-Process $file -ArgumentList '/passive InstallAllUsers=1 TargetDir="C:\Program Files\Python35" Include_doc=0 Include_tcltk=0 Include_test=0 Include_launcher=0 PrependPath=1 Shortcuts=0' -Wait -NoNewWindow -PassThru + $p = Start-Process $file -ArgumentList "/passive InstallAllUsers=1 TargetDir=`"$($ini['Settings']['Python3Dir'])`" Include_doc=0 Include_tcltk=0 Include_test=0 Include_launcher=0 PrependPath=1 Shortcuts=0" -Wait -NoNewWindow -PassThru } #------------------------------------------------------------------------------ @@ -247,7 +247,7 @@ Start_Process_and_test_exitcode "$($ini['Settings']['Scripts3Dir'])\pip.exe" "i # Move DLL's to Python Root Write-Output " - $script_name :: Moving PyWin32 DLLs . . ." -Move-Item "$($ini['Settings']['SitePkgs3Dir'])\pywin32_system32\*.dll" "$($ini['Settings']['Python3Dir'])" -Force +Move-Item "$($ini['Settings']['SitePkgs3Dir'])\pywin32_system32\*.dll" "$($ini['Settings']['SitePkgs3Dir'])\win32" -Force # Remove pywin32_system32 directory Write-Output " - $script_name :: Removing pywin32_system32 Directory . . ." @@ -257,6 +257,10 @@ Remove-Item "$($ini['Settings']['SitePkgs3Dir'])\pywin32_system32" Write-Output " - $script_name :: Removing pythonwin Directory . . ." Remove-Item "$($ini['Settings']['SitePkgs3Dir'])\pythonwin" -Force -Recurse +# Remove PyWin32 PostInstall and testall Scripts +Write-Output " - $script_name :: Removing PyWin32 scripts . . ." +Remove-Item "$($ini['Settings']['Scripts3Dir'])\pywin32_*" -Force -Recurse + #============================================================================== # Fix PyCrypto #============================================================================== diff --git a/pkg/windows/build_pkg.bat b/pkg/windows/build_pkg.bat index b5e7d42dbdd..95b185bfa7d 100644 --- a/pkg/windows/build_pkg.bat +++ b/pkg/windows/build_pkg.bat @@ -56,7 +56,7 @@ if %Python%==2 ( Set "PyVerMajor=2" Set "PyVerMinor=7" ) else ( - Set "PyDir=C:\Program Files\Python35" + Set "PyDir=C:\Python35" Set "PyVerMajor=3" Set "PyVerMinor=5" ) @@ -67,10 +67,13 @@ If not Exist "%PyDir%\python.exe" ( exit /b 1 ) -Set "CurrDir=%cd%" -Set "BinDir=%cd%\buildenv\bin" -Set "InsDir=%cd%\installer" -Set "PreDir=%cd%\prereqs" +Set "CurDir=%~dp0" +Set "BldDir=%CurDir%\buildenv" +Set "BinDir=%CurDir%\buildenv\bin" +Set "CnfDir=%CurDir%\buildenv\conf" +Set "InsDir=%CurDir%\installer" +Set "PreDir=%CurDir%\prereqs" +for /f "delims=" %%a in ('git rev-parse --show-toplevel') do @set "SrcDir=%%a" :: Find the NSIS Installer If Exist "C:\Program Files\NSIS\" ( @@ -101,6 +104,15 @@ If Exist "%BinDir%\" ( xcopy /E /Q "%PyDir%" "%BinDir%\" @echo. +:: Copy the default master and minion configs to buildenv\conf +@echo Copying configs to buildenv\conf... +@echo ---------------------------------------------------------------------- +@echo xcopy /E /Q "%SrcDir%\conf\master" "%CnfDir%\" +xcopy /Q /Y "%SrcDir%\conf\master" "%CnfDir%\" +@echo xcopy /E /Q "%SrcDir%\conf\minion" "%CnfDir%\" +xcopy /Q /Y "%SrcDir%\conf\minion" "%CnfDir%\" +@echo. + @echo Copying VCRedist to Prerequisites @echo ---------------------------------------------------------------------- :: Make sure the "prereq" directory exists @@ -127,12 +139,12 @@ If Defined ProgramFiles(x86) ( :: Remove the fixed path in .exe files @echo Removing fixed path from .exe files @echo ---------------------------------------------------------------------- -"%PyDir%\python" "%CurrDir%\portable.py" -f "%BinDir%\Scripts\easy_install.exe" -"%PyDir%\python" "%CurrDir%\portable.py" -f "%BinDir%\Scripts\easy_install-%PyVerMajor%.%PyVerMinor%.exe" -"%PyDir%\python" "%CurrDir%\portable.py" -f "%BinDir%\Scripts\pip.exe" -"%PyDir%\python" "%CurrDir%\portable.py" -f "%BinDir%\Scripts\pip%PyVerMajor%.%PyVerMinor%.exe" -"%PyDir%\python" "%CurrDir%\portable.py" -f "%BinDir%\Scripts\pip%PyVerMajor%.exe" -"%PyDir%\python" "%CurrDir%\portable.py" -f "%BinDir%\Scripts\wheel.exe" +"%PyDir%\python" "%CurDir%\portable.py" -f "%BinDir%\Scripts\easy_install.exe" +"%PyDir%\python" "%CurDir%\portable.py" -f "%BinDir%\Scripts\easy_install-%PyVerMajor%.%PyVerMinor%.exe" +"%PyDir%\python" "%CurDir%\portable.py" -f "%BinDir%\Scripts\pip.exe" +"%PyDir%\python" "%CurDir%\portable.py" -f "%BinDir%\Scripts\pip%PyVerMajor%.%PyVerMinor%.exe" +"%PyDir%\python" "%CurDir%\portable.py" -f "%BinDir%\Scripts\pip%PyVerMajor%.exe" +"%PyDir%\python" "%CurDir%\portable.py" -f "%BinDir%\Scripts\wheel.exe" @echo. @echo Cleaning up unused files and directories... @@ -534,12 +546,6 @@ If Exist "%BinDir%\Lib\site-packages\salt\states\zpool.py"^ :: Remove Unneeded Components If Exist "%BinDir%\Lib\site-packages\salt\cloud"^ rd /S /Q "%BinDir%\Lib\site-packages\salt\cloud" 1>nul -If Exist "%BinDir%\Scripts\salt-key*"^ - del /Q "%BinDir%\Scripts\salt-key*" 1>nul -If Exist "%BinDir%\Scripts\salt-master*"^ - del /Q "%BinDir%\Scripts\salt-master*" 1>nul -If Exist "%BinDir%\Scripts\salt-run*"^ - del /Q "%BinDir%\Scripts\salt-run*" 1>nul If Exist "%BinDir%\Scripts\salt-unity*"^ del /Q "%BinDir%\Scripts\salt-unity*" 1>nul @@ -547,6 +553,40 @@ If Exist "%BinDir%\Scripts\salt-unity*"^ @echo Building the installer... @echo ---------------------------------------------------------------------- +:: Make the Master installer if the nullsoft script exists +If Exist "%InsDir%\Salt-Setup.nsi"^ + makensis.exe /DSaltVersion=%Version% /DPythonVersion=%Python% "%InsDir%\Salt-Setup.nsi" + +:: Remove files not needed for Salt Minion +:: salt +:: salt has to be removed individually (can't wildcard it) +If Exist "%BinDir%\Scripts\salt"^ + del /Q "%BinDir%\Scripts\salt" 1>nul +If Exist "%BinDir%\Scripts\salt.exe"^ + del /Q "%BinDir%\Scripts\salt.exe" 1>nul +If Exist "%BldDir%\salt.bat"^ + del /Q "%BldDir%\salt.bat" 1>nul +:: salt-key +If Exist "%BinDir%\Scripts\salt-key*"^ + del /Q "%BinDir%\Scripts\salt-key*" 1>nul +If Exist "%BldDir%\salt-key.bat"^ + del /Q "%BldDir%\salt-key.bat" 1>nul +:: salt-master +If Exist "%BinDir%\Scripts\salt-master*"^ + del /Q "%BinDir%\Scripts\salt-master*" 1>nul +If Exist "%BldDir%\salt-master.bat"^ + del /Q "%BldDir%\salt-master.bat" 1>nul +:: salt-run +If Exist "%BinDir%\Scripts\salt-run*"^ + del /Q "%BinDir%\Scripts\salt-run*" 1>nul +If Exist "%BldDir%\salt-run.bat"^ + del /Q "%BldDir%\salt-run.bat" 1>nul + +:: Remove the master config file +if Exist "%CnfDir%\master"^ + del /Q "%CnfDir%\master" 1>nul + +:: Make the Salt Minion Installer makensis.exe /DSaltVersion=%Version% /DPythonVersion=%Python% "%InsDir%\Salt-Minion-Setup.nsi" @echo. diff --git a/pkg/windows/buildenv/conf/minion b/pkg/windows/buildenv/conf/minion deleted file mode 100644 index e9f168b3210..00000000000 --- a/pkg/windows/buildenv/conf/minion +++ /dev/null @@ -1,402 +0,0 @@ -##### Primary configuration settings ##### -########################################## - -ipc_mode: tcp - -# Per default the minion will automatically include all config files -# from minion.d/*.conf (minion.d is a directory in the same directory -# as the main minion config file). -#default_include: minion.d/*.conf - -# Set the location of the salt master server, if the master server cannot be -# resolved, then the minion will fail to start. -# test -#master: salt - -# Set the number of seconds to wait before attempting to resolve -# the master hostname if name resolution fails. Defaults to 30 seconds. -# Set to zero if the minion should shutdown and not retry. -# retry_dns: 30 - -# Set the port used by the master reply and authentication server -#master_port: 4506 - -# The user to run salt -#user: root - -# Specify the location of the daemon process ID file -#pidfile: /var/run/salt-minion.pid - -# The root directory prepended to these options: pki_dir, cachedir, log_file, -# sock_dir, pidfile. -root_dir: c:\salt - -# The directory to store the pki information in -#pki_dir: /etc/salt/pki/minion -pki_dir: /conf/pki/minion - -# Explicitly declare the id for this minion to use, if left commented the id -# will be the hostname as returned by the python call: socket.getfqdn() -# Since salt uses detached ids it is possible to run multiple minions on the -# same machine but with different ids, this can be useful for salt compute -# clusters. -#id: - -# Append a domain to a hostname in the event that it does not exist. This is -# useful for systems where socket.getfqdn() does not actually result in a -# FQDN (for instance, Solaris). -#append_domain: - -# Custom static grains for this minion can be specified here and used in SLS -# files just like all other grains. This example sets 4 custom grains, with -# the 'roles' grain having two values that can be matched against: -#grains: -# roles: -# - webserver -# - memcache -# deployment: datacenter4 -# cabinet: 13 -# cab_u: 14-15 - -# Where cache data goes -#cachedir: /var/cache/salt/minion - -# Verify and set permissions on configuration directories at startup -#verify_env: True - -# The minion can locally cache the return data from jobs sent to it, this -# can be a good way to keep track of jobs the minion has executed -# (on the minion side). By default this feature is disabled, to enable -# set cache_jobs to True -#cache_jobs: False - -# set the directory used to hold unix sockets -#sock_dir: /var/run/salt/minion - -# Backup files that are replaced by file.managed and file.recurse under -# 'cachedir'/file_backups relative to their original location and appended -# with a timestamp. The only valid setting is "minion". Disabled by default. -# -# Alternatively this can be specified for each file in state files: -# -# /etc/ssh/sshd_config: -# file.managed: -# - source: salt://ssh/sshd_config -# - backup: minion -# -#backup_mode: minion - -# When waiting for a master to accept the minion's public key, salt will -# continuously attempt to reconnect until successful. This is the time, in -# seconds, between those reconnection attempts. -#acceptance_wait_time: 10 - -# If this is set, the time between reconnection attempts will increase by -# acceptance_wait_time seconds per iteration, up to this maximum. If this -# is not set, the time between reconnection attempts will stay constant. -#acceptance_wait_time_max: None - -# Windows platforms lack posix IPC and must rely on slower TCP based inter- -# process communications. Set ipc_mode to 'tcp' on such systems -#ipc_mode: ipc -# -# Overwrite the default tcp ports used by the minion when in tcp mode -#tcp_pub_port: 4510 -#tcp_pull_port: 4511 - -# The minion can include configuration from other files. To enable this, -# pass a list of paths to this option. The paths can be either relative or -# absolute; if relative, they are considered to be relative to the directory -# the main minion configuration file lives in (this file). Paths can make use -# of shell-style globbing. If no files are matched by a path passed to this -# option then the minion will log a warning message. -# -# -# Include a config file from some other path: -# include: /etc/salt/extra_config -# -# Include config from several files and directories: -# include: -# - /etc/salt/extra_config -# - /etc/roles/webserver - -##### Minion module management ##### -########################################## -# Disable specific modules. This allows the admin to limit the level of -# access the master has to the minion -#disable_modules: [cmd,test] -#disable_returners: [] -# -# Modules can be loaded from arbitrary paths. This enables the easy deployment -# of third party modules. Modules for returners and minions can be loaded. -# Specify a list of extra directories to search for minion modules and -# returners. These paths must be fully qualified! -#module_dirs: [] -#returner_dirs: [] -#states_dirs: [] -#render_dirs: [] -# -# A module provider can be statically overwritten or extended for the minion -# via the providers option, in this case the default module will be -# overwritten by the specified module. In this example the pkg module will -# be provided by the yumpkg5 module instead of the system default. -# -# providers: -# pkg: yumpkg5 -# -# Enable Cython modules searching and loading. (Default: False) -#cython_enable: False -# - -##### State Management Settings ##### -########################################### -# The state management system executes all of the state templates on the minion -# to enable more granular control of system state management. The type of -# template and serialization used for state management needs to be configured -# on the minion, the default renderer is yaml_jinja. This is a yaml file -# rendered from a jinja template, the available options are: -# yaml_jinja -# yaml_mako -# yaml_wempy -# json_jinja -# json_mako -# json_wempy -# -#renderer: yaml_jinja -# -# The failhard option tells the minions to stop immediately after the first -# failure detected in the state execution, defaults to False -#failhard: False -# -# autoload_dynamic_modules Turns on automatic loading of modules found in the -# environments on the master. This is turned on by default, to turn of -# autoloading modules when states run set this value to False -#autoload_dynamic_modules: True -# -# clean_dynamic_modules keeps the dynamic modules on the minion in sync with -# the dynamic modules on the master, this means that if a dynamic module is -# not on the master it will be deleted from the minion. By default this is -# enabled and can be disabled by changing this value to False -#clean_dynamic_modules: True -# -# Normally the minion is not isolated to any single environment on the master -# when running states, but the environment can be isolated on the minion side -# by statically setting it. Remember that the recommended way to manage -# environments is to isolate via the top file. -#environment: None -# -# If using the local file directory, then the state top file name needs to be -# defined, by default this is top.sls. -#state_top: top.sls -# -# Run states when the minion daemon starts. To enable, set startup_states to: -# 'highstate' -- Execute state.highstate -# 'sls' -- Read in the sls_list option and execute the named sls files -# 'top' -- Read top_file option and execute based on that file on the Master -#startup_states: '' -# -# list of states to run when the minion starts up if startup_states is 'sls' -#sls_list: -# - edit.vim -# - hyper -# -# top file to execute if startup_states is 'top' -#top_file: '' - -##### File Directory Settings ##### -########################################## -# The Salt Minion can redirect all file server operations to a local directory, -# this allows for the same state tree that is on the master to be used if -# copied completely onto the minion. This is a literal copy of the settings on -# the master but used to reference a local directory on the minion. - -# Set the file client, the client defaults to looking on the master server for -# files, but can be directed to look at the local file directory setting -# defined below by setting it to local. -#file_client: remote - -# The file directory works on environments passed to the minion, each environment -# can have multiple root directories, the subdirectories in the multiple file -# roots cannot match, otherwise the downloaded files will not be able to be -# reliably ensured. A base environment is required to house the top file. -# Example: -# file_roots: -# base: -# - /srv/salt/ -# dev: -# - /srv/salt/dev/services -# - /srv/salt/dev/states -# prod: -# - /srv/salt/prod/services -# - /srv/salt/prod/states -# -# Default: -#file_roots: -# base: -# - /srv/salt - -# The hash_type is the hash to use when discovering the hash of a file in -# the minion directory, the default is md5, but sha1, sha224, sha256, sha384 -# and sha512 are also supported. -#hash_type: md5 - -# The Salt pillar is searched for locally if file_client is set to local. If -# this is the case, and pillar data is defined, then the pillar_roots need to -# also be configured on the minion: -#pillar_roots: -# base: -# - /srv/pillar - -###### Security settings ##### -########################################### -# Enable "open mode", this mode still maintains encryption, but turns off -# authentication, this is only intended for highly secure environments or for -# the situation where your keys end up in a bad state. If you run in open mode -# you do so at your own risk! -#open_mode: False - -# Enable permissive access to the salt keys. This allows you to run the -# master or minion as root, but have a non-root group be given access to -# your pki_dir. To make the access explicit, root must belong to the group -# you've given access to. This is potentially quite insecure. -#permissive_pki_access: False - -# The state_verbose and state_output settings can be used to change the way -# state system data is printed to the display. By default all data is printed. -# The state_verbose setting can be set to True or False, when set to False -# all data that has a result of True and no changes will be suppressed. -#state_verbose: True -# -# The state_output setting changes if the output is the full multi line -# output for each changed state if set to 'full', but if set to 'terse' -# the output will be shortened to a single line. -#state_output: full -# -# Fingerprint of the master public key to double verify the master is valid, -# the master fingerprint can be found by running "salt-key -F master" on the -# salt master. -#master_finger: '' - -###### Thread settings ##### -########################################### -# Disable multiprocessing support, by default when a minion receives a -# publication a new process is spawned and the command is executed therein. -# multiprocessing: True - -###### Logging settings ##### -########################################### -# The location of the minion log file. -# This can be a path for the log file, or, this can be, since 0.11.0, a system -# logger address, for example: -# tcp://localhost:514/LOG_USER -# tcp://localhost/LOG_DAEMON -# udp://localhost:5145/LOG_KERN -# udp://localhost -# file:///dev/log -# file:///dev/log/LOG_SYSLOG -# file:///dev/log/LOG_DAEMON -# -# The above examples are self explanatory, but: -# ://:/ -# -# Make sure you have a properly configured syslog or you won't get any warnings -# -#log_file: /var/log/salt/minion -# -# -# The level of messages to send to the console. -# One of 'garbage', 'trace', 'debug', info', 'warning', 'error', 'critical'. -# Default: 'warning' -#log_level: warning -# -# The level of messages to send to the log file. -# One of 'garbage', 'trace', 'debug', info', 'warning', 'error', 'critical'. -# Default: 'warning' -#log_level_logfile: -# -# The date and time format used in log messages. Allowed date/time formatting -# can be seen on http://docs.python.org/library/time.html#time.strftime -#log_datefmt: '%H:%M:%S' -#log_datefmt_logfile: '%Y-%m-%d %H:%M:%S' -# -# The format of the console logging messages. Allowed formatting options can -# be seen on http://docs.python.org/library/logging.html#logrecord-attributes -#log_fmt_console: '[%(levelname)-8s] %(message)s' -#log_fmt_logfile: '%(asctime)s,%(msecs)03d [%(name)-17s][%(levelname)-8s] %(message)s' -# -# Logger levels can be used to tweak specific loggers logging levels. -# For example, if you want to have the salt library at the 'warning' level, -# but you still wish to have 'salt.modules' at the 'debug' level: -# log_granular_levels: { -# 'salt': 'warning', -# 'salt.modules': 'debug' -# } -# -#log_granular_levels: {} - -###### Module configuration ##### -########################################### -# Salt allows for modules to be passed arbitrary configuration data, any data -# passed here in valid yaml format will be passed on to the salt minion modules -# for use. It is STRONGLY recommended that a naming convention be used in which -# the module name is followed by a . and then the value. Also, all top level -# data must be applied via the yaml dict construct, some examples: -# -# You can specify that all modules should run in test mode: -#test: True -# -# A simple value for the test module: -#test.foo: foo -# -# A list for the test module: -#test.bar: [baz,quo] -# -# A dict for the test module: -#test.baz: {spam: sausage, cheese: bread} - - -###### Update settings ###### -########################################### -# Using the features in Esky, a salt minion can both run as a frozen app and -# be updated on the fly. These options control how the update process -# (saltutil.update()) behaves. -# -# The url for finding and downloading updates. Disabled by default. -#update_url: False -# -# The list of services to restart after a successful update. Empty by default. -#update_restart_services: [] - - -###### Keepalive settings ###### -############################################ -# ZeroMQ now includes support for configuring SO_KEEPALIVE if supported by -# the OS. If connections between the minion and the master pass through -# a state tracking device such as a firewall or VPN gateway, there is -# the risk that it could tear down the connection the master and minion -# without informing either party that their connection has been taken away. -# Enabling TCP Keepalives prevents this from happening. -# -# Overall state of TCP Keepalives, enable (1 or True), disable (0 or False) -# or leave to the OS defaults (-1), on Linux, typically disabled. Default True, enabled. -#tcp_keepalive: True -# -# How long before the first keepalive should be sent in seconds. Default 300 -# to send the first keepalive after 5 minutes, OS default (-1) is typically 7200 seconds -# on Linux see /proc/sys/net/ipv4/tcp_keepalive_time. -#tcp_keepalive_idle: 300 -# -# How many lost probes are needed to consider the connection lost. Default -1 -# to use OS defaults, typically 9 on Linux, see /proc/sys/net/ipv4/tcp_keepalive_probes. -#tcp_keepalive_cnt: -1 -# -# How often, in seconds, to send keepalives after the first one. Default -1 to -# use OS defaults, typically 75 seconds on Linux, see -# /proc/sys/net/ipv4/tcp_keepalive_intvl. -#tcp_keepalive_intvl: -1 - - -###### Windows Software settings ###### -############################################ -# Location of the repository cache file on the master -# win_repo_cachefile: 'salt://win/repo/winrepo.p' diff --git a/pkg/windows/buildenv/salt-call.bat b/pkg/windows/buildenv/salt-call.bat index 095f51e4c17..55f7cfac3b3 100644 --- a/pkg/windows/buildenv/salt-call.bat +++ b/pkg/windows/buildenv/salt-call.bat @@ -9,5 +9,4 @@ Set Python=%SaltDir%\bin\python.exe Set Script=%SaltDir%\bin\Scripts\salt-call :: Launch Script -"%Python%" "%Script%" %* - +"%Python%" -E -s "%Script%" %* diff --git a/pkg/windows/buildenv/salt-cp.bat b/pkg/windows/buildenv/salt-cp.bat index 29274320b18..61fd1ce4443 100644 --- a/pkg/windows/buildenv/salt-cp.bat +++ b/pkg/windows/buildenv/salt-cp.bat @@ -9,5 +9,4 @@ Set Python=%SaltDir%\bin\python.exe Set Script=%SaltDir%\bin\Scripts\salt-cp :: Launch Script -"%Python%" "%Script%" %* - +"%Python%" -E -s "%Script%" %* diff --git a/pkg/windows/buildenv/salt-key.bat b/pkg/windows/buildenv/salt-key.bat index 471e3626b24..4928b696d5f 100644 --- a/pkg/windows/buildenv/salt-key.bat +++ b/pkg/windows/buildenv/salt-key.bat @@ -9,5 +9,4 @@ Set Python=%SaltDir%\bin\python.exe Set Script=%SaltDir%\bin\Scripts\salt-key :: Launch Script -"%Python%" "%Script%" %* - +"%Python%" -E -s "%Script%" %* diff --git a/pkg/windows/buildenv/salt-master.bat b/pkg/windows/buildenv/salt-master.bat index 9a124ffd467..134875c072c 100644 --- a/pkg/windows/buildenv/salt-master.bat +++ b/pkg/windows/buildenv/salt-master.bat @@ -9,5 +9,4 @@ Set Python=%SaltDir%\bin\python.exe Set Script=%SaltDir%\bin\Scripts\salt-master :: Launch Script -"%Python%" "%Script%" %* - +"%Python%" -E -s "%Script%" %* diff --git a/pkg/windows/buildenv/salt-minion-debug.bat b/pkg/windows/buildenv/salt-minion-debug.bat index ad0ebafee0c..2e6b5c5bcf7 100644 --- a/pkg/windows/buildenv/salt-minion-debug.bat +++ b/pkg/windows/buildenv/salt-minion-debug.bat @@ -12,5 +12,4 @@ Set Script=%SaltDir%\bin\Scripts\salt-minion net stop salt-minion :: Launch Script -"%Python%" "%Script%" -l debug - +"%Python%" -E -s "%Script%" -l debug diff --git a/pkg/windows/buildenv/salt-minion.bat b/pkg/windows/buildenv/salt-minion.bat index 0a1aafa0c06..0eb041219c2 100644 --- a/pkg/windows/buildenv/salt-minion.bat +++ b/pkg/windows/buildenv/salt-minion.bat @@ -9,5 +9,4 @@ Set Python=%SaltDir%\bin\python.exe Set Script=%SaltDir%\bin\Scripts\salt-minion :: Launch Script -"%Python%" "%Script%" %* - +"%Python%" -E -s "%Script%" %* diff --git a/pkg/windows/buildenv/salt-run.bat b/pkg/windows/buildenv/salt-run.bat index 5d62775d5ec..a8766fc9b04 100644 --- a/pkg/windows/buildenv/salt-run.bat +++ b/pkg/windows/buildenv/salt-run.bat @@ -9,5 +9,4 @@ Set Python=%SaltDir%\bin\python.exe Set Script=%SaltDir%\bin\Scripts\salt-run :: Launch Script -"%Python%" "%Script%" %* - +"%Python%" -E -s "%Script%" %* diff --git a/pkg/windows/buildenv/salt.bat b/pkg/windows/buildenv/salt.bat new file mode 100644 index 00000000000..5b07c732812 --- /dev/null +++ b/pkg/windows/buildenv/salt.bat @@ -0,0 +1,12 @@ +@ echo off +:: Script for starting the Salt CLI +:: Accepts all parameters that Salt CLI accepts + +:: Define Variables +Set SaltDir=%~dp0 +Set SaltDir=%SaltDir:~0,-1% +Set Python=%SaltDir%\bin\python.exe +Set Script=%SaltDir%\bin\Scripts\salt + +:: Launch Script +"%Python%" -E -s "%Script%" %* diff --git a/pkg/windows/clean_env.bat b/pkg/windows/clean_env.bat index fb6e63a661a..7c5e4978024 100644 --- a/pkg/windows/clean_env.bat +++ b/pkg/windows/clean_env.bat @@ -16,9 +16,10 @@ if %errorLevel%==0 ( ) echo. +:CheckPython2 if exist "\Python27" goto RemovePython2 -if exist "\Program Files\Python35" goto RemovePython3 -goto eof + +goto CheckPython3 :RemovePython2 rem Uninstall Python 2.7 @@ -47,25 +48,30 @@ goto eof goto eof +:CheckPython3 +if exist "\Python35" goto RemovePython3 + +goto eof + :RemovePython3 echo %0 :: Uninstalling Python 3 ... echo --------------------------------------------------------------------- :: 64 bit if exist "%LOCALAPPDATA%\Package Cache\{b94f45d6-8461-440c-aa4d-bf197b2c2499}" ( echo %0 :: - 3.5.3 64bit - "%LOCALAPPDATA%\Package Cache\{b94f45d6-8461-440c-aa4d-bf197b2c2499}\python-3.5.3-amd64.exe" /uninstall + "%LOCALAPPDATA%\Package Cache\{b94f45d6-8461-440c-aa4d-bf197b2c2499}\python-3.5.3-amd64.exe" /uninstall /passive ) :: 32 bit if exist "%LOCALAPPDATA%\Package Cache\{a10037e1-4247-47c9-935b-c5ca049d0299}" ( echo %0 :: - 3.5.3 32bit - "%LOCALAPPDATA%\Package Cache\{a10037e1-4247-47c9-935b-c5ca049d0299}\python-3.5.3" /uninstall + "%LOCALAPPDATA%\Package Cache\{a10037e1-4247-47c9-935b-c5ca049d0299}\python-3.5.3" /uninstall /passive ) rem wipe the Python directory - echo %0 :: Removing the C:\Program Files\Python35 Directory ... + echo %0 :: Removing the C:\Python35 Directory ... echo --------------------------------------------------------------------- - rd /s /q "C:\Program Files\Python35" + rd /s /q "C:\Python35" if %errorLevel%==0 ( echo Successful ) else ( diff --git a/pkg/windows/installer/Salt-Minion-Setup.nsi b/pkg/windows/installer/Salt-Minion-Setup.nsi index 39f55852c01..a8efca21018 100644 --- a/pkg/windows/installer/Salt-Minion-Setup.nsi +++ b/pkg/windows/installer/Salt-Minion-Setup.nsi @@ -44,7 +44,7 @@ ${StrStrAdv} !define CPUARCH "x86" !endif -; Part of the Trim function for Strings +# Part of the Trim function for Strings !define Trim "!insertmacro Trim" !macro Trim ResultVar String Push "${String}" @@ -61,27 +61,27 @@ ${StrStrAdv} !define MUI_UNICON "salt.ico" !define MUI_WELCOMEFINISHPAGE_BITMAP "panel.bmp" -; Welcome page +# Welcome page !insertmacro MUI_PAGE_WELCOME -; License page +# License page !insertmacro MUI_PAGE_LICENSE "LICENSE.txt" -; Configure Minion page +# Configure Minion page Page custom pageMinionConfig pageMinionConfig_Leave -; Instfiles page +# Instfiles page !insertmacro MUI_PAGE_INSTFILES -; Finish page (Customized) +# Finish page (Customized) !define MUI_PAGE_CUSTOMFUNCTION_SHOW pageFinish_Show !define MUI_PAGE_CUSTOMFUNCTION_LEAVE pageFinish_Leave !insertmacro MUI_PAGE_FINISH -; Uninstaller pages +# Uninstaller pages !insertmacro MUI_UNPAGE_INSTFILES -; Language files +# Language files !insertmacro MUI_LANGUAGE "English" @@ -201,8 +201,8 @@ ShowInstDetails show ShowUnInstDetails show -; Check and install Visual C++ redist packages -; See http://blogs.msdn.com/b/astebner/archive/2009/01/29/9384143.aspx for more info +# Check and install Visual C++ redist packages +# See http://blogs.msdn.com/b/astebner/archive/2009/01/29/9384143.aspx for more info Section -Prerequisites Var /GLOBAL VcRedistName @@ -211,12 +211,12 @@ Section -Prerequisites Var /Global CheckVcRedist StrCpy $CheckVcRedist "False" - ; Visual C++ 2015 redist packages + # Visual C++ 2015 redist packages !define PY3_VC_REDIST_NAME "VC_Redist_2015" !define PY3_VC_REDIST_X64_GUID "{50A2BC33-C9CD-3BF1-A8FF-53C10A0B183C}" !define PY3_VC_REDIST_X86_GUID "{BBF2AC74-720C-3CB3-8291-5E34039232FA}" - ; Visual C++ 2008 SP1 MFC Security Update redist packages + # Visual C++ 2008 SP1 MFC Security Update redist packages !define PY2_VC_REDIST_NAME "VC_Redist_2008_SP1_MFC" !define PY2_VC_REDIST_X64_GUID "{5FCE6D76-F5DC-37AB-B2B8-22AB8CEDB1D4}" !define PY2_VC_REDIST_X86_GUID "{9BE518E6-ECC6-35A9-88E4-87755C07200F}" @@ -239,7 +239,7 @@ Section -Prerequisites StrCpy $VcRedistGuid ${PY2_VC_REDIST_X86_GUID} ${EndIf} - ; VCRedist 2008 only needed on Windows Server 2008R2/Windows 7 and below + # VCRedist 2008 only needed on Windows Server 2008R2/Windows 7 and below ${If} ${AtMostWin2008R2} StrCpy $CheckVcRedist "True" ${EndIf} @@ -255,20 +255,41 @@ Section -Prerequisites "$VcRedistName is currently not installed. Would you like to install?" \ /SD IDYES IDNO endVcRedist - ClearErrors - ; The Correct version of VCRedist is copied over by "build_pkg.bat" + # The Correct version of VCRedist is copied over by "build_pkg.bat" SetOutPath "$INSTDIR\" File "..\prereqs\vcredist.exe" - ; /passive used by 2015 installer - ; /qb! used by 2008 installer - ; It just ignores the unrecognized switches... - ExecWait "$INSTDIR\vcredist.exe /qb! /passive" - IfErrors 0 endVcRedist + # If an output variable is specified ($0 in the case below), + # ExecWait sets the variable with the exit code (and only sets the + # error flag if an error occurs; if an error occurs, the contents + # of the user variable are undefined). + # http://nsis.sourceforge.net/Reference/ExecWait + # /passive used by 2015 installer + # /qb! used by 2008 installer + # It just ignores the unrecognized switches... + ClearErrors + ExecWait '"$INSTDIR\vcredist.exe" /qb! /passive /norestart' $0 + IfErrors 0 CheckVcRedistErrorCode MessageBox MB_OK \ "$VcRedistName failed to install. Try installing the package manually." \ /SD IDOK + Goto endVcRedist + + CheckVcRedistErrorCode: + # Check for Reboot Error Code (3010) + ${If} $0 == 3010 + MessageBox MB_OK \ + "$VcRedistName installed but requires a restart to complete." \ + /SD IDOK + + # Check for any other errors + ${ElseIfNot} $0 == 0 + MessageBox MB_OK \ + "$VcRedistName failed with ErrorCode: $0. Try installing the package manually." \ + /SD IDOK + ${EndIf} endVcRedist: + ${EndIf} ${EndIf} @@ -294,12 +315,12 @@ Function .onInit Call parseCommandLineSwitches - ; Check for existing installation + # Check for existing installation ReadRegStr $R0 HKLM \ "Software\Microsoft\Windows\CurrentVersion\Uninstall\${PRODUCT_NAME}" \ "UninstallString" StrCmp $R0 "" checkOther - ; Found existing installation, prompt to uninstall + # Found existing installation, prompt to uninstall MessageBox MB_OKCANCEL|MB_ICONEXCLAMATION \ "${PRODUCT_NAME} is already installed.$\n$\n\ Click `OK` to remove the existing installation." \ @@ -307,12 +328,12 @@ Function .onInit Abort checkOther: - ; Check for existing installation of full salt + # Check for existing installation of full salt ReadRegStr $R0 HKLM \ "Software\Microsoft\Windows\CurrentVersion\Uninstall\${PRODUCT_NAME_OTHER}" \ "UninstallString" StrCmp $R0 "" skipUninstall - ; Found existing installation, prompt to uninstall + # Found existing installation, prompt to uninstall MessageBox MB_OKCANCEL|MB_ICONEXCLAMATION \ "${PRODUCT_NAME_OTHER} is already installed.$\n$\n\ Click `OK` to remove the existing installation." \ @@ -321,22 +342,22 @@ Function .onInit uninst: - ; Get current Silent status + # Get current Silent status StrCpy $R0 0 ${If} ${Silent} StrCpy $R0 1 ${EndIf} - ; Turn on Silent mode + # Turn on Silent mode SetSilent silent - ; Don't remove all directories + # Don't remove all directories StrCpy $DeleteInstallDir 0 - ; Uninstall silently + # Uninstall silently Call uninstallSalt - ; Set it back to Normal mode, if that's what it was before + # Set it back to Normal mode, if that's what it was before ${If} $R0 == 0 SetSilent normal ${EndIf} @@ -350,7 +371,7 @@ Section -Post WriteUninstaller "$INSTDIR\uninst.exe" - ; Uninstall Registry Entries + # Uninstall Registry Entries WriteRegStr ${PRODUCT_UNINST_ROOT_KEY} "${PRODUCT_UNINST_KEY}" \ "DisplayName" "$(^Name)" WriteRegStr ${PRODUCT_UNINST_ROOT_KEY} "${PRODUCT_UNINST_KEY}" \ @@ -366,26 +387,25 @@ Section -Post WriteRegStr HKLM "SYSTEM\CurrentControlSet\services\salt-minion" \ "DependOnService" "nsi" - ; Set the estimated size + # Set the estimated size ${GetSize} "$INSTDIR\bin" "/S=OK" $0 $1 $2 IntFmt $0 "0x%08X" $0 WriteRegDWORD ${PRODUCT_UNINST_ROOT_KEY} "${PRODUCT_UNINST_KEY}" \ "EstimatedSize" "$0" - ; Commandline Registry Entries + # Commandline Registry Entries WriteRegStr HKLM "${PRODUCT_CALL_REGKEY}" "" "$INSTDIR\salt-call.bat" WriteRegStr HKLM "${PRODUCT_CALL_REGKEY}" "Path" "$INSTDIR\bin\" WriteRegStr HKLM "${PRODUCT_MINION_REGKEY}" "" "$INSTDIR\salt-minion.bat" WriteRegStr HKLM "${PRODUCT_MINION_REGKEY}" "Path" "$INSTDIR\bin\" - ; Register the Salt-Minion Service - nsExec::Exec "nssm.exe install salt-minion $INSTDIR\bin\python.exe $INSTDIR\bin\Scripts\salt-minion -c $INSTDIR\conf -l quiet" - nsExec::Exec "nssm.exe set salt-minion AppEnvironmentExtra PYTHONHOME=" + # Register the Salt-Minion Service + nsExec::Exec "nssm.exe install salt-minion $INSTDIR\bin\python.exe -E -s $INSTDIR\bin\Scripts\salt-minion -c $INSTDIR\conf -l quiet" nsExec::Exec "nssm.exe set salt-minion Description Salt Minion from saltstack.com" nsExec::Exec "nssm.exe set salt-minion Start SERVICE_AUTO_START" nsExec::Exec "nssm.exe set salt-minion AppNoConsole 1" - - RMDir /R "$INSTDIR\var\cache\salt" ; removing cache from old version + nsExec::Exec "nssm.exe set salt-minion AppStopMethodConsole 24000" + nsExec::Exec "nssm.exe set salt-minion AppStopMethodWindow 2000" Call updateMinionConfig @@ -399,12 +419,12 @@ SectionEnd Function .onInstSuccess - ; If StartMinionDelayed is 1, then set the service to start delayed + # If StartMinionDelayed is 1, then set the service to start delayed ${If} $StartMinionDelayed == 1 nsExec::Exec "nssm.exe set salt-minion Start SERVICE_DELAYED_AUTO_START" ${EndIf} - ; If start-minion is 1, then start the service + # If start-minion is 1, then start the service ${If} $StartMinion == 1 nsExec::Exec 'net start salt-minion' ${EndIf} @@ -414,10 +434,11 @@ FunctionEnd Function un.onInit - ; Load the parameters + # Load the parameters ${GetParameters} $R0 # Uninstaller: Remove Installation Directory + ClearErrors ${GetOptions} $R0 "/delete-install-dir" $R1 IfErrors delete_install_dir_not_found StrCpy $DeleteInstallDir 1 @@ -435,7 +456,7 @@ Section Uninstall Call un.uninstallSalt - ; Remove C:\salt from the Path + # Remove C:\salt from the Path Push "C:\salt" Call un.RemoveFromPath @@ -445,27 +466,27 @@ SectionEnd !macro uninstallSalt un Function ${un}uninstallSalt - ; Make sure we're in the right directory + # Make sure we're in the right directory ${If} $INSTDIR == "c:\salt\bin\Scripts" StrCpy $INSTDIR "C:\salt" ${EndIf} - ; Stop and Remove salt-minion service + # Stop and Remove salt-minion service nsExec::Exec 'net stop salt-minion' nsExec::Exec 'sc delete salt-minion' - ; Stop and remove the salt-master service + # Stop and remove the salt-master service nsExec::Exec 'net stop salt-master' nsExec::Exec 'sc delete salt-master' - ; Remove files + # Remove files Delete "$INSTDIR\uninst.exe" Delete "$INSTDIR\nssm.exe" Delete "$INSTDIR\salt*" Delete "$INSTDIR\vcredist.exe" RMDir /r "$INSTDIR\bin" - ; Remove Registry entries + # Remove Registry entries DeleteRegKey ${PRODUCT_UNINST_ROOT_KEY} "${PRODUCT_UNINST_KEY}" DeleteRegKey ${PRODUCT_UNINST_ROOT_KEY} "${PRODUCT_UNINST_KEY_OTHER}" DeleteRegKey ${PRODUCT_UNINST_ROOT_KEY} "${PRODUCT_CALL_REGKEY}" @@ -475,17 +496,17 @@ Function ${un}uninstallSalt DeleteRegKey ${PRODUCT_UNINST_ROOT_KEY} "${PRODUCT_MINION_REGKEY}" DeleteRegKey ${PRODUCT_UNINST_ROOT_KEY} "${PRODUCT_RUN_REGKEY}" - ; Automatically close when finished + # Automatically close when finished SetAutoClose true - ; Prompt to remove the Installation directory + # Prompt to remove the Installation directory ${IfNot} $DeleteInstallDir == 1 MessageBox MB_ICONQUESTION|MB_YESNO|MB_DEFBUTTON2 \ "Would you like to completely remove $INSTDIR and all of its contents?" \ /SD IDNO IDNO finished ${EndIf} - ; Make sure you're not removing Program Files + # Make sure you're not removing Program Files ${If} $INSTDIR != 'Program Files' ${AndIf} $INSTDIR != 'Program Files (x86)' RMDir /r "$INSTDIR" @@ -527,7 +548,7 @@ FunctionEnd Function Trim - Exch $R1 ; Original string + Exch $R1 # Original string Push $R2 Loop: @@ -559,36 +580,36 @@ Function Trim FunctionEnd -;------------------------------------------------------------------------------ -; StrStr Function -; - find substring in a string -; -; Usage: -; Push "this is some string" -; Push "some" -; Call StrStr -; Pop $0 ; "some string" -;------------------------------------------------------------------------------ +#------------------------------------------------------------------------------ +# StrStr Function +# - find substring in a string +# +# Usage: +# Push "this is some string" +# Push "some" +# Call StrStr +# Pop $0 ; "some string" +#------------------------------------------------------------------------------ !macro StrStr un Function ${un}StrStr - Exch $R1 ; $R1=substring, stack=[old$R1,string,...] - Exch ; stack=[string,old$R1,...] - Exch $R2 ; $R2=string, stack=[old$R2,old$R1,...] - Push $R3 ; $R3=strlen(substring) - Push $R4 ; $R4=count - Push $R5 ; $R5=tmp - StrLen $R3 $R1 ; Get the length of the Search String - StrCpy $R4 0 ; Set the counter to 0 + Exch $R1 # $R1=substring, stack=[old$R1,string,...] + Exch # stack=[string,old$R1,...] + Exch $R2 # $R2=string, stack=[old$R2,old$R1,...] + Push $R3 # $R3=strlen(substring) + Push $R4 # $R4=count + Push $R5 # $R5=tmp + StrLen $R3 $R1 # Get the length of the Search String + StrCpy $R4 0 # Set the counter to 0 loop: - StrCpy $R5 $R2 $R3 $R4 ; Create a moving window of the string that is - ; the size of the length of the search string - StrCmp $R5 $R1 done ; Is the contents of the window the same as - ; search string, then done - StrCmp $R5 "" done ; Is the window empty, then done - IntOp $R4 $R4 + 1 ; Shift the windows one character - Goto loop ; Repeat + StrCpy $R5 $R2 $R3 $R4 # Create a moving window of the string that is + # the size of the length of the search string + StrCmp $R5 $R1 done # Is the contents of the window the same as + # search string, then done + StrCmp $R5 "" done # Is the window empty, then done + IntOp $R4 $R4 + 1 # Shift the windows one character + Goto loop # Repeat done: StrCpy $R1 $R2 "" $R4 @@ -596,7 +617,7 @@ Function ${un}StrStr Pop $R4 Pop $R3 Pop $R2 - Exch $R1 ; $R1=old$R1, stack=[result,...] + Exch $R1 # $R1=old$R1, stack=[result,...] FunctionEnd !macroend @@ -604,74 +625,74 @@ FunctionEnd !insertmacro StrStr "un." -;------------------------------------------------------------------------------ -; AddToPath Function -; - Adds item to Path for All Users -; - Overcomes NSIS ReadRegStr limitation of 1024 characters by using Native -; Windows Commands -; -; Usage: -; Push "C:\path\to\add" -; Call AddToPath -;------------------------------------------------------------------------------ +#------------------------------------------------------------------------------ +# AddToPath Function +# - Adds item to Path for All Users +# - Overcomes NSIS ReadRegStr limitation of 1024 characters by using Native +# Windows Commands +# +# Usage: +# Push "C:\path\to\add" +# Call AddToPath +#------------------------------------------------------------------------------ !define Environ 'HKLM "SYSTEM\CurrentControlSet\Control\Session Manager\Environment"' Function AddToPath - Exch $0 ; Path to add - Push $1 ; Current Path - Push $2 ; Results of StrStr / Length of Path + Path to Add - Push $3 ; Handle to Reg / Length of Path - Push $4 ; Result of Registry Call + Exch $0 # Path to add + Push $1 # Current Path + Push $2 # Results of StrStr / Length of Path + Path to Add + Push $3 # Handle to Reg / Length of Path + Push $4 # Result of Registry Call - ; Open a handle to the key in the registry, handle in $3, Error in $4 + # Open a handle to the key in the registry, handle in $3, Error in $4 System::Call "advapi32::RegOpenKey(i 0x80000002, t'SYSTEM\CurrentControlSet\Control\Session Manager\Environment', *i.r3) i.r4" - ; Make sure registry handle opened successfully (returned 0) + # Make sure registry handle opened successfully (returned 0) IntCmp $4 0 0 done done - ; Load the contents of path into $1, Error Code into $4, Path length into $2 + # Load the contents of path into $1, Error Code into $4, Path length into $2 System::Call "advapi32::RegQueryValueEx(i $3, t'PATH', i 0, i 0, t.r1, *i ${NSIS_MAX_STRLEN} r2) i.r4" - ; Close the handle to the registry ($3) + # Close the handle to the registry ($3) System::Call "advapi32::RegCloseKey(i $3)" - ; Check for Error Code 234, Path too long for the variable - IntCmp $4 234 0 +4 +4 ; $4 == ERROR_MORE_DATA + # Check for Error Code 234, Path too long for the variable + IntCmp $4 234 0 +4 +4 # $4 == ERROR_MORE_DATA DetailPrint "AddToPath Failed: original length $2 > ${NSIS_MAX_STRLEN}" MessageBox MB_OK \ "You may add C:\salt to the %PATH% for convenience when issuing local salt commands from the command line." \ /SD IDOK Goto done - ; If no error, continue - IntCmp $4 0 +5 ; $4 != NO_ERROR - ; Error 2 means the Key was not found - IntCmp $4 2 +3 ; $4 != ERROR_FILE_NOT_FOUND + # If no error, continue + IntCmp $4 0 +5 # $4 != NO_ERROR + # Error 2 means the Key was not found + IntCmp $4 2 +3 # $4 != ERROR_FILE_NOT_FOUND DetailPrint "AddToPath: unexpected error code $4" Goto done StrCpy $1 "" - ; Check if already in PATH - Push "$1;" ; The string to search - Push "$0;" ; The string to find + # Check if already in PATH + Push "$1;" # The string to search + Push "$0;" # The string to find Call StrStr - Pop $2 ; The result of the search - StrCmp $2 "" 0 done ; String not found, try again with ';' at the end - ; Otherwise, it's already in the path - Push "$1;" ; The string to search - Push "$0\;" ; The string to find + Pop $2 # The result of the search + StrCmp $2 "" 0 done # String not found, try again with ';' at the end + # Otherwise, it's already in the path + Push "$1;" # The string to search + Push "$0\;" # The string to find Call StrStr - Pop $2 ; The result - StrCmp $2 "" 0 done ; String not found, continue (add) - ; Otherwise, it's already in the path + Pop $2 # The result + StrCmp $2 "" 0 done # String not found, continue (add) + # Otherwise, it's already in the path - ; Prevent NSIS string overflow - StrLen $2 $0 ; Length of path to add ($2) - StrLen $3 $1 ; Length of current path ($3) - IntOp $2 $2 + $3 ; Length of current path + path to add ($2) - IntOp $2 $2 + 2 ; Account for the additional ';' - ; $2 = strlen(dir) + strlen(PATH) + sizeof(";") + # Prevent NSIS string overflow + StrLen $2 $0 # Length of path to add ($2) + StrLen $3 $1 # Length of current path ($3) + IntOp $2 $2 + $3 # Length of current path + path to add ($2) + IntOp $2 $2 + 2 # Account for the additional ';' + # $2 = strlen(dir) + strlen(PATH) + sizeof(";") - ; Make sure the new length isn't over the NSIS_MAX_STRLEN + # Make sure the new length isn't over the NSIS_MAX_STRLEN IntCmp $2 ${NSIS_MAX_STRLEN} +4 +4 0 DetailPrint "AddToPath: new length $2 > ${NSIS_MAX_STRLEN}" MessageBox MB_OK \ @@ -679,18 +700,18 @@ Function AddToPath /SD IDOK Goto done - ; Append dir to PATH + # Append dir to PATH DetailPrint "Add to PATH: $0" - StrCpy $2 $1 1 -1 ; Copy the last character of the existing path - StrCmp $2 ";" 0 +2 ; Check for trailing ';' - StrCpy $1 $1 -1 ; remove trailing ';' - StrCmp $1 "" +2 ; Make sure Path is not empty - StrCpy $0 "$1;$0" ; Append new path at the end ($0) + StrCpy $2 $1 1 -1 # Copy the last character of the existing path + StrCmp $2 ";" 0 +2 # Check for trailing ';' + StrCpy $1 $1 -1 # remove trailing ';' + StrCmp $1 "" +2 # Make sure Path is not empty + StrCpy $0 "$1;$0" # Append new path at the end ($0) - ; We can use the NSIS command here. Only 'ReadRegStr' is affected + # We can use the NSIS command here. Only 'ReadRegStr' is affected WriteRegExpandStr ${Environ} "PATH" $0 - ; Broadcast registry change to open programs + # Broadcast registry change to open programs SendMessage ${HWND_BROADCAST} ${WM_WININICHANGE} 0 "STR:Environment" /TIMEOUT=5000 done: @@ -703,16 +724,16 @@ Function AddToPath FunctionEnd -;------------------------------------------------------------------------------ -; RemoveFromPath Function -; - Removes item from Path for All Users -; - Overcomes NSIS ReadRegStr limitation of 1024 characters by using Native -; Windows Commands -; -; Usage: -; Push "C:\path\to\add" -; Call un.RemoveFromPath -;------------------------------------------------------------------------------ +#------------------------------------------------------------------------------ +# RemoveFromPath Function +# - Removes item from Path for All Users +# - Overcomes NSIS ReadRegStr limitation of 1024 characters by using Native +# Windows Commands +# +# Usage: +# Push "C:\path\to\add" +# Call un.RemoveFromPath +#------------------------------------------------------------------------------ Function un.RemoveFromPath Exch $0 @@ -723,59 +744,59 @@ Function un.RemoveFromPath Push $5 Push $6 - ; Open a handle to the key in the registry, handle in $3, Error in $4 + # Open a handle to the key in the registry, handle in $3, Error in $4 System::Call "advapi32::RegOpenKey(i 0x80000002, t'SYSTEM\CurrentControlSet\Control\Session Manager\Environment', *i.r3) i.r4" - ; Make sure registry handle opened successfully (returned 0) + # Make sure registry handle opened successfully (returned 0) IntCmp $4 0 0 done done - ; Load the contents of path into $1, Error Code into $4, Path length into $2 + # Load the contents of path into $1, Error Code into $4, Path length into $2 System::Call "advapi32::RegQueryValueEx(i $3, t'PATH', i 0, i 0, t.r1, *i ${NSIS_MAX_STRLEN} r2) i.r4" - ; Close the handle to the registry ($3) + # Close the handle to the registry ($3) System::Call "advapi32::RegCloseKey(i $3)" - ; Check for Error Code 234, Path too long for the variable - IntCmp $4 234 0 +4 +4 ; $4 == ERROR_MORE_DATA + # Check for Error Code 234, Path too long for the variable + IntCmp $4 234 0 +4 +4 # $4 == ERROR_MORE_DATA DetailPrint "AddToPath: original length $2 > ${NSIS_MAX_STRLEN}" Goto done - ; If no error, continue - IntCmp $4 0 +5 ; $4 != NO_ERROR - ; Error 2 means the Key was not found - IntCmp $4 2 +3 ; $4 != ERROR_FILE_NOT_FOUND + # If no error, continue + IntCmp $4 0 +5 # $4 != NO_ERROR + # Error 2 means the Key was not found + IntCmp $4 2 +3 # $4 != ERROR_FILE_NOT_FOUND DetailPrint "AddToPath: unexpected error code $4" Goto done StrCpy $1 "" - ; Ensure there's a trailing ';' - StrCpy $5 $1 1 -1 ; Copy the last character of the path - StrCmp $5 ";" +2 ; Check for trailing ';', if found continue - StrCpy $1 "$1;" ; ensure trailing ';' + # Ensure there's a trailing ';' + StrCpy $5 $1 1 -1 # Copy the last character of the path + StrCmp $5 ";" +2 # Check for trailing ';', if found continue + StrCpy $1 "$1;" # ensure trailing ';' - ; Check for our directory inside the path - Push $1 ; String to Search - Push "$0;" ; Dir to Find + # Check for our directory inside the path + Push $1 # String to Search + Push "$0;" # Dir to Find Call un.StrStr - Pop $2 ; The results of the search - StrCmp $2 "" done ; If results are empty, we're done, otherwise continue + Pop $2 # The results of the search + StrCmp $2 "" done # If results are empty, we're done, otherwise continue - ; Remove our Directory from the Path + # Remove our Directory from the Path DetailPrint "Remove from PATH: $0" - StrLen $3 "$0;" ; Get the length of our dir ($3) - StrLen $4 $2 ; Get the length of the return from StrStr ($4) - StrCpy $5 $1 -$4 ; $5 is now the part before the path to remove - StrCpy $6 $2 "" $3 ; $6 is now the part after the path to remove - StrCpy $3 "$5$6" ; Combine $5 and $6 + StrLen $3 "$0;" # Get the length of our dir ($3) + StrLen $4 $2 # Get the length of the return from StrStr ($4) + StrCpy $5 $1 -$4 # $5 is now the part before the path to remove + StrCpy $6 $2 "" $3 # $6 is now the part after the path to remove + StrCpy $3 "$5$6" # Combine $5 and $6 - ; Check for Trailing ';' - StrCpy $5 $3 1 -1 ; Load the last character of the string - StrCmp $5 ";" 0 +2 ; Check for ';' - StrCpy $3 $3 -1 ; remove trailing ';' + # Check for Trailing ';' + StrCpy $5 $3 1 -1 # Load the last character of the string + StrCmp $5 ";" 0 +2 # Check for ';' + StrCpy $3 $3 -1 # remove trailing ';' - ; Write the new path to the registry + # Write the new path to the registry WriteRegExpandStr ${Environ} "PATH" $3 - ; Broadcast the change to all open applications + # Broadcast the change to all open applications SendMessage ${HWND_BROADCAST} ${WM_WININICHANGE} 0 "STR:Environment" /TIMEOUT=5000 done: @@ -809,6 +830,7 @@ Function getMinionConfig confFound: FileOpen $0 "$INSTDIR\conf\minion" r + ClearErrors confLoop: FileRead $0 $1 IfErrors EndOfFile @@ -839,68 +861,69 @@ FunctionEnd Function updateMinionConfig ClearErrors - FileOpen $0 "$INSTDIR\conf\minion" "r" ; open target file for reading - GetTempFileName $R0 ; get new temp file name - FileOpen $1 $R0 "w" ; open temp file for writing + FileOpen $0 "$INSTDIR\conf\minion" "r" # open target file for reading + GetTempFileName $R0 # get new temp file name + FileOpen $1 $R0 "w" # open temp file for writing - loop: ; loop through each line - FileRead $0 $2 ; read line from target file - IfErrors done ; end if errors are encountered (end of line) + loop: # loop through each line + FileRead $0 $2 # read line from target file + IfErrors done # end if errors are encountered (end of line) - ${If} $MasterHost_State != "" ; if master is empty - ${AndIf} $MasterHost_State != "salt" ; and if master is not 'salt' - ${StrLoc} $3 $2 "master:" ">" ; where is 'master:' in this line - ${If} $3 == 0 ; is it in the first... - ${OrIf} $3 == 1 ; or second position (account for comments) - StrCpy $2 "master: $MasterHost_State$\r$\n" ; write the master - ${EndIf} ; close if statement - ${EndIf} ; close if statement + ${If} $MasterHost_State != "" # if master is empty + ${AndIf} $MasterHost_State != "salt" # and if master is not 'salt' + ${StrLoc} $3 $2 "master:" ">" # where is 'master:' in this line + ${If} $3 == 0 # is it in the first... + ${OrIf} $3 == 1 # or second position (account for comments) + StrCpy $2 "master: $MasterHost_State$\r$\n" # write the master + ${EndIf} # close if statement + ${EndIf} # close if statement - ${If} $MinionName_State != "" ; if minion is empty - ${AndIf} $MinionName_State != "hostname" ; and if minion is not 'hostname' - ${StrLoc} $3 $2 "id:" ">" ; where is 'id:' in this line - ${If} $3 == 0 ; is it in the first... - ${OrIf} $3 == 1 ; or the second position (account for comments) - StrCpy $2 "id: $MinionName_State$\r$\n" ; change line - ${EndIf} ; close if statement - ${EndIf} ; close if statement + ${If} $MinionName_State != "" # if minion is empty + ${AndIf} $MinionName_State != "hostname" # and if minion is not 'hostname' + ${StrLoc} $3 $2 "id:" ">" # where is 'id:' in this line + ${If} $3 == 0 # is it in the first... + ${OrIf} $3 == 1 # or the second position (account for comments) + StrCpy $2 "id: $MinionName_State$\r$\n" # change line + ${EndIf} # close if statement + ${EndIf} # close if statement - FileWrite $1 $2 ; write changed or unchanged line to temp file + FileWrite $1 $2 # write changed or unchanged line to temp file Goto loop done: - FileClose $0 ; close target file - FileClose $1 ; close temp file - Delete "$INSTDIR\conf\minion" ; delete target file - CopyFiles /SILENT $R0 "$INSTDIR\conf\minion" ; copy temp file to target file - Delete $R0 ; delete temp file + FileClose $0 # close target file + FileClose $1 # close temp file + Delete "$INSTDIR\conf\minion" # delete target file + CopyFiles /SILENT $R0 "$INSTDIR\conf\minion" # copy temp file to target file + Delete $R0 # delete temp file FunctionEnd Function parseCommandLineSwitches - ; Load the parameters + # Load the parameters ${GetParameters} $R0 - ; Check for start-minion switches - ; /start-service is to be deprecated, so we must check for both + # Check for start-minion switches + # /start-service is to be deprecated, so we must check for both ${GetOptions} $R0 "/start-service=" $R1 ${GetOptions} $R0 "/start-minion=" $R2 # Service: Start Salt Minion ${IfNot} $R2 == "" - ; If start-minion was passed something, then set it + # If start-minion was passed something, then set it StrCpy $StartMinion $R2 ${ElseIfNot} $R1 == "" - ; If start-service was passed something, then set StartMinion to that + # If start-service was passed something, then set StartMinion to that StrCpy $StartMinion $R1 ${Else} - ; Otherwise default to 1 + # Otherwise default to 1 StrCpy $StartMinion 1 ${EndIf} # Service: Minion Startup Type Delayed + ClearErrors ${GetOptions} $R0 "/start-minion-delayed" $R1 IfErrors start_minion_delayed_not_found StrCpy $StartMinionDelayed 1 diff --git a/pkg/windows/modules/get-settings.psm1 b/pkg/windows/modules/get-settings.psm1 index 292732cb83b..5c57738fd35 100644 --- a/pkg/windows/modules/get-settings.psm1 +++ b/pkg/windows/modules/get-settings.psm1 @@ -19,9 +19,9 @@ Function Get-Settings { "Python2Dir" = "C:\Python27" "Scripts2Dir" = "C:\Python27\Scripts" "SitePkgs2Dir" = "C:\Python27\Lib\site-packages" - "Python3Dir" = "C:\Program Files\Python35" - "Scripts3Dir" = "C:\Program Files\Python35\Scripts" - "SitePkgs3Dir" = "C:\Program Files\Python35\Lib\site-packages" + "Python3Dir" = "C:\Python35" + "Scripts3Dir" = "C:\Python35\Scripts" + "SitePkgs3Dir" = "C:\Python35\Lib\site-packages" "DownloadDir" = "$env:Temp\DevSalt" } # The script deletes the DownLoadDir (above) for each install. diff --git a/requirements/dev_python27.txt b/requirements/dev_python27.txt index ab7e5ce96c0..c714cec7957 100644 --- a/requirements/dev_python27.txt +++ b/requirements/dev_python27.txt @@ -6,7 +6,6 @@ boto>=2.32.1 boto3>=1.2.1 moto>=0.3.6 SaltPyLint>=v2017.3.6 -GitPython>=0.3 pytest git+https://github.com/eisensheng/pytest-catchlog.git@develop#egg=Pytest-catchlog git+https://github.com/saltstack/pytest-salt.git@master#egg=pytest-salt diff --git a/requirements/dev_python34.txt b/requirements/dev_python34.txt index a2a707d1e90..b42bac072e2 100644 --- a/requirements/dev_python34.txt +++ b/requirements/dev_python34.txt @@ -11,6 +11,5 @@ moto>=0.3.6 # prevent it from being successfully installed (at least on Python 3.4). httpretty SaltPyLint>=v2017.2.29 -GitPython>=0.3 pytest git+https://github.com/saltstack/pytest-salt.git@master#egg=pytest-salt diff --git a/requirements/opt.txt b/requirements/opt.txt index e4c9181db79..8b738158592 100644 --- a/requirements/opt.txt +++ b/requirements/opt.txt @@ -5,3 +5,4 @@ yappi>=0.8.2 --allow-unverified python-neutronclient>2.3.6 python-gnupg cherrypy>=3.2.2 +libnacl diff --git a/salt/__init__.py b/salt/__init__.py index 84e5d84b514..21302c96057 100644 --- a/salt/__init__.py +++ b/salt/__init__.py @@ -7,6 +7,7 @@ Salt package from __future__ import absolute_import import warnings +# future lint: disable=non-unicode-string # All salt related deprecation warnings should be shown once each! warnings.filterwarnings( 'once', # Show once @@ -14,18 +15,19 @@ warnings.filterwarnings( DeprecationWarning, # This filter is for DeprecationWarnings r'^(salt|salt\.(.*))$' # Match module(s) 'salt' and 'salt.' ) +# future lint: enable=non-unicode-string # While we are supporting Python2.6, hide nested with-statements warnings warnings.filterwarnings( - 'ignore', - 'With-statements now directly support multiple context managers', + u'ignore', + u'With-statements now directly support multiple context managers', DeprecationWarning ) # Filter the backports package UserWarning about being re-imported warnings.filterwarnings( - 'ignore', - '^Module backports was already imported from (.*), but (.*) is being added to sys.path$', + u'ignore', + u'^Module backports was already imported from (.*), but (.*) is being added to sys.path$', UserWarning ) @@ -37,7 +39,7 @@ def __define_global_system_encoding_variable__(): # and reset to None encoding = None - if not sys.platform.startswith('win') and sys.stdin is not None: + if not sys.platform.startswith(u'win') and sys.stdin is not None: # On linux we can rely on sys.stdin for the encoding since it # most commonly matches the filesystem encoding. This however # does not apply to windows @@ -63,16 +65,16 @@ def __define_global_system_encoding_variable__(): # the way back to ascii encoding = sys.getdefaultencoding() if not encoding: - if sys.platform.startswith('darwin'): + if sys.platform.startswith(u'darwin'): # Mac OS X uses UTF-8 - encoding = 'utf-8' - elif sys.platform.startswith('win'): + encoding = u'utf-8' + elif sys.platform.startswith(u'win'): # Windows uses a configurable encoding; on Windows, Python uses the name “mbcs” # to refer to whatever the currently configured encoding is. - encoding = 'mbcs' + encoding = u'mbcs' else: # On linux default to ascii as a last resort - encoding = 'ascii' + encoding = u'ascii' # We can't use six.moves.builtins because these builtins get deleted sooner # than expected. See: @@ -83,7 +85,7 @@ def __define_global_system_encoding_variable__(): import builtins # pylint: disable=import-error # Define the detected encoding as a built-in variable for ease of use - setattr(builtins, '__salt_system_encoding__', encoding) + setattr(builtins, u'__salt_system_encoding__', encoding) # This is now garbage collectable del sys diff --git a/salt/_compat.py b/salt/_compat.py index 9b10646ace7..052e68c0504 100644 --- a/salt/_compat.py +++ b/salt/_compat.py @@ -46,7 +46,7 @@ else: if HAS_XML: - if not hasattr(ElementTree, 'ParseError'): + if not hasattr(ElementTree, u'ParseError'): class ParseError(Exception): ''' older versions of ElementTree do not have ParseError @@ -56,7 +56,7 @@ if HAS_XML: ElementTree.ParseError = ParseError -def text_(s, encoding='latin-1', errors='strict'): +def text_(s, encoding=u'latin-1', errors=u'strict'): ''' If ``s`` is an instance of ``binary_type``, return ``s.decode(encoding, errors)``, otherwise return ``s`` @@ -66,7 +66,7 @@ def text_(s, encoding='latin-1', errors='strict'): return s -def bytes_(s, encoding='latin-1', errors='strict'): +def bytes_(s, encoding=u'latin-1', errors=u'strict'): ''' If ``s`` is an instance of ``text_type``, return ``s.encode(encoding, errors)``, otherwise return ``s`` @@ -79,25 +79,25 @@ def bytes_(s, encoding='latin-1', errors='strict'): if PY3: def ascii_native_(s): if isinstance(s, text_type): - s = s.encode('ascii') - return str(s, 'ascii', 'strict') + s = s.encode(u'ascii') + return str(s, u'ascii', u'strict') else: def ascii_native_(s): if isinstance(s, text_type): - s = s.encode('ascii') + s = s.encode(u'ascii') return str(s) ascii_native_.__doc__ = ''' Python 3: If ``s`` is an instance of ``text_type``, return -``s.encode('ascii')``, otherwise return ``str(s, 'ascii', 'strict')`` +``s.encode(u'ascii')``, otherwise return ``str(s, 'ascii', 'strict')`` Python 2: If ``s`` is an instance of ``text_type``, return -``s.encode('ascii')``, otherwise return ``str(s)`` -''' +``s.encode(u'ascii')``, otherwise return ``str(s)`` +''' # future lint: disable=non-unicode-string if PY3: - def native_(s, encoding='latin-1', errors='strict'): + def native_(s, encoding=u'latin-1', errors=u'strict'): ''' If ``s`` is an instance of ``text_type``, return ``s``, otherwise return ``str(s, encoding, errors)`` @@ -106,7 +106,7 @@ if PY3: return s return str(s, encoding, errors) else: - def native_(s, encoding='latin-1', errors='strict'): + def native_(s, encoding=u'latin-1', errors=u'strict'): ''' If ``s`` is an instance of ``text_type``, return ``s.encode(encoding, errors)``, otherwise return ``str(s)`` @@ -121,7 +121,7 @@ return ``str(s, encoding, errors)`` Python 2: If ``s`` is an instance of ``text_type``, return ``s.encode(encoding, errors)``, otherwise return ``str(s)`` -''' +''' # future lint: disable=non-unicode-string def string_io(data=None): # cStringIO can't handle unicode diff --git a/salt/acl/__init__.py b/salt/acl/__init__.py index dcaddcb43aa..00efe2e40c0 100644 --- a/salt/acl/__init__.py +++ b/salt/acl/__init__.py @@ -10,8 +10,13 @@ found by reading the salt documentation: # Import python libraries from __future__ import absolute_import + +# Import salt libs import salt.utils +# Import 3rd-party libs +from salt.ext import six + class PublisherACL(object): ''' @@ -30,7 +35,7 @@ class PublisherACL(object): def cmd_is_blacklisted(self, cmd): # If this is a regular command, it is a single function - if isinstance(cmd, str): + if isinstance(cmd, six.string_types): cmd = [cmd] for fun in cmd: if not salt.utils.check_whitelist_blacklist(fun, blacklist=self.blacklist.get('modules', [])): diff --git a/salt/auth/__init__.py b/salt/auth/__init__.py index 563597aac5f..38e3c3362ed 100644 --- a/salt/auth/__init__.py +++ b/salt/auth/__init__.py @@ -17,9 +17,7 @@ from __future__ import absolute_import # Import python libs from __future__ import print_function -import os import collections -import hashlib import time import logging import random @@ -31,7 +29,10 @@ import salt.config import salt.loader import salt.transport.client import salt.utils +import salt.utils.args +import salt.utils.files import salt.utils.minions +import salt.utils.versions import salt.payload log = logging.getLogger(__name__) @@ -55,6 +56,7 @@ class LoadAuth(object): self.max_fail = 1.0 self.serial = salt.payload.Serial(opts) self.auth = salt.loader.auth(opts) + self.tokens = salt.loader.eauth_tokens(opts) self.ckminions = ckminions or salt.utils.minions.CkMinions(opts) def load_name(self, load): @@ -68,7 +70,7 @@ class LoadAuth(object): if fstr not in self.auth: return '' try: - pname_arg = salt.utils.arg_lookup(self.auth[fstr])['args'][0] + pname_arg = salt.utils.args.arg_lookup(self.auth[fstr])['args'][0] return load[pname_arg] except IndexError: return '' @@ -199,13 +201,6 @@ class LoadAuth(object): ''' if not self.authenticate_eauth(load): return {} - fstr = '{0}.auth'.format(load['eauth']) - hash_type = getattr(hashlib, self.opts.get('hash_type', 'md5')) - tok = str(hash_type(os.urandom(512)).hexdigest()) - t_path = os.path.join(self.opts['token_dir'], tok) - while os.path.isfile(t_path): - tok = str(hash_type(os.urandom(512)).hexdigest()) - t_path = os.path.join(self.opts['token_dir'], tok) if self._allow_custom_expire(load): token_expire = load.pop('token_expire', self.opts['token_expire']) @@ -216,38 +211,27 @@ class LoadAuth(object): tdata = {'start': time.time(), 'expire': time.time() + token_expire, 'name': self.load_name(load), - 'eauth': load['eauth'], - 'token': tok} + 'eauth': load['eauth']} if self.opts['keep_acl_in_token']: acl_ret = self.__get_acl(load) tdata['auth_list'] = acl_ret - if 'groups' in load: - tdata['groups'] = load['groups'] + groups = self.get_groups(load) + if groups: + tdata['groups'] = groups - try: - with salt.utils.fopen(t_path, 'w+b') as fp_: - fp_.write(self.serial.dumps(tdata)) - except (IOError, OSError): - log.warning('Authentication failure: can not write token file "{0}".'.format(t_path)) - return {} - return tdata + return self.tokens["{0}.mk_token".format(self.opts['eauth_tokens'])](self.opts, tdata) def get_tok(self, tok): ''' Return the name associated with the token, or False if the token is not valid ''' - t_path = os.path.join(self.opts['token_dir'], tok) - if not os.path.isfile(t_path): - return {} - try: - with salt.utils.fopen(t_path, 'rb') as fp_: - tdata = self.serial.loads(fp_.read()) - except (IOError, OSError): - log.warning('Authentication failure: can not read token file "{0}".'.format(t_path)) + tdata = self.tokens["{0}.get_token".format(self.opts['eauth_tokens'])](self.opts, tok) + if not tdata: return {} + rm_tok = False if 'expire' not in tdata: # invalid token, delete it! @@ -255,13 +239,22 @@ class LoadAuth(object): if tdata.get('expire', '0') < time.time(): rm_tok = True if rm_tok: - try: - os.remove(t_path) - return {} - except (IOError, OSError): - pass + self.rm_token(tok) + return tdata + def list_tokens(self): + ''' + List all tokens in eauth_tokn storage. + ''' + return self.tokens["{0}.list_tokens".format(self.opts['eauth_tokens'])](self.opts) + + def rm_token(self, tok): + ''' + Remove the given token from token storage. + ''' + self.tokens["{0}.rm_token".format(self.opts['eauth_tokens'])](self.opts, tok) + def authenticate_token(self, load): ''' Authenticate a user by the token specified in load. @@ -301,29 +294,31 @@ class LoadAuth(object): def authenticate_key(self, load, key): ''' Authenticate a user by the key passed in load. - Return the effective user id (name) if it's differ from the specified one (for sudo). - If the effective user id is the same as passed one return True on success or False on + Return the effective user id (name) if it's different from the specified one (for sudo). + If the effective user id is the same as the passed one, return True on success or False on failure. ''' - auth_key = load.pop('key') - if not auth_key: - log.warning('Authentication failure of type "user" occurred.') + error_msg = 'Authentication failure of type "user" occurred.' + auth_key = load.pop('key', None) + if auth_key is None: + log.warning(error_msg) return False + if 'user' in load: auth_user = AuthUser(load['user']) if auth_user.is_sudo(): # If someone sudos check to make sure there is no ACL's around their username if auth_key != key[self.opts.get('user', 'root')]: - log.warning('Authentication failure of type "user" occurred.') + log.warning(error_msg) return False return auth_user.sudo_name() elif load['user'] == self.opts.get('user', 'root') or load['user'] == 'root': if auth_key != key[self.opts.get('user', 'root')]: - log.warning('Authentication failure of type "user" occurred.') + log.warning(error_msg) return False elif auth_user.is_running_user(): if auth_key != key.get(load['user']): - log.warning('Authentication failure of type "user" occurred.') + log.warning(error_msg) return False elif auth_key == key.get('root'): pass @@ -331,19 +326,19 @@ class LoadAuth(object): if load['user'] in key: # User is authorised, check key and check perms if auth_key != key[load['user']]: - log.warning('Authentication failure of type "user" occurred.') + log.warning(error_msg) return False return load['user'] else: - log.warning('Authentication failure of type "user" occurred.') + log.warning(error_msg) return False else: if auth_key != key[salt.utils.get_user()]: - log.warning('Authentication failure of type "other" occurred.') + log.warning(error_msg) return False return True - def get_auth_list(self, load): + def get_auth_list(self, load, token=None): ''' Retrieve access list for the user specified in load. The list is built by eauth module or from master eauth configuration. @@ -351,30 +346,37 @@ class LoadAuth(object): list if the user has no rights to execute anything on this master and returns non-empty list if user is allowed to execute particular functions. ''' + # Get auth list from token + if token and self.opts['keep_acl_in_token'] and 'auth_list' in token: + return token['auth_list'] # Get acl from eauth module. auth_list = self.__get_acl(load) if auth_list is not None: return auth_list - if load['eauth'] not in self.opts['external_auth']: + eauth = token['eauth'] if token else load['eauth'] + if eauth not in self.opts['external_auth']: # No matching module is allowed in config log.warning('Authorization failure occurred.') return None - name = self.load_name(load) # The username we are attempting to auth with - groups = self.get_groups(load) # The groups this user belongs to - eauth_config = self.opts['external_auth'][load['eauth']] - if groups is None or groups is False: + if token: + name = token['name'] + groups = token.get('groups') + else: + name = self.load_name(load) # The username we are attempting to auth with + groups = self.get_groups(load) # The groups this user belongs to + eauth_config = self.opts['external_auth'][eauth] + if not groups: groups = [] group_perm_keys = [item for item in eauth_config if item.endswith('%')] # The configured auth groups # First we need to know if the user is allowed to proceed via any of their group memberships. group_auth_match = False for group_config in group_perm_keys: - group_config = group_config.rstrip('%') - for group in groups: - if group == group_config: - group_auth_match = True + if group_config.rstrip('%') in groups: + group_auth_match = True + break # If a group_auth_match is set it means only that we have a # user which matches at least one or more of the groups defined # in the configuration file. @@ -414,12 +416,77 @@ class LoadAuth(object): return auth_list + def check_authentication(self, load, auth_type, key=None, show_username=False): + ''' + .. versionadded:: Oxygen + + Go through various checks to see if the token/eauth/user can be authenticated. + + Returns a dictionary containing the following keys: + + - auth_list + - username + - error + + If an error is encountered, return immediately with the relevant error dictionary + as authentication has failed. Otherwise, return the username and valid auth_list. + ''' + auth_list = [] + username = load.get('username', 'UNKNOWN') + ret = {'auth_list': auth_list, + 'username': username, + 'error': {}} + + # Authenticate + if auth_type == 'token': + token = self.authenticate_token(load) + if not token: + ret['error'] = {'name': 'TokenAuthenticationError', + 'message': 'Authentication failure of type "token" occurred.'} + return ret + + # Update username for token + username = token['name'] + ret['username'] = username + auth_list = self.get_auth_list(load, token=token) + elif auth_type == 'eauth': + if not self.authenticate_eauth(load): + ret['error'] = {'name': 'EauthAuthenticationError', + 'message': 'Authentication failure of type "eauth" occurred for ' + 'user {0}.'.format(username)} + return ret + + auth_list = self.get_auth_list(load) + elif auth_type == 'user': + if not self.authenticate_key(load, key): + if show_username: + msg = 'Authentication failure of type "user" occurred for user {0}.'.format(username) + else: + msg = 'Authentication failure of type "user" occurred' + ret['error'] = {'name': 'UserAuthenticationError', 'message': msg} + return ret + else: + ret['error'] = {'name': 'SaltInvocationError', + 'message': 'Authentication type not supported.'} + return ret + + # Authentication checks passed + ret['auth_list'] = auth_list + return ret + class Authorize(object): ''' The authorization engine used by EAUTH ''' def __init__(self, opts, load, loadauth=None): + salt.utils.versions.warn_until( + 'Neon', + 'The \'Authorize\' class has been deprecated. Please use the ' + '\'LoadAuth\', \'Reslover\', or \'AuthUser\' classes instead. ' + 'Support for the \'Authorze\' class will be removed in Salt ' + '{version}.' + ) self.opts = salt.config.master_config(opts['conf_file']) self.load = load self.ckminions = salt.utils.minions.CkMinions(opts) @@ -552,6 +619,15 @@ class Authorize(object): load.get('arg', None), load.get('tgt', None), load.get('tgt_type', 'glob')) + + # Handle possible return of dict data structure from any_auth call to + # avoid a stacktrace. As mentioned in PR #43181, this entire class is + # dead code and is marked for removal in Salt Neon. But until then, we + # should handle the dict return, which is an error and should return + # False until this class is removed. + if isinstance(good, dict): + return False + if not good: # Accept find_job so the CLI will function cleanly if load.get('fun', '') != 'saltutil.find_job': @@ -564,7 +640,7 @@ class Authorize(object): authorization Note: this will check that the user has at least one right that will let - him execute "load", this does not deal with conflicting rules + the user execute "load", this does not deal with conflicting rules ''' adata = self.auth_data @@ -636,7 +712,7 @@ class Resolver(object): 'not available').format(eauth)) return ret - args = salt.utils.arg_lookup(self.auth[fstr]) + args = salt.utils.args.arg_lookup(self.auth[fstr]) for arg in args['args']: if arg in self.opts: ret[arg] = self.opts[arg] @@ -666,14 +742,12 @@ class Resolver(object): tdata = self._send_token_request(load) if 'token' not in tdata: return tdata - oldmask = os.umask(0o177) try: - with salt.utils.fopen(self.opts['token_file'], 'w+') as fp_: - fp_.write(tdata['token']) + with salt.utils.files.set_umask(0o177): + with salt.utils.files.fopen(self.opts['token_file'], 'w+') as fp_: + fp_.write(tdata['token']) except (IOError, OSError): pass - finally: - os.umask(oldmask) return tdata def mk_token(self, load): diff --git a/salt/auth/django.py b/salt/auth/django.py index 726c550b8d4..2b77b877232 100644 --- a/salt/auth/django.py +++ b/salt/auth/django.py @@ -55,7 +55,7 @@ import sys # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six # pylint: disable=import-error try: import django diff --git a/salt/auth/file.py b/salt/auth/file.py index afc56f33171..1abaece4ca5 100644 --- a/salt/auth/file.py +++ b/salt/auth/file.py @@ -101,7 +101,8 @@ import logging import os # Import salt utils -import salt.utils +import salt.utils.files +import salt.utils.versions log = logging.getLogger(__name__) @@ -158,7 +159,7 @@ def _text(username, password, **kwargs): username_field = kwargs['username_field']-1 password_field = kwargs['password_field']-1 - with salt.utils.fopen(filename, 'r') as pwfile: + with salt.utils.files.fopen(filename, 'r') as pwfile: for line in pwfile.readlines(): fields = line.strip().split(field_separator) @@ -199,7 +200,7 @@ def _htpasswd(username, password, **kwargs): pwfile = HtpasswdFile(kwargs['filename']) # passlib below version 1.6 uses 'verify' function instead of 'check_password' - if salt.utils.version_cmp(kwargs['passlib_version'], '1.6') < 0: + if salt.utils.versions.version_cmp(kwargs['passlib_version'], '1.6') < 0: return pwfile.verify(username, password) else: return pwfile.check_password(username, password) @@ -221,7 +222,7 @@ def _htdigest(username, password, **kwargs): pwfile = HtdigestFile(kwargs['filename']) # passlib below version 1.6 uses 'verify' function instead of 'check_password' - if salt.utils.version_cmp(kwargs['passlib_version'], '1.6') < 0: + if salt.utils.versions.version_cmp(kwargs['passlib_version'], '1.6') < 0: return pwfile.verify(username, realm, password) else: return pwfile.check_password(username, realm, password) diff --git a/salt/auth/ldap.py b/salt/auth/ldap.py index 396c1d00a2e..6d82a740135 100644 --- a/salt/auth/ldap.py +++ b/salt/auth/ldap.py @@ -8,7 +8,7 @@ Provide authentication using simple LDAP binds # Import python libs from __future__ import absolute_import import logging -import salt.ext.six as six +from salt.ext import six # Import salt libs from salt.exceptions import CommandExecutionError, SaltInvocationError @@ -280,8 +280,14 @@ def auth(username, password): ''' Simple LDAP auth ''' - if _bind(username, password, anonymous=_config('auth_by_group_membership_only', mandatory=False) and - _config('anonymous', mandatory=False)): + #If bind credentials are configured, use them instead of user's + if _config('binddn', mandatory=False) and _config('bindpw', mandatory=False): + bind = _bind_for_search(anonymous=_config('anonymous', mandatory=False)) + else: + bind = _bind(username, password, anonymous=_config('auth_by_group_membership_only', mandatory=False) and + _config('anonymous', mandatory=False)) + + if bind: log.debug('LDAP authentication successful') return True else: @@ -306,8 +312,9 @@ def groups(username, **kwargs): ''' group_list = [] - bind = _bind(username, kwargs['password'], - anonymous=_config('anonymous', mandatory=False)) + # Perform un-authenticated bind to determine group membership + bind = _bind_for_search(anonymous=_config('anonymous', mandatory=False)) + if bind: log.debug('ldap bind to determine group membership succeeded!') @@ -371,7 +378,7 @@ def groups(username, **kwargs): search_results = bind.search_s(search_base, ldap.SCOPE_SUBTREE, search_string, - [_config('accountattributename'), 'cn']) + [_config('accountattributename'), 'cn', _config('groupattribute')]) for _, entry in search_results: if username in entry[_config('accountattributename')]: group_list.append(entry['cn'][0]) @@ -381,7 +388,11 @@ def groups(username, **kwargs): group_list.append(group.split(',')[0].split('=')[-1]) log.debug('User {0} is a member of groups: {1}'.format(username, group_list)) - if not auth(username, kwargs['password']): + # Only test user auth on first call for job. + # 'show_jid' only exists on first payload so we can use that for the conditional. + if 'show_jid' in kwargs and not _bind(username, kwargs.get('password'), + anonymous=_config('auth_by_group_membership_only', mandatory=False) and + _config('anonymous', mandatory=False)): log.error('LDAP username and password do not match') return [] else: diff --git a/salt/auth/pam.py b/salt/auth/pam.py index 861cb99eed5..8387e910f88 100644 --- a/salt/auth/pam.py +++ b/salt/auth/pam.py @@ -42,11 +42,11 @@ from ctypes import c_void_p, c_uint, c_char_p, c_char, c_int from ctypes.util import find_library # Import Salt libs -from salt.utils import get_group_list +import salt.utils # Can be removed once get_group_list is moved from salt.ext.six.moves import range # pylint: disable=import-error,redefined-builtin # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six LIBPAM = CDLL(find_library('pam')) LIBC = CDLL(find_library('c')) @@ -214,4 +214,4 @@ def groups(username, *args, **kwargs): Uses system groups ''' - return get_group_list(username) + return salt.utils.get_group_list(username) diff --git a/salt/auth/pki.py b/salt/auth/pki.py index 9f7ea37995c..53190112cb2 100644 --- a/salt/auth/pki.py +++ b/salt/auth/pki.py @@ -33,7 +33,7 @@ except ImportError: # pylint: enable=import-error # Import salt libs -import salt.utils +import salt.utils.files log = logging.getLogger(__name__) @@ -74,7 +74,7 @@ def auth(username, password, **kwargs): cert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, pem) cacert_file = __salt__['config.get']('external_auth:pki:ca_file') - with salt.utils.fopen(cacert_file) as f: + with salt.utils.files.fopen(cacert_file) as f: cacert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, f.read()) log.debug('Attempting to authenticate via pki.') diff --git a/salt/auth/stormpath.py b/salt/auth/stormpath.py deleted file mode 100644 index 0a710db46ce..00000000000 --- a/salt/auth/stormpath.py +++ /dev/null @@ -1,71 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Provide authentication using Stormpath. - -This driver requires some extra configuration beyond that which Stormpath -normally requires. - -.. code-block:: yaml - - stormpath: - apiid: 1234567890 - apikey: 1234567890/ABCDEF - # Can use an application ID - application: 6789012345 - # Or can use a directory ID - directory: 3456789012 - # But not both - -.. versionadded:: 2015.8.0 -''' - -from __future__ import absolute_import -import json -import base64 -import urllib -import salt.utils.http -import logging - -log = logging.getLogger(__name__) - - -def auth(username, password): - ''' - Authenticate using a Stormpath directory or application - ''' - apiid = __opts__.get('stormpath', {}).get('apiid', None) - apikey = __opts__.get('stormpath', {}).get('apikey', None) - application = __opts__.get('stormpath', {}).get('application', None) - path = 'https://api.stormpath.com/v1' - - if application is not None: - path = '{0}/applications/{1}/loginAttempts'.format(path, application) - else: - return False - - username = urllib.quote(username) - data = { - 'type': 'basic', - 'value': base64.b64encode('{0}:{1}'.format(username, password)) - } - log.debug('{0}:{1}'.format(username, password)) - log.debug(path) - log.debug(data) - log.debug(json.dumps(data)) - - result = salt.utils.http.query( - path, - method='POST', - username=apiid, - password=apikey, - data=json.dumps(data), - header_dict={'Content-type': 'application/json;charset=UTF-8'}, - decode=False, - status=True, - opts=__opts__, - ) - log.debug(result) - if result.get('status', 403) == 200: - return True - - return False diff --git a/salt/beacons/__init__.py b/salt/beacons/__init__.py index 860b47e5a39..54bea7aa96f 100644 --- a/salt/beacons/__init__.py +++ b/salt/beacons/__init__.py @@ -37,8 +37,9 @@ class Beacon(object): .. code_block:: yaml beacons: inotify: - - /etc/fstab: {} - - /var/cache/foo: {} + - files: + - /etc/fstab: {} + - /var/cache/foo: {} ''' ret = [] b_config = copy.deepcopy(config) @@ -54,13 +55,11 @@ class Beacon(object): current_beacon_config = {} list(map(current_beacon_config.update, config[mod])) elif isinstance(config[mod], dict): - raise CommandExecutionError( - 'Beacon configuration should be a list instead of a dictionary.' - ) + current_beacon_config = config[mod] if 'enabled' in current_beacon_config: if not current_beacon_config['enabled']: - log.trace('Beacon {0} disabled'.format(mod)) + log.trace('Beacon %s disabled', mod) continue else: # remove 'enabled' item before processing the beacon @@ -69,18 +68,19 @@ class Beacon(object): else: self._remove_list_item(config[mod], 'enabled') - log.trace('Beacon processing: {0}'.format(mod)) + log.trace('Beacon processing: %s', mod) fun_str = '{0}.beacon'.format(mod) + validate_str = '{0}.validate'.format(mod) if fun_str in self.beacons: runonce = self._determine_beacon_config(current_beacon_config, 'run_once') interval = self._determine_beacon_config(current_beacon_config, 'interval') if interval: b_config = self._trim_config(b_config, mod, 'interval') if not self._process_interval(mod, interval): - log.trace('Skipping beacon {0}. Interval not reached.'.format(mod)) + log.trace('Skipping beacon %s. Interval not reached.', mod) continue if self._determine_beacon_config(current_beacon_config, 'disable_during_state_run'): - log.trace('Evaluting if beacon {0} should be skipped due to a state run.'.format(mod)) + log.trace('Evaluting if beacon %s should be skipped due to a state run.', mod) b_config = self._trim_config(b_config, mod, 'disable_during_state_run') is_running = False running_jobs = salt.utils.minion.running(self.opts) @@ -90,13 +90,24 @@ class Beacon(object): if is_running: close_str = '{0}.close'.format(mod) if close_str in self.beacons: - log.info('Closing beacon {0}. State run in progress.'.format(mod)) + log.info('Closing beacon %s. State run in progress.', mod) self.beacons[close_str](b_config[mod]) else: - log.info('Skipping beacon {0}. State run in progress.'.format(mod)) + log.info('Skipping beacon %s. State run in progress.', mod) continue # Update __grains__ on the beacon self.beacons[fun_str].__globals__['__grains__'] = grains + + # Run the validate function if it's available, + # otherwise there is a warning about it being missing + if validate_str in self.beacons: + valid, vcomment = self.beacons[validate_str](b_config[mod]) + + if not valid: + log.info('Beacon %s configuration invalid, ' + 'not running.\n%s', mod, vcomment) + continue + # Fire the beacon! raw = self.beacons[fun_str](b_config[mod]) for data in raw: @@ -109,7 +120,7 @@ class Beacon(object): if runonce: self.disable_beacon(mod) else: - log.warning('Unable to process beacon {0}'.format(mod)) + log.warning('Unable to process beacon %s', mod) return ret def _trim_config(self, b_config, mod, key): @@ -138,19 +149,19 @@ class Beacon(object): Process beacons with intervals Return True if a beacon should be run on this loop ''' - log.trace('Processing interval {0} for beacon mod {1}'.format(interval, mod)) + log.trace('Processing interval %s for beacon mod %s', interval, mod) loop_interval = self.opts['loop_interval'] if mod in self.interval_map: log.trace('Processing interval in map') counter = self.interval_map[mod] - log.trace('Interval counter: {0}'.format(counter)) + log.trace('Interval counter: %s', counter) if counter * loop_interval >= interval: self.interval_map[mod] = 1 return True else: self.interval_map[mod] += 1 else: - log.trace('Interval process inserting mod: {0}'.format(mod)) + log.trace('Interval process inserting mod: %s', mod) self.interval_map[mod] = 1 return False @@ -194,13 +205,50 @@ class Beacon(object): ''' # Fire the complete event back along with the list of beacons evt = salt.utils.event.get_event('minion', opts=self.opts) - b_conf = self.functions['config.merge']('beacons') - self.opts['beacons'].update(b_conf) evt.fire_event({'complete': True, 'beacons': self.opts['beacons']}, tag='/salt/minion/minion_beacons_list_complete') return True + def list_available_beacons(self): + ''' + List the available beacons + ''' + _beacons = ['{0}'.format(_beacon.replace('.beacon', '')) + for _beacon in self.beacons if '.beacon' in _beacon] + + # Fire the complete event back along with the list of beacons + evt = salt.utils.event.get_event('minion', opts=self.opts) + evt.fire_event({'complete': True, 'beacons': _beacons}, + tag='/salt/minion/minion_beacons_list_available_complete') + + return True + + def validate_beacon(self, name, beacon_data): + ''' + Return available beacon functions + ''' + validate_str = '{}.validate'.format(name) + # Run the validate function if it's available, + # otherwise there is a warning about it being missing + if validate_str in self.beacons: + if 'enabled' in beacon_data: + del beacon_data['enabled'] + valid, vcomment = self.beacons[validate_str](beacon_data) + else: + log.info('Beacon %s does not have a validate' + ' function, skipping validation.', name) + valid = True + + # Fire the complete event back along with the list of beacons + evt = salt.utils.event.get_event('minion', opts=self.opts) + evt.fire_event({'complete': True, + 'vcomment': vcomment, + 'valid': valid}, + tag='/salt/minion/minion_beacon_validation_complete') + + return True + def add_beacon(self, name, beacon_data): ''' Add a beacon item @@ -211,9 +259,9 @@ class Beacon(object): if name in self.opts['beacons']: log.info('Updating settings for beacon ' - 'item: {0}'.format(name)) + 'item: %s', name) else: - log.info('Added new beacon item {0}'.format(name)) + log.info('Added new beacon item %s', name) self.opts['beacons'].update(data) # Fire the complete event back along with updated list of beacons @@ -232,7 +280,7 @@ class Beacon(object): data[name] = beacon_data log.info('Updating settings for beacon ' - 'item: {0}'.format(name)) + 'item: %s', name) self.opts['beacons'].update(data) # Fire the complete event back along with updated list of beacons @@ -248,7 +296,7 @@ class Beacon(object): ''' if name in self.opts['beacons']: - log.info('Deleting beacon item {0}'.format(name)) + log.info('Deleting beacon item %s', name) del self.opts['beacons'][name] # Fire the complete event back along with updated list of beacons diff --git a/salt/beacons/adb.py b/salt/beacons/adb.py index 003f2d6206d..1fc1c2a3b65 100644 --- a/salt/beacons/adb.py +++ b/salt/beacons/adb.py @@ -10,7 +10,8 @@ from __future__ import absolute_import import logging # Salt libs -import salt.utils +import salt.utils.path +from salt.ext.six.moves import map log = logging.getLogger(__name__) @@ -21,32 +22,41 @@ last_state_extra = {'value': False, 'no_devices': False} def __virtual__(): - which_result = salt.utils.which('adb') + which_result = salt.utils.path.which('adb') if which_result is None: return False else: return __virtualname__ -def __validate__(config): +def validate(config): ''' Validate the beacon configuration ''' # Configuration for adb beacon should be a dictionary with states array - if not isinstance(config, dict): - log.info('Configuration for adb beacon must be a dict.') - return False, ('Configuration for adb beacon must be a dict.') - elif 'states' not in config.keys(): + if not isinstance(config, list): + log.info('Configuration for adb beacon must be a list.') + return False, ('Configuration for adb beacon must be a list.') + + _config = {} + list(map(_config.update, config)) + + if 'states' not in _config: log.info('Configuration for adb beacon must include a states array.') return False, ('Configuration for adb beacon must include a states array.') else: - states = ['offline', 'bootloader', 'device', 'host', 'recovery', 'no permissions', - 'sideload', 'unauthorized', 'unknown', 'missing'] - if any(s not in states for s in config['states']): - log.info('Need a one of the following adb ' - 'states: {0}'.format(', '.join(states))) - return False, ('Need a one of the following adb ' - 'states: {0}'.format(', '.join(states))) + if not isinstance(_config['states'], list): + log.info('Configuration for adb beacon must include a states array.') + return False, ('Configuration for adb beacon must include a states array.') + else: + states = ['offline', 'bootloader', 'device', 'host', + 'recovery', 'no permissions', + 'sideload', 'unauthorized', 'unknown', 'missing'] + if any(s not in states for s in _config['states']): + log.info('Need a one of the following adb ' + 'states: {0}'.format(', '.join(states))) + return False, ('Need a one of the following adb ' + 'states: {0}'.format(', '.join(states))) return True, 'Valid beacon configuration' @@ -74,11 +84,10 @@ def beacon(config): log.trace('adb beacon starting') ret = [] - _validate = __validate__(config) - if not _validate[0]: - return ret + _config = {} + list(map(_config.update, config)) - out = __salt__['cmd.run']('adb devices', runas=config.get('user', None)) + out = __salt__['cmd.run']('adb devices', runas=_config.get('user', None)) lines = out.split('\n')[1:] last_state_devices = list(last_state.keys()) @@ -90,21 +99,21 @@ def beacon(config): found_devices.append(device) if device not in last_state_devices or \ ('state' in last_state[device] and last_state[device]['state'] != state): - if state in config['states']: + if state in _config['states']: ret.append({'device': device, 'state': state, 'tag': state}) last_state[device] = {'state': state} - if 'battery_low' in config: + if 'battery_low' in _config: val = last_state.get(device, {}) cmd = 'adb -s {0} shell cat /sys/class/power_supply/*/capacity'.format(device) - battery_levels = __salt__['cmd.run'](cmd, runas=config.get('user', None)).split('\n') + battery_levels = __salt__['cmd.run'](cmd, runas=_config.get('user', None)).split('\n') for l in battery_levels: battery_level = int(l) if 0 < battery_level < 100: if 'battery' not in val or battery_level != val['battery']: - if ('battery' not in val or val['battery'] > config['battery_low']) and \ - battery_level <= config['battery_low']: + if ('battery' not in val or val['battery'] > _config['battery_low']) and \ + battery_level <= _config['battery_low']: ret.append({'device': device, 'battery_level': battery_level, 'tag': 'battery_low'}) if device not in last_state: @@ -118,13 +127,13 @@ def beacon(config): # Find missing devices and remove them / send an event for device in last_state_devices: if device not in found_devices: - if 'missing' in config['states']: + if 'missing' in _config['states']: ret.append({'device': device, 'state': 'missing', 'tag': 'missing'}) del last_state[device] # Maybe send an event if we don't have any devices - if 'no_devices_event' in config and config['no_devices_event'] is True: + if 'no_devices_event' in _config and _config['no_devices_event'] is True: if len(found_devices) == 0 and not last_state_extra['no_devices']: ret.append({'tag': 'no_devices'}) diff --git a/salt/beacons/avahi_announce.py b/salt/beacons/avahi_announce.py index ee4d9a7bb75..382660a2f3b 100644 --- a/salt/beacons/avahi_announce.py +++ b/salt/beacons/avahi_announce.py @@ -15,6 +15,7 @@ Dependencies from __future__ import absolute_import import logging import time +from salt.ext.six.moves import map # Import 3rd Party libs try: @@ -54,17 +55,23 @@ def __virtual__(): '\'python-avahi\' dependency is missing.'.format(__virtualname__) -def __validate__(config): +def validate(config): ''' Validate the beacon configuration ''' - if not isinstance(config, dict): - return False, ('Configuration for avahi_announcement ' - 'beacon must be a dictionary') - elif not all(x in list(config.keys()) for x in ('servicetype', 'port', 'txt')): + _config = {} + list(map(_config.update, config)) + + if not isinstance(config, list): + return False, ('Configuration for avahi_announce ' + 'beacon must be a list.') + + elif not all(x in _config for x in ('servicetype', + 'port', + 'txt')): return False, ('Configuration for avahi_announce beacon ' - 'must contain servicetype, port and txt items') - return True, 'Valid beacon configuration' + 'must contain servicetype, port and txt items.') + return True, 'Valid beacon configuration.' def _enforce_txt_record_maxlen(key, value): @@ -138,13 +145,13 @@ def beacon(config): beacons: avahi_announce: - run_once: True - servicetype: _demo._tcp - port: 1234 - txt: - ProdName: grains.productname - SerialNo: grains.serialnumber - Comments: 'this is a test' + - run_once: True + - servicetype: _demo._tcp + - port: 1234 + - txt: + ProdName: grains.productname + SerialNo: grains.serialnumber + Comments: 'this is a test' ''' ret = [] changes = {} @@ -152,30 +159,27 @@ def beacon(config): global LAST_GRAINS - _validate = __validate__(config) - if not _validate[0]: - log.warning('Beacon {0} configuration invalid, ' - 'not adding. {1}'.format(__virtualname__, _validate[1])) - return ret + _config = {} + list(map(_config.update, config)) - if 'servicename' in config: - servicename = config['servicename'] + if 'servicename' in _config: + servicename = _config['servicename'] else: servicename = __grains__['host'] # Check for hostname change if LAST_GRAINS and LAST_GRAINS['host'] != servicename: changes['servicename'] = servicename - if LAST_GRAINS and config.get('reset_on_change', False): + if LAST_GRAINS and _config.get('reset_on_change', False): # Check for IP address change in the case when we reset on change if LAST_GRAINS.get('ipv4', []) != __grains__.get('ipv4', []): changes['ipv4'] = __grains__.get('ipv4', []) if LAST_GRAINS.get('ipv6', []) != __grains__.get('ipv6', []): changes['ipv6'] = __grains__.get('ipv6', []) - for item in config['txt']: - if config['txt'][item].startswith('grains.'): - grain = config['txt'][item][7:] + for item in _config['txt']: + if _config['txt'][item].startswith('grains.'): + grain = _config['txt'][item][7:] grain_index = None square_bracket = grain.find('[') if square_bracket != -1 and grain[-1] == ']': @@ -192,7 +196,7 @@ def beacon(config): if LAST_GRAINS and (LAST_GRAINS.get(grain, '') != __grains__.get(grain, '')): changes[str('txt.' + item)] = txt[item] else: - txt[item] = _enforce_txt_record_maxlen(item, config['txt'][item]) + txt[item] = _enforce_txt_record_maxlen(item, _config['txt'][item]) if not LAST_GRAINS: changes[str('txt.' + item)] = txt[item] @@ -200,33 +204,33 @@ def beacon(config): if changes: if not LAST_GRAINS: changes['servicename'] = servicename - changes['servicetype'] = config['servicetype'] - changes['port'] = config['port'] + changes['servicetype'] = _config['servicetype'] + changes['port'] = _config['port'] changes['ipv4'] = __grains__.get('ipv4', []) changes['ipv6'] = __grains__.get('ipv6', []) GROUP.AddService(avahi.IF_UNSPEC, avahi.PROTO_UNSPEC, dbus.UInt32(0), - servicename, config['servicetype'], '', '', - dbus.UInt16(config['port']), avahi.dict_to_txt_array(txt)) + servicename, _config['servicetype'], '', '', + dbus.UInt16(_config['port']), avahi.dict_to_txt_array(txt)) GROUP.Commit() - elif config.get('reset_on_change', False) or 'servicename' in changes: + elif _config.get('reset_on_change', False) or 'servicename' in changes: # A change in 'servicename' requires a reset because we can only # directly update TXT records GROUP.Reset() - reset_wait = config.get('reset_wait', 0) + reset_wait = _config.get('reset_wait', 0) if reset_wait > 0: time.sleep(reset_wait) GROUP.AddService(avahi.IF_UNSPEC, avahi.PROTO_UNSPEC, dbus.UInt32(0), - servicename, config['servicetype'], '', '', - dbus.UInt16(config['port']), avahi.dict_to_txt_array(txt)) + servicename, _config['servicetype'], '', '', + dbus.UInt16(_config['port']), avahi.dict_to_txt_array(txt)) GROUP.Commit() else: GROUP.UpdateServiceTxt(avahi.IF_UNSPEC, avahi.PROTO_UNSPEC, dbus.UInt32(0), - servicename, config['servicetype'], '', + servicename, _config['servicetype'], '', avahi.dict_to_txt_array(txt)) ret.append({'tag': 'result', 'changes': changes}) - if config.get('copy_grains', False): + if _config.get('copy_grains', False): LAST_GRAINS = __grains__.copy() else: LAST_GRAINS = __grains__ diff --git a/salt/beacons/bonjour_announce.py b/salt/beacons/bonjour_announce.py index fce5211464f..f225c6211df 100644 --- a/salt/beacons/bonjour_announce.py +++ b/salt/beacons/bonjour_announce.py @@ -9,6 +9,7 @@ import atexit import logging import select import time +from salt.ext.six.moves import map # Import 3rd Party libs try: @@ -47,17 +48,23 @@ def _register_callback(sdRef, flags, errorCode, name, regtype, domain): # pylin log.error('Bonjour registration failed with error code {0}'.format(errorCode)) -def __validate__(config): +def validate(config): ''' Validate the beacon configuration ''' - if not isinstance(config, dict): - return False, ('Configuration for bonjour_announcement ' - 'beacon must be a dictionary') - elif not all(x in list(config.keys()) for x in ('servicetype', 'port', 'txt')): + _config = {} + list(map(_config.update, config)) + + if not isinstance(config, list): + return False, ('Configuration for bonjour_announce ' + 'beacon must be a list.') + + elif not all(x in _config for x in ('servicetype', + 'port', + 'txt')): return False, ('Configuration for bonjour_announce beacon ' - 'must contain servicetype, port and txt items') - return True, 'Valid beacon configuration' + 'must contain servicetype, port and txt items.') + return True, 'Valid beacon configuration.' def _enforce_txt_record_maxlen(key, value): @@ -131,13 +138,13 @@ def beacon(config): beacons: bonjour_announce: - run_once: True - servicetype: _demo._tcp - port: 1234 - txt: - ProdName: grains.productname - SerialNo: grains.serialnumber - Comments: 'this is a test' + - run_once: True + - servicetype: _demo._tcp + - port: 1234 + - txt: + ProdName: grains.productname + SerialNo: grains.serialnumber + Comments: 'this is a test' ''' ret = [] changes = {} @@ -146,30 +153,27 @@ def beacon(config): global LAST_GRAINS global SD_REF - _validate = __validate__(config) - if not _validate[0]: - log.warning('Beacon {0} configuration invalid, ' - 'not adding. {1}'.format(__virtualname__, _validate[1])) - return ret + _config = {} + list(map(_config.update, config)) - if 'servicename' in config: - servicename = config['servicename'] + if 'servicename' in _config: + servicename = _config['servicename'] else: servicename = __grains__['host'] # Check for hostname change if LAST_GRAINS and LAST_GRAINS['host'] != servicename: changes['servicename'] = servicename - if LAST_GRAINS and config.get('reset_on_change', False): + if LAST_GRAINS and _config.get('reset_on_change', False): # Check for IP address change in the case when we reset on change if LAST_GRAINS.get('ipv4', []) != __grains__.get('ipv4', []): changes['ipv4'] = __grains__.get('ipv4', []) if LAST_GRAINS.get('ipv6', []) != __grains__.get('ipv6', []): changes['ipv6'] = __grains__.get('ipv6', []) - for item in config['txt']: - if config['txt'][item].startswith('grains.'): - grain = config['txt'][item][7:] + for item in _config['txt']: + if _config['txt'][item].startswith('grains.'): + grain = _config['txt'][item][7:] grain_index = None square_bracket = grain.find('[') if square_bracket != -1 and grain[-1] == ']': @@ -186,7 +190,7 @@ def beacon(config): if LAST_GRAINS and (LAST_GRAINS.get(grain, '') != __grains__.get(grain, '')): changes[str('txt.' + item)] = txt[item] else: - txt[item] = _enforce_txt_record_maxlen(item, config['txt'][item]) + txt[item] = _enforce_txt_record_maxlen(item, _config['txt'][item]) if not LAST_GRAINS: changes[str('txt.' + item)] = txt[item] @@ -195,32 +199,32 @@ def beacon(config): txt_record = pybonjour.TXTRecord(items=txt) if not LAST_GRAINS: changes['servicename'] = servicename - changes['servicetype'] = config['servicetype'] - changes['port'] = config['port'] + changes['servicetype'] = _config['servicetype'] + changes['port'] = _config['port'] changes['ipv4'] = __grains__.get('ipv4', []) changes['ipv6'] = __grains__.get('ipv6', []) SD_REF = pybonjour.DNSServiceRegister( name=servicename, - regtype=config['servicetype'], - port=config['port'], + regtype=_config['servicetype'], + port=_config['port'], txtRecord=txt_record, callBack=_register_callback) atexit.register(_close_sd_ref) ready = select.select([SD_REF], [], []) if SD_REF in ready[0]: pybonjour.DNSServiceProcessResult(SD_REF) - elif config.get('reset_on_change', False) or 'servicename' in changes: + elif _config.get('reset_on_change', False) or 'servicename' in changes: # A change in 'servicename' requires a reset because we can only # directly update TXT records SD_REF.close() SD_REF = None - reset_wait = config.get('reset_wait', 0) + reset_wait = _config.get('reset_wait', 0) if reset_wait > 0: time.sleep(reset_wait) SD_REF = pybonjour.DNSServiceRegister( name=servicename, - regtype=config['servicetype'], - port=config['port'], + regtype=_config['servicetype'], + port=_config['port'], txtRecord=txt_record, callBack=_register_callback) ready = select.select([SD_REF], [], []) @@ -236,7 +240,7 @@ def beacon(config): ret.append({'tag': 'result', 'changes': changes}) - if config.get('copy_grains', False): + if _config.get('copy_grains', False): LAST_GRAINS = __grains__.copy() else: LAST_GRAINS = __grains__ diff --git a/salt/beacons/btmp.py b/salt/beacons/btmp.py index 2d2c8c93f56..40b50470d88 100644 --- a/salt/beacons/btmp.py +++ b/salt/beacons/btmp.py @@ -5,7 +5,7 @@ Beacon to fire events at failed login of users .. code-block:: yaml beacons: - btmp: {} + btmp: [] ''' # Import python libs @@ -14,7 +14,10 @@ import os import struct # Import Salt Libs -import salt.utils +import salt.utils.files + +# Import 3rd-party libs +from salt.ext import six __virtualname__ = 'btmp' BTMP = '/var/log/btmp' @@ -49,14 +52,14 @@ def _get_loc(): return __context__[LOC_KEY] -def __validate__(config): +def validate(config): ''' Validate the beacon configuration ''' # Configuration for load beacon should be a list of dicts - if not isinstance(config, dict): + if not isinstance(config, list): return False, ('Configuration for btmp beacon must ' - 'be a list of dictionaries.') + 'be a list.') return True, 'Valid beacon configuration' @@ -68,10 +71,10 @@ def beacon(config): .. code-block:: yaml beacons: - btmp: {} + btmp: [] ''' ret = [] - with salt.utils.fopen(BTMP, 'rb') as fp_: + with salt.utils.files.fopen(BTMP, 'rb') as fp_: loc = __context__.get(LOC_KEY, 0) if loc == 0: fp_.seek(0, 2) @@ -88,7 +91,7 @@ def beacon(config): event = {} for ind, field in enumerate(FIELDS): event[field] = pack[ind] - if isinstance(event[field], str): + if isinstance(event[field], six.string_types): event[field] = event[field].strip('\x00') ret.append(event) return ret diff --git a/salt/beacons/diskusage.py b/salt/beacons/diskusage.py index 635f13954b4..99d71ed0a91 100644 --- a/salt/beacons/diskusage.py +++ b/salt/beacons/diskusage.py @@ -31,14 +31,14 @@ def __virtual__(): return __virtualname__ -def __validate__(config): +def validate(config): ''' Validate the beacon configuration ''' # Configuration for diskusage beacon should be a list of dicts - if not isinstance(config, dict): + if not isinstance(config, list): return False, ('Configuration for diskusage beacon ' - 'must be a dictionary.') + 'must be a list.') return True, 'Valid beacon configuration' @@ -86,25 +86,24 @@ def beacon(config): parts = psutil.disk_partitions(all=False) ret = [] for mounts in config: - mount = mounts.keys()[0] + mount = next(iter(mounts)) - try: - _current_usage = psutil.disk_usage(mount) - except OSError: - # Ensure a valid mount point - log.warning('{0} is not a valid mount point, try regex.'.format(mount)) - for part in parts: - if re.match(mount, part.mountpoint): - row = {} - row[part.mountpoint] = mounts[mount] - config.append(row) - continue + for part in parts: + if re.match(mount, part.mountpoint): + _mount = part.mountpoint - current_usage = _current_usage.percent - monitor_usage = mounts[mount] - if '%' in monitor_usage: - monitor_usage = re.sub('%', '', monitor_usage) - monitor_usage = float(monitor_usage) - if current_usage >= monitor_usage: - ret.append({'diskusage': current_usage, 'mount': mount}) + try: + _current_usage = psutil.disk_usage(mount) + except OSError: + log.warning('{0} is not a valid mount point.'.format(mount)) + continue + + current_usage = _current_usage.percent + monitor_usage = mounts[mount] + log.info('current_usage {}'.format(current_usage)) + if '%' in monitor_usage: + monitor_usage = re.sub('%', '', monitor_usage) + monitor_usage = float(monitor_usage) + if current_usage >= monitor_usage: + ret.append({'diskusage': current_usage, 'mount': _mount}) return ret diff --git a/salt/beacons/glxinfo.py b/salt/beacons/glxinfo.py index a1961abcc01..005c4294744 100644 --- a/salt/beacons/glxinfo.py +++ b/salt/beacons/glxinfo.py @@ -10,7 +10,8 @@ from __future__ import absolute_import import logging # Salt libs -import salt.utils +import salt.utils.path +from salt.ext.six.moves import map log = logging.getLogger(__name__) @@ -21,21 +22,25 @@ last_state = {} def __virtual__(): - which_result = salt.utils.which('glxinfo') + which_result = salt.utils.path.which('glxinfo') if which_result is None: return False else: return __virtualname__ -def __validate__(config): +def validate(config): ''' Validate the beacon configuration ''' # Configuration for glxinfo beacon should be a dictionary - if not isinstance(config, dict): - return False, ('Configuration for glxinfo beacon must be a dict.') - if 'user' not in config: + if not isinstance(config, list): + return False, ('Configuration for glxinfo beacon must be a list.') + + _config = {} + list(map(_config.update, config)) + + if 'user' not in _config: return False, ('Configuration for glxinfo beacon must ' 'include a user as glxinfo is not available to root.') return True, 'Valid beacon configuration' @@ -45,27 +50,28 @@ def beacon(config): ''' Emit the status of a connected display to the minion - Mainly this is used to detect when the display fails to connect for whatever reason. + Mainly this is used to detect when the display fails to connect + for whatever reason. .. code-block:: yaml beacons: glxinfo: - user: frank - screen_event: True + - user: frank + - screen_event: True ''' log.trace('glxinfo beacon starting') ret = [] - _validate = __validate__(config) - if not _validate[0]: - return ret + _config = {} + list(map(_config.update, config)) - retcode = __salt__['cmd.retcode']('DISPLAY=:0 glxinfo', runas=config['user'], python_shell=True) + retcode = __salt__['cmd.retcode']('DISPLAY=:0 glxinfo', + runas=_config['user'], python_shell=True) - if 'screen_event' in config and config['screen_event']: + if 'screen_event' in _config and _config['screen_event']: last_value = last_state.get('screen_available', False) screen_available = retcode == 0 if last_value != screen_available or 'screen_available' not in last_state: diff --git a/salt/beacons/haproxy.py b/salt/beacons/haproxy.py index 9873803d29a..269c6e67dd9 100644 --- a/salt/beacons/haproxy.py +++ b/salt/beacons/haproxy.py @@ -9,7 +9,7 @@ Fire an event when over a specified threshold. # Import Python libs from __future__ import absolute_import import logging - +from salt.ext.six.moves import map log = logging.getLogger(__name__) @@ -23,17 +23,39 @@ def __virtual__(): if 'haproxy.get_sessions' in __salt__: return __virtualname__ else: + log.debug('Not loading haproxy beacon') return False -def __validate__(config): +def validate(config): ''' Validate the beacon configuration ''' - if not isinstance(config, dict): - return False, ('Configuration for haproxy beacon must be a dictionary.') - if 'haproxy' not in config: - return False, ('Configuration for haproxy beacon requires a list of backends and servers') + if not isinstance(config, list): + return False, ('Configuration for haproxy beacon must ' + 'be a list.') + else: + _config = {} + list(map(_config.update, config)) + + if 'backends' not in _config: + return False, ('Configuration for haproxy beacon ' + 'requires backends.') + else: + if not isinstance(_config['backends'], dict): + return False, ('Backends for haproxy beacon ' + 'must be a dictionary.') + else: + for backend in _config['backends']: + log.debug('_config {}'.format(_config['backends'][backend])) + if 'servers' not in _config['backends'][backend]: + return False, ('Backends for haproxy beacon ' + 'require servers.') + else: + _servers = _config['backends'][backend]['servers'] + if not isinstance(_servers, list): + return False, ('Servers for haproxy beacon ' + 'must be a list.') return True, 'Valid beacon configuration' @@ -46,22 +68,23 @@ def beacon(config): beacons: haproxy: - - www-backend: - threshold: 45 - servers: - - web1 - - web2 + - backends: + www-backend: + threshold: 45 + servers: + - web1 + - web2 - interval: 120 ''' - log.debug('haproxy beacon starting') ret = [] - _validate = __validate__(config) - if not _validate: - log.debug('haproxy beacon unable to validate') - return ret - for backend in config: - threshold = config[backend]['threshold'] - for server in config[backend]['servers']: + + _config = {} + list(map(_config.update, config)) + + for backend in _config.get('backends', ()): + backend_config = _config['backends'][backend] + threshold = backend_config['threshold'] + for server in backend_config['servers']: scur = __salt__['haproxy.get_sessions'](server, backend) if scur: if int(scur) > int(threshold): @@ -69,6 +92,10 @@ def beacon(config): 'scur': scur, 'threshold': threshold, } - log.debug('Emit because {0} > {1} for {2} in {3}'.format(scur, threshold, server, backend)) + log.debug('Emit because {0} > {1}' + ' for {2} in {3}'.format(scur, + threshold, + server, + backend)) ret.append(_server) return ret diff --git a/salt/beacons/inotify.py b/salt/beacons/inotify.py index 728a89efc72..ee1dfa4f785 100644 --- a/salt/beacons/inotify.py +++ b/salt/beacons/inotify.py @@ -23,6 +23,7 @@ import re # Import salt libs import salt.ext.six +from salt.ext.six.moves import map # Import third party libs try: @@ -79,7 +80,7 @@ def _get_notifier(config): return __context__['inotify.notifier'] -def __validate__(config): +def validate(config): ''' Validate the beacon configuration ''' @@ -105,37 +106,45 @@ def __validate__(config): ] # Configuration for inotify beacon should be a dict of dicts - log.debug('config {0}'.format(config)) - if not isinstance(config, dict): - return False, 'Configuration for inotify beacon must be a dictionary.' + if not isinstance(config, list): + return False, 'Configuration for inotify beacon must be a list.' else: - for config_item in config: - if not isinstance(config[config_item], dict): - return False, ('Configuration for inotify beacon must ' - 'be a dictionary of dictionaries.') - else: - if not any(j in ['mask', 'recurse', 'auto_add'] for j in config[config_item]): + _config = {} + list(map(_config.update, config)) + + if 'files' not in _config: + return False, 'Configuration for inotify beacon must include files.' + else: + for path in _config.get('files'): + + if not isinstance(_config['files'][path], dict): return False, ('Configuration for inotify beacon must ' - 'contain mask, recurse or auto_add items.') + 'be a list of dictionaries.') + else: + if not any(j in ['mask', + 'recurse', + 'auto_add'] for j in _config['files'][path]): + return False, ('Configuration for inotify beacon must ' + 'contain mask, recurse or auto_add items.') - if 'auto_add' in config[config_item]: - if not isinstance(config[config_item]['auto_add'], bool): - return False, ('Configuration for inotify beacon ' - 'auto_add must be boolean.') + if 'auto_add' in _config['files'][path]: + if not isinstance(_config['files'][path]['auto_add'], bool): + return False, ('Configuration for inotify beacon ' + 'auto_add must be boolean.') - if 'recurse' in config[config_item]: - if not isinstance(config[config_item]['recurse'], bool): - return False, ('Configuration for inotify beacon ' - 'recurse must be boolean.') + if 'recurse' in _config['files'][path]: + if not isinstance(_config['files'][path]['recurse'], bool): + return False, ('Configuration for inotify beacon ' + 'recurse must be boolean.') - if 'mask' in config[config_item]: - if not isinstance(config[config_item]['mask'], list): - return False, ('Configuration for inotify beacon ' - 'mask must be list.') - for mask in config[config_item]['mask']: - if mask not in VALID_MASK: - return False, ('Configuration for inotify beacon ' - 'invalid mask option {0}.'.format(mask)) + if 'mask' in _config['files'][path]: + if not isinstance(_config['files'][path]['mask'], list): + return False, ('Configuration for inotify beacon ' + 'mask must be list.') + for mask in _config['files'][path]['mask']: + if mask not in VALID_MASK: + return False, ('Configuration for inotify beacon ' + 'invalid mask option {0}.'.format(mask)) return True, 'Valid beacon configuration' @@ -149,19 +158,20 @@ def beacon(config): beacons: inotify: - /path/to/file/or/dir: - mask: - - open - - create - - close_write - recurse: True - auto_add: True - exclude: - - /path/to/file/or/dir/exclude1 - - /path/to/file/or/dir/exclude2 - - /path/to/file/or/dir/regex[a-m]*$: - regex: True - coalesce: True + - files: + /path/to/file/or/dir: + mask: + - open + - create + - close_write + recurse: True + auto_add: True + exclude: + - /path/to/file/or/dir/exclude1 + - /path/to/file/or/dir/exclude2 + - /path/to/file/or/dir/regex[a-m]*$: + regex: True + - coalesce: True The mask list can contain the following events (the default mask is create, delete, and modify): @@ -203,8 +213,11 @@ def beacon(config): affects all paths that are being watched. This is due to this option being at the Notifier level in pyinotify. ''' + _config = {} + list(map(_config.update, config)) + ret = [] - notifier = _get_notifier(config) + notifier = _get_notifier(_config) wm = notifier._watch_manager # Read in existing events @@ -219,11 +232,13 @@ def beacon(config): # Find the matching path in config path = event.path while path != '/': - if path in config: + if path in _config.get('files', {}): break path = os.path.dirname(path) - excludes = config[path].get('exclude', '') + for path in _config.get('files', {}): + excludes = _config['files'][path].get('exclude', '') + if excludes and isinstance(excludes, list): for exclude in excludes: if isinstance(exclude, dict): @@ -257,9 +272,10 @@ def beacon(config): # Update existing watches and add new ones # TODO: make the config handle more options - for path in config: - if isinstance(config[path], dict): - mask = config[path].get('mask', DEFAULT_MASK) + for path in _config.get('files', ()): + + if isinstance(_config['files'][path], dict): + mask = _config['files'][path].get('mask', DEFAULT_MASK) if isinstance(mask, list): r_mask = 0 for sub in mask: @@ -269,8 +285,8 @@ def beacon(config): else: r_mask = mask mask = r_mask - rec = config[path].get('recurse', False) - auto_add = config[path].get('auto_add', False) + rec = _config['files'][path].get('recurse', False) + auto_add = _config['files'][path].get('auto_add', False) else: mask = DEFAULT_MASK rec = False @@ -287,7 +303,7 @@ def beacon(config): if update: wm.update_watch(wd, mask=mask, rec=rec, auto_add=auto_add) elif os.path.exists(path): - excludes = config[path].get('exclude', '') + excludes = _config['files'][path].get('exclude', '') excl = None if isinstance(excludes, list): excl = [] diff --git a/salt/beacons/journald.py b/salt/beacons/journald.py index d5cfdca9b47..9b566d587d8 100644 --- a/salt/beacons/journald.py +++ b/salt/beacons/journald.py @@ -10,6 +10,7 @@ from __future__ import absolute_import import salt.utils import salt.utils.locales import salt.ext.six +from salt.ext.six.moves import map # Import third party libs try: @@ -43,18 +44,21 @@ def _get_journal(): return __context__['systemd.journald'] -def __validate__(config): +def validate(config): ''' Validate the beacon configuration ''' # Configuration for journald beacon should be a list of dicts - if not isinstance(config, dict): - return False + if not isinstance(config, list): + return (False, 'Configuration for journald beacon must be a list.') else: - for item in config: - if not isinstance(config[item], dict): - return False, ('Configuration for journald beacon must ' - 'be a dictionary of dictionaries.') + _config = {} + list(map(_config.update, config)) + + for name in _config.get('services', {}): + if not isinstance(_config['services'][name], dict): + return False, ('Services configuration for journald beacon ' + 'must be a list of dictionaries.') return True, 'Valid beacon configuration' @@ -69,25 +73,31 @@ def beacon(config): beacons: journald: - sshd: - SYSLOG_IDENTIFIER: sshd - PRIORITY: 6 + - services: + sshd: + SYSLOG_IDENTIFIER: sshd + PRIORITY: 6 ''' ret = [] journal = _get_journal() + + _config = {} + list(map(_config.update, config)) + while True: cur = journal.get_next() if not cur: break - for name in config: + + for name in _config.get('services', {}): n_flag = 0 - for key in config[name]: + for key in _config['services'][name]: if isinstance(key, salt.ext.six.string_types): key = salt.utils.locales.sdecode(key) if key in cur: - if config[name][key] == cur[key]: + if _config['services'][name][key] == cur[key]: n_flag += 1 - if n_flag == len(config[name]): + if n_flag == len(_config['services'][name]): # Match! sub = salt.utils.simple_types_filter(cur) sub.update({'tag': name}) diff --git a/salt/beacons/load.py b/salt/beacons/load.py index a3cba14fcb4..8b57904b848 100644 --- a/salt/beacons/load.py +++ b/salt/beacons/load.py @@ -9,7 +9,8 @@ import logging import os # Import Salt libs -import salt.utils +import salt.utils.platform +from salt.ext.six.moves import map # Import Py3 compat from salt.ext.six.moves import zip @@ -22,13 +23,13 @@ LAST_STATUS = {} def __virtual__(): - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): return False else: return __virtualname__ -def __validate__(config): +def validate(config): ''' Validate the beacon configuration ''' @@ -37,25 +38,26 @@ def __validate__(config): if not isinstance(config, list): return False, ('Configuration for load beacon must be a list.') else: - for config_item in config: - if not isinstance(config_item, dict): - return False, ('Configuration for load beacon must ' - 'be a list of dictionaries.') - else: - if not all(j in ['1m', '5m', '15m'] for j in config_item.keys()): - return False, ('Configuration for load beacon must ' - 'contain 1m, 5m or 15m items.') + _config = {} + list(map(_config.update, config)) + + if 'averages' not in _config: + return False, ('Averages configuration is required' + ' for load beacon.') + else: + + if not any(j in ['1m', '5m', '15m'] for j + in _config.get('averages', {})): + return False, ('Averages configuration for load beacon ' + 'must contain 1m, 5m or 15m items.') for item in ['1m', '5m', '15m']: - if item not in config_item: - continue - - if not isinstance(config_item[item], list): - return False, ('Configuration for load beacon: ' + if not isinstance(_config['averages'][item], list): + return False, ('Averages configuration for load beacon: ' '1m, 5m and 15m items must be ' 'a list of two items.') else: - if len(config_item[item]) != 2: + if len(_config['averages'][item]) != 2: return False, ('Configuration for load beacon: ' '1m, 5m and 15m items must be ' 'a list of two items.') @@ -82,33 +84,37 @@ def beacon(config): beacons: load: - 1m: - - 0.0 - - 2.0 - 5m: - - 0.0 - - 1.5 - 15m: - - 0.1 - - 1.0 - emitatstartup: True - onchangeonly: False + - averages: + 1m: + - 0.0 + - 2.0 + 5m: + - 0.0 + - 1.5 + 15m: + - 0.1 + - 1.0 + - emitatstartup: True + - onchangeonly: False ''' log.trace('load beacon starting') + _config = {} + list(map(_config.update, config)) + # Default config if not present - if 'emitatstartup' not in config: - config['emitatstartup'] = True - if 'onchangeonly' not in config: - config['onchangeonly'] = False + if 'emitatstartup' not in _config: + _config['emitatstartup'] = True + if 'onchangeonly' not in _config: + _config['onchangeonly'] = False ret = [] avgs = os.getloadavg() avg_keys = ['1m', '5m', '15m'] avg_dict = dict(zip(avg_keys, avgs)) - if config['onchangeonly']: + if _config['onchangeonly']: if not LAST_STATUS: for k in ['1m', '5m', '15m']: LAST_STATUS[k] = avg_dict[k] @@ -120,27 +126,40 @@ def beacon(config): # Check each entry for threshold for k in ['1m', '5m', '15m']: - if k in config: - if config['onchangeonly']: - # Emit if current is more that threshold and old value less that threshold - if float(avg_dict[k]) > float(config[k][1]) and float(LAST_STATUS[k]) < float(config[k][1]): - log.debug('Emit because {0} > {1} and last was {2}'.format(float(avg_dict[k]), float(config[k][1]), float(LAST_STATUS[k]))) + if k in _config.get('averages', {}): + if _config['onchangeonly']: + # Emit if current is more that threshold and old value less + # that threshold + if float(avg_dict[k]) > float(_config['averages'][k][1]) and \ + float(LAST_STATUS[k]) < float(_config['averages'][k][1]): + log.debug('Emit because {0} > {1} and last was ' + '{2}'.format(float(avg_dict[k]), + float(_config['averages'][k][1]), + float(LAST_STATUS[k]))) send_beacon = True break - # Emit if current is less that threshold and old value more that threshold - if float(avg_dict[k]) < float(config[k][0]) and float(LAST_STATUS[k]) > float(config[k][0]): - log.debug('Emit because {0} < {1} and last was {2}'.format(float(avg_dict[k]), float(config[k][0]), float(LAST_STATUS[k]))) + # Emit if current is less that threshold and old value more + # that threshold + if float(avg_dict[k]) < float(_config['averages'][k][0]) and \ + float(LAST_STATUS[k]) > float(_config['averages'][k][0]): + log.debug('Emit because {0} < {1} and last was' + '{2}'.format(float(avg_dict[k]), + float(_config['averages'][k][0]), + float(LAST_STATUS[k]))) send_beacon = True break else: # Emit no matter LAST_STATUS - if float(avg_dict[k]) < float(config[k][0]) or \ - float(avg_dict[k]) > float(config[k][1]): - log.debug('Emit because {0} < {1} or > {2}'.format(float(avg_dict[k]), float(config[k][0]), float(config[k][1]))) + if float(avg_dict[k]) < float(_config['averages'][k][0]) or \ + float(avg_dict[k]) > float(_config['averages'][k][1]): + log.debug('Emit because {0} < {1} or > ' + '{2}'.format(float(avg_dict[k]), + float(_config['averages'][k][0]), + float(_config['averages'][k][1]))) send_beacon = True break - if config['onchangeonly']: + if _config['onchangeonly']: for k in ['1m', '5m', '15m']: LAST_STATUS[k] = avg_dict[k] diff --git a/salt/beacons/log.py b/salt/beacons/log.py index 4d063092f96..5ce4be30290 100644 --- a/salt/beacons/log.py +++ b/salt/beacons/log.py @@ -11,7 +11,9 @@ from __future__ import absolute_import import logging # Import salt libs -import salt.utils +import salt.utils.files +import salt.utils.platform +from salt.ext.six.moves import map try: @@ -34,7 +36,7 @@ log = logging.getLogger(__name__) def __virtual__(): - if not salt.utils.is_windows() and HAS_REGEX: + if not salt.utils.platform.is_windows() and HAS_REGEX: return __virtualname__ return False @@ -47,13 +49,20 @@ def _get_loc(): return __context__[LOC_KEY] -def __validate__(config): +def validate(config): ''' Validate the beacon configuration ''' + _config = {} + list(map(_config.update, config)) + # Configuration for log beacon should be a list of dicts - if not isinstance(config, dict): - return False, ('Configuration for log beacon must be a dictionary.') + if not isinstance(config, list): + return False, ('Configuration for log beacon must be a list.') + + if 'file' not in _config: + return False, ('Configuration for log beacon ' + 'must contain file option.') return True, 'Valid beacon configuration' @@ -66,20 +75,24 @@ def beacon(config): beacons: log: - file: - : - regex: + - file: + - tags: + : + regex: ''' + _config = {} + list(map(_config.update, config)) + ret = [] - if 'file' not in config: + if 'file' not in _config: event = SKEL.copy() event['tag'] = 'global' event['error'] = 'file not defined in config' ret.append(event) return ret - with salt.utils.fopen(config['file'], 'r') as fp_: + with salt.utils.files.fopen(_config['file'], 'r') as fp_: loc = __context__.get(LOC_KEY, 0) if loc == 0: fp_.seek(0, 2) @@ -91,16 +104,17 @@ def beacon(config): fp_.seek(loc) txt = fp_.read() + log.info('txt {}'.format(txt)) d = {} - for tag in config: - if 'regex' not in config[tag]: + for tag in _config.get('tags', {}): + if 'regex' not in _config['tags'][tag]: continue - if len(config[tag]['regex']) < 1: + if len(_config['tags'][tag]['regex']) < 1: continue try: - d[tag] = re.compile(r'{0}'.format(config[tag]['regex'])) - except Exception: + d[tag] = re.compile(r'{0}'.format(_config['tags'][tag]['regex'])) + except Exception as e: event = SKEL.copy() event['tag'] = tag event['error'] = 'bad regex' diff --git a/salt/beacons/memusage.py b/salt/beacons/memusage.py index d779f9c58ef..a64dc0794db 100644 --- a/salt/beacons/memusage.py +++ b/salt/beacons/memusage.py @@ -11,6 +11,7 @@ Beacon to monitor memory usage. from __future__ import absolute_import import logging import re +from salt.ext.six.moves import map # Import Third Party Libs try: @@ -31,14 +32,22 @@ def __virtual__(): return __virtualname__ -def __validate__(config): +def validate(config): ''' Validate the beacon configuration ''' # Configuration for memusage beacon should be a list of dicts - if not isinstance(config, dict): + if not isinstance(config, list): return False, ('Configuration for memusage ' - 'beacon must be a dictionary.') + 'beacon must be a list.') + else: + _config = {} + list(map(_config.update, config)) + + if 'percent' not in _config: + return False, ('Configuration for memusage beacon ' + 'requires percent.') + return True, 'Valid beacon configuration' @@ -46,7 +55,8 @@ def beacon(config): ''' Monitor the memory usage of the minion - Specify thresholds for percent used and only emit a beacon if it is exceeded. + Specify thresholds for percent used and only emit a beacon + if it is exceeded. .. code-block:: yaml @@ -55,15 +65,17 @@ def beacon(config): - percent: 63% ''' ret = [] - for memusage in config: - mount = memusage.keys()[0] - _current_usage = psutil.virtual_memory() - current_usage = _current_usage.percent - monitor_usage = memusage[mount] - if '%' in monitor_usage: - monitor_usage = re.sub('%', '', monitor_usage) - monitor_usage = float(monitor_usage) - if current_usage >= monitor_usage: - ret.append({'memusage': current_usage}) + _config = {} + list(map(_config.update, config)) + + _current_usage = psutil.virtual_memory() + + current_usage = _current_usage.percent + monitor_usage = _config['percent'] + if '%' in monitor_usage: + monitor_usage = re.sub('%', '', monitor_usage) + monitor_usage = float(monitor_usage) + if current_usage >= monitor_usage: + ret.append({'memusage': current_usage}) return ret diff --git a/salt/beacons/napalm_beacon.py b/salt/beacons/napalm_beacon.py new file mode 100644 index 00000000000..66502154711 --- /dev/null +++ b/salt/beacons/napalm_beacon.py @@ -0,0 +1,353 @@ +# -*- coding: utf-8 -*- +''' +NAPALM functions +================ + +.. versionadded:: Oxygen + +Watch NAPALM functions and fire events on specific triggers. + +.. note:: + + The ``NAPALM`` beacon only works only when running under + a regular Minion or a Proxy Minion, managed via NAPALM_. + Check the documentation for the + :mod:`NAPALM proxy module `. + + _NAPALM: http://napalm.readthedocs.io/en/latest/index.html + +The configuration accepts a list of Salt functions to be +invoked, and the corresponding output hierarchy that should +be matched against. To invoke a function with certain +arguments, they can be specified using the ``_args`` key, or +``_kwargs`` for more specific key-value arguments. + +The match structure follows the output hierarchy of the NAPALM +functions, under the ``out`` key. + +For example, the following is normal structure returned by the +:mod:`ntp.stats ` execution function: + +.. code-block:: json + + { + "comment": "", + "result": true, + "out": [ + { + "referenceid": ".GPSs.", + "remote": "172.17.17.1", + "synchronized": true, + "reachability": 377, + "offset": 0.461, + "when": "860", + "delay": 143.606, + "hostpoll": 1024, + "stratum": 1, + "jitter": 0.027, + "type": "-" + }, + { + "referenceid": ".INIT.", + "remote": "172.17.17.2", + "synchronized": false, + "reachability": 0, + "offset": 0.0, + "when": "-", + "delay": 0.0, + "hostpoll": 1024, + "stratum": 16, + "jitter": 4000.0, + "type": "-" + } + ] + } + +In order to fire events when the synchronization is lost with +one of the NTP peers, e.g., ``172.17.17.2``, we can match it explicitly as: + +.. code-block:: yaml + + ntp.stats: + remote: 172.17.17.2 + synchronized: false + +There is one single nesting level, as the output of ``ntp.stats`` is +just a list of dictionaries, and this beacon will compare each dictionary +from the list with the structure examplified above. + +.. note:: + + When we want to match on any element at a certain level, we can + configure ``*`` to match anything. + +Considering a more complex structure consisting on multiple nested levels, +e.g., the output of the :mod:`bgp.neighbors ` +execution function, to check when any neighbor from the ``global`` +routing table is down, the match structure would have the format: + +.. code-block:: yaml + + bgp.neighbors: + global: + '*': + up: false + +The match structure above will match any BGP neighbor, with +any network (``*`` matches any AS number), under the ``global`` VRF. +In other words, this beacon will push an event on the Salt bus +when there's a BGP neighbor down. + +The right operand can also accept mathematical operations +(i.e., ``<``, ``<=``, ``!=``, ``>``, ``>=`` etc.) when comparing +numerical values. + +Configuration Example: + +.. code-block:: yaml + + beacons: + napalm: + - net.interfaces: + # fire events when any interfaces is down + '*': + is_up: false + - net.interfaces: + # fire events only when the xe-0/0/0 interface is down + 'xe-0/0/0': + is_up: false + - ntp.stats: + # fire when there's any NTP peer unsynchornized + synchronized: false + - ntp.stats: + # fire only when the synchronization + # with with the 172.17.17.2 NTP server is lost + _args: + - 172.17.17.2 + synchronized: false + - ntp.stats: + # fire only when there's a NTP peer with + # synchronization stratum > 5 + stratum: '> 5' + +Event structure example: + +.. code-block:: json + + salt/beacon/edge01.bjm01/napalm/junos/ntp.stats { + "_stamp": "2017-09-05T09:51:09.377202", + "args": [], + "data": { + "comment": "", + "out": [ + { + "delay": 0.0, + "hostpoll": 1024, + "jitter": 4000.0, + "offset": 0.0, + "reachability": 0, + "referenceid": ".INIT.", + "remote": "172.17.17.1", + "stratum": 16, + "synchronized": false, + "type": "-", + "when": "-" + } + ], + "result": true + }, + "fun": "ntp.stats", + "id": "edge01.bjm01", + "kwargs": {}, + "match": { + "stratum": "> 5" + } + } + +The event examplified above has been fired when the device +identified by the Minion id ``edge01.bjm01`` has been synchronized +with a NTP server at a stratum level greater than 5. +''' +from __future__ import absolute_import + +# Import Python std lib +import re +import logging + +# Import Salt modules +from salt.ext import six +import salt.utils.napalm + +log = logging.getLogger(__name__) +_numeric_regex = re.compile(r'^(<|>|<=|>=|==|!=)\s*(\d+(\.\d+){0,1})$') +# the numeric regex will match the right operand, e.g '>= 20', '< 100', '!= 20', '< 1000.12' etc. +_numeric_operand = { + '<': '__lt__', + '>': '__gt__', + '>=': '__ge__', + '<=': '__le__', + '==': '__eq__', + '!=': '__ne__', +} # mathematical operand - private method map + + +__virtualname__ = 'napalm' + + +def __virtual__(): + ''' + This beacon can only work when running under a regular or a proxy minion, managed through napalm. + ''' + return salt.utils.napalm.virtual(__opts__, __virtualname__, __file__) + + +def _compare(cur_cmp, cur_struct): + ''' + Compares two objects and return a boolean value + when there's a match. + ''' + if isinstance(cur_cmp, dict) and isinstance(cur_struct, dict): + log.debug('Comparing dict to dict') + for cmp_key, cmp_value in six.iteritems(cur_cmp): + if cmp_key == '*': + # matches any key from the source dictionary + if isinstance(cmp_value, dict): + found = False + for _, cur_struct_val in six.iteritems(cur_struct): + found |= _compare(cmp_value, cur_struct_val) + return found + else: + found = False + if isinstance(cur_struct, (list, tuple)): + for cur_ele in cur_struct: + found |= _compare(cmp_value, cur_ele) + elif isinstance(cur_struct, dict): + for _, cur_ele in six.iteritems(cur_struct): + found |= _compare(cmp_value, cur_ele) + return found + else: + if isinstance(cmp_value, dict): + if cmp_key not in cur_struct: + return False + return _compare(cmp_value, cur_struct[cmp_key]) + if isinstance(cmp_value, list): + found = False + for _, cur_struct_val in six.iteritems(cur_struct): + found |= _compare(cmp_value, cur_struct_val) + return found + else: + return _compare(cmp_value, cur_struct[cmp_key]) + elif isinstance(cur_cmp, (list, tuple)) and isinstance(cur_struct, (list, tuple)): + log.debug('Comparing list to list') + found = False + for cur_cmp_ele in cur_cmp: + for cur_struct_ele in cur_struct: + found |= _compare(cur_cmp_ele, cur_struct_ele) + return found + elif isinstance(cur_cmp, dict) and isinstance(cur_struct, (list, tuple)): + log.debug('Comparing dict to list (of dicts?)') + found = False + for cur_struct_ele in cur_struct: + found |= _compare(cur_cmp, cur_struct_ele) + return found + elif isinstance(cur_cmp, bool) and isinstance(cur_struct, bool): + log.debug('Comparing booleans: %s ? %s', cur_cmp, cur_struct) + return cur_cmp == cur_struct + elif isinstance(cur_cmp, (six.string_types, six.text_type)) and \ + isinstance(cur_struct, (six.string_types, six.text_type)): + log.debug('Comparing strings (and regex?): %s ? %s', cur_cmp, cur_struct) + # Trying literal match + matched = re.match(cur_cmp, cur_struct, re.I) + if matched: + return True + return False + elif isinstance(cur_cmp, (six.integer_types, float)) and \ + isinstance(cur_struct, (six.integer_types, float)): + log.debug('Comparing numeric values: %d ? %d', cur_cmp, cur_struct) + # numeric compare + return cur_cmp == cur_struct + elif isinstance(cur_struct, (six.integer_types, float)) and \ + isinstance(cur_cmp, (six.string_types, six.text_type)): + # Comapring the numerical value agains a presumably mathematical value + log.debug('Comparing a numeric value (%d) with a string (%s)', cur_struct, cur_cmp) + numeric_compare = _numeric_regex.match(cur_cmp) + # determine if the value to compare agains is a mathematical operand + if numeric_compare: + compare_value = numeric_compare.group(2) + return getattr(float(cur_struct), _numeric_operand[numeric_compare.group(1)])(float(compare_value)) + return False + return False + + +def validate(config): + ''' + Validate the beacon configuration. + ''' + # Must be a list of dicts. + if not isinstance(config, list): + return False, 'Configuration for napalm beacon must be a list.' + for mod in config: + fun = mod.keys()[0] + fun_cfg = mod.values()[0] + if not isinstance(fun_cfg, dict): + return False, 'The match structure for the {} execution function output must be a dictionary'.format(fun) + if fun not in __salt__: + return False, 'Execution function {} is not availabe!'.format(fun) + return True, 'Valid configuration for the napal beacon!' + + +def beacon(config): + ''' + Watch napalm function and fire events. + ''' + log.debug('Executing napalm beacon with config:') + log.debug(config) + ret = [] + for mod in config: + if not mod: + continue + event = {} + fun = mod.keys()[0] + fun_cfg = mod.values()[0] + args = fun_cfg.pop('_args', []) + kwargs = fun_cfg.pop('_kwargs', {}) + log.debug('Executing {fun} with {args} and {kwargs}'.format( + fun=fun, + args=args, + kwargs=kwargs + )) + fun_ret = __salt__[fun](*args, **kwargs) + log.debug('Got the reply from the minion:') + log.debug(fun_ret) + if not fun_ret.get('result', False): + log.error('Error whilst executing {}'.format(fun)) + log.error(fun_ret) + continue + fun_ret_out = fun_ret['out'] + log.debug('Comparing to:') + log.debug(fun_cfg) + try: + fun_cmp_result = _compare(fun_cfg, fun_ret_out) + except Exception as err: + log.error(err, exc_info=True) + # catch any exception and continue + # to not jeopardise the execution of the next function in the list + continue + log.debug('Result of comparison: {res}'.format(res=fun_cmp_result)) + if fun_cmp_result: + log.info('Matched {fun} with {cfg}'.format( + fun=fun, + cfg=fun_cfg + )) + event['tag'] = '{os}/{fun}'.format(os=__grains__['os'], fun=fun) + event['fun'] = fun + event['args'] = args + event['kwargs'] = kwargs + event['data'] = fun_ret + event['match'] = fun_cfg + log.debug('Queueing event:') + log.debug(event) + ret.append(event) + log.debug('NAPALM beacon generated the events:') + log.debug(ret) + return ret diff --git a/salt/beacons/network_info.py b/salt/beacons/network_info.py index be3f82a30fb..e2592713393 100644 --- a/salt/beacons/network_info.py +++ b/salt/beacons/network_info.py @@ -16,6 +16,9 @@ try: HAS_PSUTIL = True except ImportError: HAS_PSUTIL = False + +from salt.ext.six.moves import map + # pylint: enable=import-error log = logging.getLogger(__name__) @@ -45,7 +48,7 @@ def __virtual__(): return __virtualname__ -def __validate__(config): +def validate(config): ''' Validate the beacon configuration ''' @@ -57,15 +60,19 @@ def __validate__(config): ] # Configuration for load beacon should be a list of dicts - if not isinstance(config, dict): - return False, ('Configuration for load beacon must be a dictionary.') + if not isinstance(config, list): + return False, ('Configuration for network_info beacon must be a list.') else: - for item in config: - if not isinstance(config[item], dict): - return False, ('Configuration for load beacon must ' - 'be a dictionary of dictionaries.') + + _config = {} + list(map(_config.update, config)) + + for item in _config.get('interfaces', {}): + if not isinstance(_config['interfaces'][item], dict): + return False, ('Configuration for network_info beacon must ' + 'be a list of dictionaries.') else: - if not any(j in VALID_ITEMS for j in config[item]): + if not any(j in VALID_ITEMS for j in _config['interfaces'][item]): return False, ('Invalid configuration item in ' 'Beacon configuration.') return True, 'Valid beacon configuration' @@ -86,16 +93,17 @@ def beacon(config): beacons: network_info: - - eth0: - type: equal - bytes_sent: 100000 - bytes_recv: 100000 - packets_sent: 100000 - packets_recv: 100000 - errin: 100 - errout: 100 - dropin: 100 - dropout: 100 + - interfaces: + eth0: + type: equal + bytes_sent: 100000 + bytes_recv: 100000 + packets_sent: 100000 + packets_recv: 100000 + errin: 100 + errout: 100 + dropin: 100 + dropout: 100 Emit beacon when any values are greater than configured values. @@ -104,46 +112,53 @@ def beacon(config): beacons: network_info: - - eth0: - type: greater - bytes_sent: 100000 - bytes_recv: 100000 - packets_sent: 100000 - packets_recv: 100000 - errin: 100 - errout: 100 - dropin: 100 - dropout: 100 + - interfaces: + eth0: + type: greater + bytes_sent: 100000 + bytes_recv: 100000 + packets_sent: 100000 + packets_recv: 100000 + errin: 100 + errout: 100 + dropin: 100 + dropout: 100 ''' ret = [] + _config = {} + list(map(_config.update, config)) + + log.debug('psutil.net_io_counters {}'.format(psutil.net_io_counters)) + _stats = psutil.net_io_counters(pernic=True) - for interface_config in config: - interface = interface_config.keys()[0] + log.debug('_stats {}'.format(_stats)) + for interface in _config.get('interfaces', {}): if interface in _stats: + interface_config = _config['interfaces'][interface] _if_stats = _stats[interface] _diff = False for attr in __attrs: - if attr in interface_config[interface]: - if 'type' in interface_config[interface] and \ - interface_config[interface]['type'] == 'equal': + if attr in interface_config: + if 'type' in interface_config and \ + interface_config['type'] == 'equal': if getattr(_if_stats, attr, None) == \ - int(interface_config[interface][attr]): + int(interface_config[attr]): _diff = True - elif 'type' in interface_config[interface] and \ - interface_config[interface]['type'] == 'greater': + elif 'type' in interface_config and \ + interface_config['type'] == 'greater': if getattr(_if_stats, attr, None) > \ - int(interface_config[interface][attr]): + int(interface_config[attr]): _diff = True else: log.debug('attr {}'.format(getattr(_if_stats, attr, None))) else: if getattr(_if_stats, attr, None) == \ - int(interface_config[interface][attr]): + int(interface_config[attr]): _diff = True if _diff: ret.append({'interface': interface, diff --git a/salt/beacons/network_settings.py b/salt/beacons/network_settings.py index 78c387b2f29..ad545f19a30 100644 --- a/salt/beacons/network_settings.py +++ b/salt/beacons/network_settings.py @@ -18,6 +18,8 @@ import ast import re import salt.loader import logging +from salt.ext.six.moves import map + log = logging.getLogger(__name__) __virtual_name__ = 'network_settings' @@ -45,22 +47,23 @@ def __virtual__(): return False -def __validate__(config): +def validate(config): ''' Validate the beacon configuration ''' - if not isinstance(config, dict): + if not isinstance(config, list): return False, ('Configuration for network_settings ' - 'beacon must be a dictionary.') + 'beacon must be a list.') else: - for item in config: - if item == 'coalesce': - continue - if not isinstance(config[item], dict): - return False, ('Configuration for network_settings beacon must be a ' - 'dictionary of dictionaries.') + _config = {} + list(map(_config.update, config)) + + for item in _config.get('interfaces', {}): + if not isinstance(_config['interfaces'][item], dict): + return False, ('Configuration for network_settings beacon ' + ' must be a list of dictionaries.') else: - if not all(j in ATTRS for j in config[item]): + if not all(j in ATTRS for j in _config['interfaces'][item]): return False, ('Invalid configuration item in Beacon ' 'configuration.') return True, 'Valid beacon configuration' @@ -75,9 +78,10 @@ def _copy_interfaces_info(interfaces): for interface in interfaces: _interface_attrs_cpy = set() for attr in ATTRS: - attr_dict = Hashabledict() - attr_dict[attr] = repr(interfaces[interface][attr]) - _interface_attrs_cpy.add(attr_dict) + if attr in interfaces[interface]: + attr_dict = Hashabledict() + attr_dict[attr] = repr(interfaces[interface][attr]) + _interface_attrs_cpy.add(attr_dict) ret[interface] = _interface_attrs_cpy return ret @@ -89,8 +93,8 @@ def beacon(config): By default, the beacon will emit when there is a value change on one of the settings on watch. The config also support the onvalue parameter for each - setting, which instruct the beacon to only emit if the setting changed to the - value defined. + setting, which instruct the beacon to only emit if the setting changed to + the value defined. Example Config @@ -98,12 +102,13 @@ def beacon(config): beacons: network_settings: - eth0: - ipaddr: - promiscuity: - onvalue: 1 - eth1: - linkmode: + - interfaces: + - eth0: + ipaddr: + promiscuity: + onvalue: 1 + - eth1: + linkmode: The config above will check for value changes on eth0 ipaddr and eth1 linkmode. It will also emit if the promiscuity value changes to 1. @@ -118,12 +123,16 @@ def beacon(config): beacons: network_settings: - coalesce: True - eth0: - ipaddr: - promiscuity: + - coalesce: True + - interfaces: + - eth0: + ipaddr: + promiscuity: ''' + _config = {} + list(map(_config.update, config)) + ret = [] interfaces = [] expanded_config = {} @@ -137,45 +146,51 @@ def beacon(config): if not LAST_STATS: LAST_STATS = _stats - if 'coalesce' in config and config['coalesce']: + if 'coalesce' in _config and _config['coalesce']: coalesce = True changes = {} + log.debug('_stats {}'.format(_stats)) # Get list of interfaces included in config that are registered in the # system, including interfaces defined by wildcards (eth*, wlan*) - for item in config: - if item == 'coalesce': - continue - if item in _stats: - interfaces.append(item) + for interface in _config.get('interfaces', {}): + if interface in _stats: + interfaces.append(interface) else: # No direct match, try with * wildcard regexp - interface_regexp = item.replace('*', '[0-9]+') + interface_regexp = interface.replace('*', '[0-9]+') for interface in _stats: match = re.search(interface_regexp, interface) if match: interfaces.append(match.group()) - expanded_config[match.group()] = config[item] + expanded_config[match.group()] = config['interfaces'][interface] if expanded_config: config.update(expanded_config) + # config updated so update _config + list(map(_config.update, config)) + + log.debug('interfaces {}'.format(interfaces)) for interface in interfaces: _send_event = False _diff_stats = _stats[interface] - LAST_STATS[interface] _ret_diff = {} + interface_config = _config['interfaces'][interface] + log.debug('_diff_stats {}'.format(_diff_stats)) if _diff_stats: _diff_stats_dict = {} LAST_STATS[interface] = _stats[interface] for item in _diff_stats: _diff_stats_dict.update(item) - for attr in config[interface]: + for attr in interface_config: if attr in _diff_stats_dict: config_value = None - if config[interface][attr] and 'onvalue' in config[interface][attr]: - config_value = config[interface][attr]['onvalue'] + if interface_config[attr] and \ + 'onvalue' in interface_config[attr]: + config_value = interface_config[attr]['onvalue'] new_value = ast.literal_eval(_diff_stats_dict[attr]) if not config_value or config_value == new_value: _send_event = True @@ -185,7 +200,9 @@ def beacon(config): if coalesce: changes[interface] = _ret_diff else: - ret.append({'tag': interface, 'interface': interface, 'change': _ret_diff}) + ret.append({'tag': interface, + 'interface': interface, + 'change': _ret_diff}) if coalesce and changes: grains_info = salt.loader.grains(__opts__, True) diff --git a/salt/beacons/pkg.py b/salt/beacons/pkg.py index 25cbddc9e78..b57b61f6a01 100644 --- a/salt/beacons/pkg.py +++ b/salt/beacons/pkg.py @@ -21,15 +21,25 @@ def __virtual__(): return __virtualname__ if 'pkg.upgrade_available' in __salt__ else False -def __validate__(config): +def validate(config): ''' Validate the beacon configuration ''' # Configuration for pkg beacon should be a list - if not isinstance(config, dict): - return False, ('Configuration for pkg beacon must be a dictionary.') - if 'pkgs' not in config: - return False, ('Configuration for pkg beacon requires list of pkgs.') + if not isinstance(config, list): + return False, ('Configuration for pkg beacon must be a list.') + + # Configuration for pkg beacon should contain pkgs + pkgs_found = False + pkgs_not_list = False + for config_item in config: + if 'pkgs' in config_item: + pkgs_found = True + if isinstance(config_item['pkgs'], list): + pkgs_not_list = True + + if not pkgs_found or not pkgs_not_list: + return False, 'Configuration for pkg beacon requires list of pkgs.' return True, 'Valid beacon configuration' @@ -48,14 +58,16 @@ def beacon(config): - refresh: True ''' ret = [] - _validate = __validate__(config) - if not _validate[0]: - return ret _refresh = False - if 'refresh' in config and config['refresh']: - _refresh = True - for pkg in config['pkgs']: + pkgs = [] + for config_item in config: + if 'pkgs' in config_item: + pkgs += config_item['pkgs'] + if 'refresh' in config and config['refresh']: + _refresh = True + + for pkg in pkgs: _installed = __salt__['pkg.version'](pkg) _latest = __salt__['pkg.latest_version'](pkg, refresh=_refresh) if _installed and _latest: diff --git a/salt/beacons/proxy_example.py b/salt/beacons/proxy_example.py index 329dae86714..1a93db93c85 100644 --- a/salt/beacons/proxy_example.py +++ b/salt/beacons/proxy_example.py @@ -15,6 +15,7 @@ import logging # Import salt libs import salt.utils.http +from salt.ext.six.moves import map # Important: If used with salt-proxy # this is required for the beacon to load!!! @@ -33,12 +34,12 @@ def __virtual__(): return True -def __validate__(config): +def validate(config): ''' Validate the beacon configuration ''' - if not isinstance(config, dict): - return False, ('Configuration for rest_example beacon must be a dictionary.') + if not isinstance(config, list): + return False, ('Configuration for proxy_example beacon must be a list.') return True, 'Valid beacon configuration' @@ -51,7 +52,7 @@ def beacon(config): beacons: proxy_example: - endpoint: beacon + - endpoint: beacon ''' # Important!!! # Although this toy example makes an HTTP call @@ -59,8 +60,11 @@ def beacon(config): # please be advised that doing CPU or IO intensive # operations in this method will cause the beacon loop # to block. + _config = {} + list(map(_config.update, config)) + beacon_url = '{0}{1}'.format(__opts__['proxy']['url'], - config['endpoint']) + _config['endpoint']) ret = salt.utils.http.query(beacon_url, decode_type='json', decode=True) diff --git a/salt/beacons/ps.py b/salt/beacons/ps.py index 38b1a91b04f..b1f18431f49 100644 --- a/salt/beacons/ps.py +++ b/salt/beacons/ps.py @@ -14,6 +14,9 @@ try: HAS_PSUTIL = True except ImportError: HAS_PSUTIL = False + +from salt.ext.six.moves import map + # pylint: enable=import-error log = logging.getLogger(__name__) # pylint: disable=invalid-name @@ -23,17 +26,27 @@ __virtualname__ = 'ps' def __virtual__(): if not HAS_PSUTIL: - return (False, 'cannot load network_info beacon: psutil not available') + return (False, 'cannot load ps beacon: psutil not available') return __virtualname__ -def __validate__(config): +def validate(config): ''' Validate the beacon configuration ''' # Configuration for ps beacon should be a list of dicts - if not isinstance(config, dict): - return False, ('Configuration for ps beacon must be a dictionary.') + if not isinstance(config, list): + return False, ('Configuration for ps beacon must be a list.') + else: + _config = {} + list(map(_config.update, config)) + + if 'processes' not in _config: + return False, ('Configuration for ps beacon requires processes.') + else: + if not isinstance(_config['processes'], dict): + return False, ('Processes for ps beacon must be a dictionary.') + return True, 'Valid beacon configuration' @@ -47,8 +60,9 @@ def beacon(config): beacons: ps: - - salt-master: running - - mysql: stopped + - processes: + salt-master: running + mysql: stopped The config above sets up beacons to check that processes are running or stopped. @@ -60,19 +74,21 @@ def beacon(config): if _name not in procs: procs.append(_name) - for entry in config: - for process in entry: - ret_dict = {} - if entry[process] == 'running': - if process in procs: - ret_dict[process] = 'Running' - ret.append(ret_dict) - elif entry[process] == 'stopped': - if process not in procs: - ret_dict[process] = 'Stopped' - ret.append(ret_dict) - else: - if process not in procs: - ret_dict[process] = False - ret.append(ret_dict) + _config = {} + list(map(_config.update, config)) + + for process in _config.get('processes', {}): + ret_dict = {} + if _config['processes'][process] == 'running': + if process in procs: + ret_dict[process] = 'Running' + ret.append(ret_dict) + elif _config['processes'][process] == 'stopped': + if process not in procs: + ret_dict[process] = 'Stopped' + ret.append(ret_dict) + else: + if process not in procs: + ret_dict[process] = False + ret.append(ret_dict) return ret diff --git a/salt/beacons/salt_proxy.py b/salt/beacons/salt_proxy.py index 8ad6acd5747..f542d8b0dd9 100644 --- a/salt/beacons/salt_proxy.py +++ b/salt/beacons/salt_proxy.py @@ -9,6 +9,7 @@ # Import python libs from __future__ import absolute_import import logging +from salt.ext.six.moves import map log = logging.getLogger(__name__) @@ -20,9 +21,7 @@ def _run_proxy_processes(proxies): aren't running ''' ret = [] - for prox_ in proxies: - # prox_ is a dict - proxy = prox_.keys()[0] + for proxy in proxies: result = {} if not __salt__['salt_proxy.is_running'](proxy)['result']: __salt__['salt_proxy.configure_proxy'](proxy, start=True) @@ -35,7 +34,29 @@ def _run_proxy_processes(proxies): return ret -def beacon(proxies): +def validate(config): + ''' + Validate the beacon configuration + ''' + # Configuration for adb beacon should be a dictionary with states array + if not isinstance(config, list): + log.info('Configuration for salt_proxy beacon must be a list.') + return False, ('Configuration for salt_proxy beacon must be a list.') + + else: + _config = {} + list(map(_config.update, config)) + + if 'proxies' not in _config: + return False, ('Configuration for salt_proxy' + ' beacon requires proxies.') + else: + if not isinstance(_config['proxies'], dict): + return False, ('Proxies for salt_proxy ' + 'beacon must be a dictionary.') + + +def beacon(config): ''' Handle configured proxies @@ -43,9 +64,13 @@ def beacon(proxies): beacons: salt_proxy: - - p8000: {} - - p8001: {} + - proxies: + p8000: {} + p8001: {} ''' log.trace('salt proxy beacon called') - return _run_proxy_processes(proxies) + _config = {} + list(map(_config.update, config)) + + return _run_proxy_processes(_config['proxies']) diff --git a/salt/beacons/sensehat.py b/salt/beacons/sensehat.py index 65f2ff134b5..0784bb2bfd4 100644 --- a/salt/beacons/sensehat.py +++ b/salt/beacons/sensehat.py @@ -11,6 +11,7 @@ of a Raspberry Pi. from __future__ import absolute_import import logging import re +from salt.ext.six.moves import map log = logging.getLogger(__name__) @@ -19,14 +20,22 @@ def __virtual__(): return 'sensehat.get_pressure' in __salt__ -def __validate__(config): +def validate(config): ''' Validate the beacon configuration ''' - # Configuration for sensehat beacon should be a dict - if not isinstance(config, dict): + # Configuration for sensehat beacon should be a list + if not isinstance(config, list): return False, ('Configuration for sensehat beacon ' - 'must be a dictionary.') + 'must be a list.') + else: + _config = {} + list(map(_config.update, config)) + + if 'sensors' not in _config: + return False, ('Configuration for sensehat' + ' beacon requires sensors.') + return True, 'Valid beacon configuration' @@ -48,10 +57,11 @@ def beacon(config): beacons: sensehat: - humidity: 70% - temperature: [20, 40] - temperature_from_pressure: 40 - pressure: 1500 + - sensors: + humidity: 70% + temperature: [20, 40] + temperature_from_pressure: 40 + pressure: 1500 ''' ret = [] min_default = { @@ -60,13 +70,16 @@ def beacon(config): 'temperature': '-273.15' } - for sensor in config: + _config = {} + list(map(_config.update, config)) + + for sensor in _config.get('sensors', {}): sensor_function = 'sensehat.get_{0}'.format(sensor) if sensor_function not in __salt__: log.error('No sensor for meassuring {0}. Skipping.'.format(sensor)) continue - sensor_config = config[sensor] + sensor_config = _config['sensors'][sensor] if isinstance(sensor_config, list): sensor_min = str(sensor_config[0]) sensor_max = str(sensor_config[1]) diff --git a/salt/beacons/service.py b/salt/beacons/service.py index 3e7e28e2547..701f8e4cd0e 100644 --- a/salt/beacons/service.py +++ b/salt/beacons/service.py @@ -8,19 +8,36 @@ from __future__ import absolute_import import os import logging +import time +from salt.ext.six.moves import map log = logging.getLogger(__name__) # pylint: disable=invalid-name LAST_STATUS = {} +__virtualname__ = 'service' -def __validate__(config): + +def validate(config): ''' Validate the beacon configuration ''' # Configuration for service beacon should be a list of dicts - if not isinstance(config, dict): - return False, ('Configuration for service beacon must be a dictionary.') + if not isinstance(config, list): + return False, ('Configuration for service beacon must be a list.') + else: + _config = {} + list(map(_config.update, config)) + + if 'services' not in _config: + return False, ('Configuration for service beacon' + ' requires services.') + else: + for config_item in _config['services']: + if not isinstance(_config['services'][config_item], dict): + return False, ('Configuration for service beacon must ' + 'be a list of dictionaries.') + return True, 'Valid beacon configuration' @@ -34,8 +51,9 @@ def beacon(config): beacons: service: - salt-master: - mysql: + - services: + salt-master: + mysql: The config above sets up beacons to check for the salt-master and mysql services. @@ -46,6 +64,10 @@ def beacon(config): events only when the service status changes. Otherwise, it will fire an event at each beacon interval. The default is False. + `delay`: when `delay` is greater than 0 the beacon will fire events only + after the service status changes, and the delay (in seconds) has passed. + Applicable only when `onchangeonly` is True. The default is 0. + `emitatstartup`: when `emitatstartup` is False the beacon will not fire event when the minion is reload. Applicable only when `onchangeonly` is True. The default is True. @@ -66,7 +88,7 @@ def beacon(config): process). The 'uncleanshutdown' option might not be of much use there, unless the unit file is modified. - Here is an example that will fire an event whenever the state of nginx + Here is an example that will fire an event 30 seconds after the state of nginx changes and report an uncleanshutdown. This example is for Arch, which places nginx's pid file in `/run`. @@ -74,43 +96,68 @@ def beacon(config): beacons: service: - nginx: - onchangeonly: True - uncleanshutdown: /run/nginx.pid + - services: + nginx: + onchangeonly: True + delay: 30 + uncleanshutdown: /run/nginx.pid ''' ret = [] - for service in config: + _config = {} + list(map(_config.update, config)) + + for service in _config.get('services', {}): ret_dict = {} + + service_config = _config['services'][service] + ret_dict[service] = {'running': __salt__['service.status'](service)} ret_dict['service_name'] = service + ret_dict['tag'] = service + currtime = time.time() # If no options is given to the service, we fall back to the defaults # assign a False value to oncleanshutdown and onchangeonly. Those # key:values are then added to the service dictionary. - if 'oncleanshutdown' not in config[service]: - config[service]['oncleanshutdown'] = False - if 'emitatstartup' not in config[service]: - config[service]['emitatstartup'] = True - if 'onchangeonly' not in config[service]: - config[service]['onchangeonly'] = False + if not service_config: + service_config = {} + if 'oncleanshutdown' not in service_config: + service_config['oncleanshutdown'] = False + if 'emitatstartup' not in service_config: + service_config['emitatstartup'] = True + if 'onchangeonly' not in service_config: + service_config['onchangeonly'] = False + if 'delay' not in service_config: + service_config['delay'] = 0 # We only want to report the nature of the shutdown # if the current running status is False # as well as if the config for the beacon asks for it - if 'uncleanshutdown' in config[service] and not ret_dict[service]['running']: - filename = config[service]['uncleanshutdown'] + if 'uncleanshutdown' in service_config and not ret_dict[service]['running']: + filename = service_config['uncleanshutdown'] ret_dict[service]['uncleanshutdown'] = True if os.path.exists(filename) else False - if 'onchangeonly' in config[service] and config[service]['onchangeonly'] is True: + if 'onchangeonly' in service_config and service_config['onchangeonly'] is True: if service not in LAST_STATUS: LAST_STATUS[service] = ret_dict[service] - if not config[service]['emitatstartup']: + if service_config['delay'] > 0: + LAST_STATUS[service]['time'] = currtime + elif not service_config['emitatstartup']: continue else: ret.append(ret_dict) - if LAST_STATUS[service] != ret_dict[service]: + if LAST_STATUS[service]['running'] != ret_dict[service]['running']: LAST_STATUS[service] = ret_dict[service] - ret.append(ret_dict) + if service_config['delay'] > 0: + LAST_STATUS[service]['time'] = currtime + else: + ret.append(ret_dict) + + if 'time' in LAST_STATUS[service]: + elapsedtime = int(round(currtime - LAST_STATUS[service]['time'])) + if elapsedtime > service_config['delay']: + del LAST_STATUS[service]['time'] + ret.append(ret_dict) else: ret.append(ret_dict) diff --git a/salt/beacons/sh.py b/salt/beacons/sh.py index c55462bd5de..7375a1352ab 100644 --- a/salt/beacons/sh.py +++ b/salt/beacons/sh.py @@ -9,6 +9,7 @@ import time # Import salt libs import salt.utils +import salt.utils.path import salt.utils.vt __virtualname__ = 'sh' @@ -21,7 +22,7 @@ def __virtual__(): ''' Only load if strace is installed ''' - return __virtualname__ if salt.utils.which('strace') else False + return __virtualname__ if salt.utils.path.which('strace') else False def _get_shells(): @@ -40,13 +41,13 @@ def _get_shells(): return __context__['sh.shells'] -def __validate__(config): +def validate(config): ''' Validate the beacon configuration ''' # Configuration for sh beacon should be a list of dicts - if not isinstance(config, dict): - return False, ('Configuration for sh beacon must be a dictionary.') + if not isinstance(config, list): + return False, ('Configuration for sh beacon must be a list.') return True, 'Valid beacon configuration' @@ -57,7 +58,7 @@ def beacon(config): .. code-block:: yaml beacons: - sh: {} + sh: [] ''' ret = [] pkey = 'sh.vt' diff --git a/salt/beacons/status.py b/salt/beacons/status.py index bf1f5b3688d..c06793a0398 100644 --- a/salt/beacons/status.py +++ b/salt/beacons/status.py @@ -15,7 +15,7 @@ the minion config: .. code-block:: yaml beacons: - status: {} + status: [] By default, all of the information from the following execution module functions will be returned: @@ -96,19 +96,19 @@ import datetime import salt.exceptions # Import salt libs -import salt.utils +import salt.utils.platform log = logging.getLogger(__name__) __virtualname__ = 'status' -def __validate__(config): +def validate(config): ''' Validate the the config is a dict ''' - if not isinstance(config, dict): - return False, ('Configuration for status beacon must be a dictionary.') + if not isinstance(config, list): + return False, ('Configuration for status beacon must be a list.') return True, 'Valid beacon configuration' @@ -123,7 +123,7 @@ def beacon(config): log.debug(config) ctime = datetime.datetime.utcnow().isoformat() ret = {} - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): return [{ 'tag': ctime, 'data': ret, diff --git a/salt/beacons/telegram_bot_msg.py b/salt/beacons/telegram_bot_msg.py index 22026410711..19ca7ea37b6 100644 --- a/salt/beacons/telegram_bot_msg.py +++ b/salt/beacons/telegram_bot_msg.py @@ -1,11 +1,15 @@ # -*- coding: utf-8 -*- ''' Beacon to emit Telegram messages + +Requires the python-telegram-bot library + ''' # Import Python libs from __future__ import absolute_import import logging +from salt.ext.six.moves import map # Import 3rd Party libs try: @@ -28,20 +32,23 @@ def __virtual__(): return False -def __validate__(config): +def validate(config): ''' Validate the beacon configuration ''' - if not isinstance(config, dict): + if not isinstance(config, list): return False, ('Configuration for telegram_bot_msg ' - 'beacon must be a dictionary.') + 'beacon must be a list.') - if not all(config.get(required_config) + _config = {} + list(map(_config.update, config)) + + if not all(_config.get(required_config) for required_config in ['token', 'accept_from']): return False, ('Not all required configuration for ' 'telegram_bot_msg are set.') - if not isinstance(config.get('accept_from'), list): + if not isinstance(_config.get('accept_from'), list): return False, ('Configuration for telegram_bot_msg, ' 'accept_from must be a list of usernames.') @@ -57,18 +64,22 @@ def beacon(config): beacons: telegram_bot_msg: - token: "" - accept_from: + - token: "" + - accept_from: - "" - interval: 10 + - interval: 10 ''' + + _config = {} + list(map(_config.update, config)) + log.debug('telegram_bot_msg beacon starting') ret = [] output = {} output['msgs'] = [] - bot = telegram.Bot(config['token']) + bot = telegram.Bot(_config['token']) updates = bot.get_updates(limit=100, timeout=0, network_delay=10) log.debug('Num updates: {0}'.format(len(updates))) @@ -83,7 +94,7 @@ def beacon(config): if update.update_id > latest_update_id: latest_update_id = update.update_id - if message.chat.username in config['accept_from']: + if message.chat.username in _config['accept_from']: output['msgs'].append(message.to_dict()) # mark in the server that previous messages are processed diff --git a/salt/beacons/twilio_txt_msg.py b/salt/beacons/twilio_txt_msg.py index 29bb6eab7f6..3034add116f 100644 --- a/salt/beacons/twilio_txt_msg.py +++ b/salt/beacons/twilio_txt_msg.py @@ -6,10 +6,17 @@ Beacon to emit Twilio text messages # Import Python libs from __future__ import absolute_import import logging +from salt.ext.six.moves import map # Import 3rd Party libs try: - from twilio.rest import TwilioRestClient + import twilio + # Grab version, ensure elements are ints + twilio_version = tuple([int(x) for x in twilio.__version_info__]) + if twilio_version > (5, ): + from twilio.rest import Client as TwilioRestClient + else: + from twilio.rest import TwilioRestClient HAS_TWILIO = True except ImportError: HAS_TWILIO = False @@ -26,14 +33,24 @@ def __virtual__(): return False -def __validate__(config): +def validate(config): ''' Validate the beacon configuration ''' # Configuration for twilio_txt_msg beacon should be a list of dicts - if not isinstance(config, dict): + if not isinstance(config, list): return False, ('Configuration for twilio_txt_msg beacon ' - 'must be a dictionary.') + 'must be a list.') + else: + _config = {} + list(map(_config.update, config)) + + if not all(x in _config for x in ('account_sid', + 'auth_token', + 'twilio_number')): + return False, ('Configuration for twilio_txt_msg beacon ' + 'must contain account_sid, auth_token ' + 'and twilio_number items.') return True, 'Valid beacon configuration' @@ -46,20 +63,26 @@ def beacon(config): beacons: twilio_txt_msg: - account_sid: "" - auth_token: "" - twilio_number: "+15555555555" - interval: 10 + - account_sid: "" + - auth_token: "" + - twilio_number: "+15555555555" + - interval: 10 ''' log.trace('twilio_txt_msg beacon starting') + + _config = {} + list(map(_config.update, config)) + ret = [] - if not all([config['account_sid'], config['auth_token'], config['twilio_number']]): + if not all([_config['account_sid'], + _config['auth_token'], + _config['twilio_number']]): return ret output = {} output['texts'] = [] - client = TwilioRestClient(config['account_sid'], config['auth_token']) - messages = client.messages.list(to=config['twilio_number']) + client = TwilioRestClient(_config['account_sid'], _config['auth_token']) + messages = client.messages.list(to=_config['twilio_number']) log.trace('Num messages: {0}'.format(len(messages))) if len(messages) < 1: log.trace('Twilio beacon has no texts') diff --git a/salt/beacons/wtmp.py b/salt/beacons/wtmp.py index 8d4da3fa7d5..c10a335e0cd 100644 --- a/salt/beacons/wtmp.py +++ b/salt/beacons/wtmp.py @@ -5,7 +5,7 @@ Beacon to fire events at login of users as registered in the wtmp file .. code-block:: yaml beacons: - wtmp: {} + wtmp: [] ''' # Import Python libs @@ -14,7 +14,10 @@ import os import struct # Import salt libs -import salt.utils +import salt.utils.files + +# Import 3rd-party libs +from salt.ext import six __virtualname__ = 'wtmp' WTMP = '/var/log/wtmp' @@ -52,13 +55,13 @@ def _get_loc(): return __context__[LOC_KEY] -def __validate__(config): +def validate(config): ''' Validate the beacon configuration ''' # Configuration for wtmp beacon should be a list of dicts - if not isinstance(config, dict): - return False, ('Configuration for wtmp beacon must be a dictionary.') + if not isinstance(config, list): + return False, ('Configuration for wtmp beacon must be a list.') return True, 'Valid beacon configuration' @@ -70,10 +73,10 @@ def beacon(config): .. code-block:: yaml beacons: - wtmp: {} + wtmp: [] ''' ret = [] - with salt.utils.fopen(WTMP, 'rb') as fp_: + with salt.utils.files.fopen(WTMP, 'rb') as fp_: loc = __context__.get(LOC_KEY, 0) if loc == 0: fp_.seek(0, 2) @@ -90,7 +93,7 @@ def beacon(config): event = {} for ind, field in enumerate(FIELDS): event[field] = pack[ind] - if isinstance(event[field], str): + if isinstance(event[field], six.string_types): event[field] = event[field].strip('\x00') ret.append(event) return ret diff --git a/salt/cache/__init__.py b/salt/cache/__init__.py index b8f91103da1..94d7a36f1e5 100644 --- a/salt/cache/__init__.py +++ b/salt/cache/__init__.py @@ -77,6 +77,7 @@ class Cache(object): self.serial = Serial(opts) self._modules = None self._kwargs = kwargs + self._kwargs['cachedir'] = self.cachedir def __lazy_init(self): self._modules = salt.loader.cache(self.opts, self.serial) @@ -223,7 +224,7 @@ class Cache(object): fun = '{0}.flush'.format(self.driver) return self.modules[fun](bank, key=key, **self._kwargs) - def ls(self, bank): + def list(self, bank): ''' Lists entries stored in the specified bank. @@ -239,11 +240,9 @@ class Cache(object): Raises an exception if cache driver detected an error accessing data in the cache backend (auth, permissions, etc). ''' - fun = '{0}.ls'.format(self.driver) + fun = '{0}.list'.format(self.driver) return self.modules[fun](bank, **self._kwargs) - list = ls - def contains(self, bank, key=None): ''' Checks if the specified bank contains the specified key. diff --git a/salt/cache/consul.py b/salt/cache/consul.py index b545c96eada..d226cad64cf 100644 --- a/salt/cache/consul.py +++ b/salt/cache/consul.py @@ -4,6 +4,8 @@ Minion data cache plugin for Consul key/value data store. .. versionadded:: 2016.11.2 +:depends: python-consul >= 0.2.0 + It is up to the system administrator to set up and configure the Consul infrastructure. All is needed for this plugin is a working Consul agent with a read-write access to the key-value store. @@ -61,7 +63,7 @@ api = None # Define the module's virtual name __virtualname__ = 'consul' -__func_alias__ = {'list': 'ls'} +__func_alias__ = {'list_': 'list'} def __virtual__(): @@ -81,8 +83,11 @@ def __virtual__(): 'verify': __opts__.get('consul.verify', True), } - global api - api = consul.Consul(**consul_kwargs) + try: + global api + api = consul.Consul(**consul_kwargs) + except AttributeError: + return (False, "Failed to invoke consul.Consul, please make sure you have python-consul >= 0.2.0 installed") return __virtualname__ @@ -139,7 +144,7 @@ def flush(bank, key=None): ) -def ls(bank): +def list_(bank): ''' Return an iterable object containing all entries stored in the specified bank. ''' diff --git a/salt/cache/localfs.py b/salt/cache/localfs.py index 68cd5aee5c3..0a73d62403e 100644 --- a/salt/cache/localfs.py +++ b/salt/cache/localfs.py @@ -20,10 +20,11 @@ import tempfile from salt.exceptions import SaltCacheError import salt.utils import salt.utils.atomicfile +import salt.utils.files log = logging.getLogger(__name__) -__func_alias__ = {'list': 'ls'} +__func_alias__ = {'list_': 'list'} def __cachedir(kwargs=None): @@ -58,7 +59,7 @@ def store(bank, key, data, cachedir): tmpfh, tmpfname = tempfile.mkstemp(dir=base) os.close(tmpfh) try: - with salt.utils.fopen(tmpfname, 'w+b') as fh_: + with salt.utils.files.fopen(tmpfname, 'w+b') as fh_: fh_.write(__context__['serial'].dumps(data)) # On Windows, os.rename will fail if the destination file exists. salt.utils.atomicfile.atomic_rename(tmpfname, outfile) @@ -85,7 +86,7 @@ def fetch(bank, key, cachedir): log.debug('Cache file "%s" does not exist', key_file) return {} try: - with salt.utils.fopen(key_file, 'rb') as fh_: + with salt.utils.files.fopen(key_file, 'rb') as fh_: if inkey: return __context__['serial'].load(fh_)[key] else: @@ -143,7 +144,7 @@ def flush(bank, key=None, cachedir=None): return True -def ls(bank, cachedir): +def list_(bank, cachedir): ''' Return an iterable object containing all entries stored in the specified bank. ''' diff --git a/salt/cache/redis_cache.py b/salt/cache/redis_cache.py index fba83e9792f..5276a8e43cf 100644 --- a/salt/cache/redis_cache.py +++ b/salt/cache/redis_cache.py @@ -67,6 +67,29 @@ host: ``localhost`` port: ``6379`` The Redis server port. +cluster_mode: ``False`` + Whether cluster_mode is enabled or not + +cluster.startup_nodes: + A list of host, port dictionaries pointing to cluster members. At least one is required + but multiple nodes are better + + .. code-block::yaml + + cache.redis.cluster.startup_nodes + - host: redis-member-1 + port: 6379 + - host: redis-member-2 + port: 6379 + +cluster.skip_full_coverage_check: ``False`` + Some cluster providers restrict certain redis commands such as CONFIG for enhanced security. + Set this option to true to skip checks that required advanced privileges. + + .. note:: + + Most cloud hosted redis clusters will require this to be set to ``True`` + db: ``'0'`` The database index. @@ -89,6 +112,24 @@ Configuration Example: cache.redis.bank_keys_prefix: #BANKEYS cache.redis.key_prefix: #KEY cache.redis.separator: '@' + +Cluster Configuration Example: + +.. code-block::yaml + + cache.redis.cluster_mode: true + cache.redis.cluster.skip_full_coverage_check: true + cache.redis.cluster.startup_nodes: + - host: redis-member-1 + port: 6379 + - host: redis-member-2 + port: 6379 + cache.redis.db: '0' + cache.redis.password: my pass + cache.redis.bank_prefix: #BANK + cache.redis.bank_keys_prefix: #BANKEYS + cache.redis.key_prefix: #KEY + cache.redis.separator: '@' ''' from __future__ import absolute_import @@ -105,6 +146,12 @@ try: except ImportError: HAS_REDIS = False +try: + from rediscluster import StrictRedisCluster + HAS_REDIS_CLUSTER = True +except ImportError: + HAS_REDIS_CLUSTER = False + # Import salt from salt.ext.six.moves import range from salt.exceptions import SaltCacheError @@ -114,9 +161,7 @@ from salt.exceptions import SaltCacheError # ----------------------------------------------------------------------------- __virtualname__ = 'redis' -__func_alias__ = { - 'list_': 'list' -} +__func_alias__ = {'list_': 'list'} log = logging.getLogger(__file__) @@ -135,9 +180,13 @@ REDIS_SERVER = None def __virtual__(): ''' The redis library must be installed for this module to work. + + The redis redis cluster library must be installed if cluster_mode is True ''' if not HAS_REDIS: return (False, "Please install the python-redis package.") + if not HAS_REDIS_CLUSTER and _get_redis_cache_opts()['cluster_mode']: + return (False, "Please install the redis-py-cluster package.") return __virtualname__ @@ -145,6 +194,9 @@ def __virtual__(): # helper functions -- will not be exported # ----------------------------------------------------------------------------- +def init_kwargs(kwargs): + return {} + def _get_redis_cache_opts(): ''' @@ -154,7 +206,10 @@ def _get_redis_cache_opts(): 'host': __opts__.get('cache.redis.host', 'localhost'), 'port': __opts__.get('cache.redis.port', 6379), 'db': __opts__.get('cache.redis.db', '0'), - 'password': __opts__.get('cache.redis.password', '') + 'password': __opts__.get('cache.redis.password', ''), + 'cluster_mode': __opts__.get('cache.redis.cluster_mode', False), + 'startup_nodes': __opts__.get('cache.redis.cluster.startup_nodes', {}), + 'skip_full_coverage_check': __opts__.get('cache.redis.cluster.skip_full_coverage_check', False), } @@ -168,10 +223,16 @@ def _get_redis_server(opts=None): return REDIS_SERVER if not opts: opts = _get_redis_cache_opts() - REDIS_SERVER = redis.Redis(opts['host'], - opts['port'], - db=opts['db'], - password=opts['password']) + + if opts['cluster_mode']: + REDIS_SERVER = StrictRedisCluster(startup_nodes=opts['startup_nodes'], + skip_full_coverage_check=opts['skip_full_coverage_check'], + decode_responses=True) + else: + REDIS_SERVER = redis.StrictRedis(opts['host'], + opts['port'], + db=opts['db'], + password=opts['password']) return REDIS_SERVER diff --git a/salt/cli/batch.py b/salt/cli/batch.py index 3617b717a50..55880c3b836 100644 --- a/salt/cli/batch.py +++ b/salt/cli/batch.py @@ -11,14 +11,14 @@ import copy from datetime import datetime, timedelta # Import salt libs +import salt.utils # Can be removed once print_cli is moved import salt.client import salt.output import salt.exceptions -from salt.utils import print_cli # Import 3rd-party libs # pylint: disable=import-error,no-name-in-module,redefined-builtin -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import range # pylint: enable=import-error,no-name-in-module,redefined-builtin import logging @@ -73,7 +73,7 @@ class Batch(object): m = next(six.iterkeys(ret)) except StopIteration: if not self.quiet: - print_cli('No minions matched the target.') + salt.utils.print_cli('No minions matched the target.') break if m is not None: fret.add(m) @@ -95,7 +95,7 @@ class Batch(object): return int(self.opts['batch']) except ValueError: if not self.quiet: - print_cli('Invalid batch data sent: {0}\nData must be in the ' + salt.utils.print_cli('Invalid batch data sent: {0}\nData must be in the ' 'form of %10, 10% or 3'.format(self.opts['batch'])) def __update_wait(self, wait): @@ -146,7 +146,7 @@ class Batch(object): # We already know some minions didn't respond to the ping, so inform # the user we won't be attempting to run a job on them for down_minion in self.down_minions: - print_cli('Minion {0} did not respond. No job will be sent.'.format(down_minion)) + salt.utils.print_cli('Minion {0} did not respond. No job will be sent.'.format(down_minion)) # Iterate while we still have things to execute while len(ret) < len(self.minions): @@ -171,7 +171,7 @@ class Batch(object): if next_: if not self.quiet: - print_cli('\nExecuting run on {0}\n'.format(sorted(next_))) + salt.utils.print_cli('\nExecuting run on {0}\n'.format(sorted(next_))) # create a new iterator for this batch of minions new_iter = self.local.cmd_iter_no_block( *args, @@ -218,14 +218,14 @@ class Batch(object): if part['data']['id'] in minion_tracker[queue]['minions']: minion_tracker[queue]['minions'].remove(part['data']['id']) else: - print_cli('minion {0} was already deleted from tracker, probably a duplicate key'.format(part['id'])) + salt.utils.print_cli('minion {0} was already deleted from tracker, probably a duplicate key'.format(part['id'])) else: parts.update(part) for id in part: if id in minion_tracker[queue]['minions']: minion_tracker[queue]['minions'].remove(id) else: - print_cli('minion {0} was already deleted from tracker, probably a duplicate key'.format(id)) + salt.utils.print_cli('minion {0} was already deleted from tracker, probably a duplicate key'.format(id)) except StopIteration: # if a iterator is done: # - set it to inactive diff --git a/salt/cli/call.py b/salt/cli/call.py index b7cc235deb5..3fb49348e80 100644 --- a/salt/cli/call.py +++ b/salt/cli/call.py @@ -1,16 +1,15 @@ # -*- coding: utf-8 -*- -from __future__ import print_function -from __future__ import absolute_import +from __future__ import absolute_import, print_function import os -from salt.utils import parsers +import salt.utils.parsers from salt.utils.verify import verify_log from salt.config import _expand_glob_path import salt.cli.caller import salt.defaults.exitcodes -class SaltCall(parsers.SaltCallOptionParser): +class SaltCall(salt.utils.parsers.SaltCallOptionParser): ''' Used to locally execute a salt command ''' diff --git a/salt/cli/caller.py b/salt/cli/caller.py index b69438c3d47..5c04bab11c4 100644 --- a/salt/cli/caller.py +++ b/salt/cli/caller.py @@ -20,18 +20,17 @@ import salt.minion import salt.output import salt.payload import salt.transport +import salt.utils # Can be removed once print_cli, activate_profile, and output_profile are moved import salt.utils.args +import salt.utils.files import salt.utils.jid +import salt.utils.kinds as kinds import salt.utils.minion import salt.defaults.exitcodes -from salt.log import LOG_LEVELS -from salt.utils import is_windows -from salt.utils import print_cli -from salt.utils import kinds -from salt.utils import activate_profile -from salt.utils import output_profile -from salt.utils.process import MultiprocessingProcess from salt.cli import daemons +from salt.log import LOG_LEVELS +from salt.utils.platform import is_windows +from salt.utils.process import MultiprocessingProcess try: from raet import raeting, nacling @@ -46,7 +45,7 @@ except ImportError: pass # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six # Custom exceptions from salt.exceptions import ( @@ -114,7 +113,7 @@ class BaseCaller(object): docs[name] = func.__doc__ for name in sorted(docs): if name.startswith(self.opts.get('fun', '')): - print_cli('{0}:\n{1}\n'.format(name, docs[name])) + salt.utils.print_cli('{0}:\n{1}\n'.format(name, docs[name])) def print_grains(self): ''' @@ -129,14 +128,14 @@ class BaseCaller(object): ''' profiling_enabled = self.opts.get('profiling_enabled', False) try: - pr = activate_profile(profiling_enabled) + pr = salt.utils.activate_profile(profiling_enabled) try: ret = self.call() finally: - output_profile(pr, - stats_path=self.opts.get('profiling_path', - '/tmp/stats'), - stop=True) + salt.utils.output_profile( + pr, + stats_path=self.opts.get('profiling_path', '/tmp/stats'), + stop=True) out = ret.get('out', 'nested') if self.opts['print_metadata']: print_ret = ret @@ -158,12 +157,18 @@ class BaseCaller(object): ''' ret = {} fun = self.opts['fun'] - ret['jid'] = salt.utils.jid.gen_jid() + ret['jid'] = salt.utils.jid.gen_jid(self.opts) proc_fn = os.path.join( salt.minion.get_proc_dir(self.opts['cachedir']), ret['jid'] ) if fun not in self.minion.functions: + docs = self.minion.functions['sys.doc']('{0}*'.format(fun)) + if docs: + docs[fun] = self.minion.functions.missing_fun_string(fun) + ret['out'] = 'nested' + ret['return'] = docs + return ret sys.stderr.write(self.minion.functions.missing_fun_string(fun)) mod_name = fun.split('.')[0] if mod_name in self.minion.function_errors: @@ -189,7 +194,7 @@ class BaseCaller(object): no_parse=self.opts.get('no_parse', [])), data=sdata) try: - with salt.utils.fopen(proc_fn, 'w+b') as fp_: + with salt.utils.files.fopen(proc_fn, 'w+b') as fp_: fp_.write(self.serial.dumps(sdata)) except NameError: # Don't require msgpack with local @@ -204,7 +209,7 @@ class BaseCaller(object): ret['return'] = func(*args, **kwargs) except TypeError as exc: sys.stderr.write('\nPassed invalid arguments: {0}.\n\nUsage:\n'.format(exc)) - print_cli(func.__doc__) + salt.utils.print_cli(func.__doc__) active_level = LOG_LEVELS.get( self.opts['log_level'].lower(), logging.ERROR) if active_level <= logging.DEBUG: diff --git a/salt/cli/cp.py b/salt/cli/cp.py index dd1d17e5e89..b45edc0c4d1 100644 --- a/salt/cli/cp.py +++ b/salt/cli/cp.py @@ -18,12 +18,16 @@ import sys # Import salt libs import salt.client +import salt.output +import salt.utils +import salt.utils.files import salt.utils.gzip_util import salt.utils.itertools import salt.utils.minions -from salt.utils import parsers, to_bytes -from salt.utils.verify import verify_log -import salt.output +import salt.utils.parsers +import salt.utils.platform +import salt.utils.stringutils +import salt.utils.verify # Import 3rd party libs from salt.ext import six @@ -31,7 +35,7 @@ from salt.ext import six log = logging.getLogger(__name__) -class SaltCPCli(parsers.SaltCPOptionParser): +class SaltCPCli(salt.utils.parsers.SaltCPOptionParser): ''' Run the salt-cp command line client ''' @@ -44,7 +48,7 @@ class SaltCPCli(parsers.SaltCPOptionParser): # Setup file logging! self.setup_logfile_logger() - verify_log(self.config) + salt.utils.verify.verify_log(self.config) cp_ = SaltCP(self.config) cp_.run() @@ -56,7 +60,7 @@ class SaltCP(object): ''' def __init__(self, opts): self.opts = opts - self.is_windows = salt.utils.is_windows() + self.is_windows = salt.utils.platform.is_windows() def _mode(self, path): if self.is_windows: @@ -101,10 +105,70 @@ class SaltCP(object): empty_dirs.update(empty_dirs_) return files, sorted(empty_dirs) + def _file_dict(self, fn_): + ''' + Take a path and return the contents of the file as a string + ''' + if not os.path.isfile(fn_): + err = 'The referenced file, {0} is not available.'.format(fn_) + sys.stderr.write(err + '\n') + sys.exit(42) + with salt.utils.files.fopen(fn_, 'r') as fp_: + data = fp_.read() + return {fn_: data} + + def _load_files(self): + ''' + Parse the files indicated in opts['src'] and load them into a python + object for transport + ''' + files = {} + for fn_ in self.opts['src']: + if os.path.isfile(fn_): + files.update(self._file_dict(fn_)) + elif os.path.isdir(fn_): + salt.utils.print_cli(fn_ + ' is a directory, only files are supported ' + 'in non-chunked mode. Use "--chunked" command ' + 'line argument.') + sys.exit(1) + return files + def run(self): ''' Make the salt client call ''' + if self.opts['chunked']: + ret = self.run_chunked() + else: + ret = self.run_oldstyle() + + salt.output.display_output( + ret, + self.opts.get('output', 'nested'), + self.opts) + + def run_oldstyle(self): + ''' + Make the salt client call in old-style all-in-one call method + ''' + arg = [self._load_files(), self.opts['dest']] + local = salt.client.get_local_client(self.opts['conf_file']) + args = [self.opts['tgt'], + 'cp.recv', + arg, + self.opts['timeout'], + ] + + selected_target_option = self.opts.get('selected_target_option', None) + if selected_target_option is not None: + args.append(selected_target_option) + + return local.cmd(*args) + + def run_chunked(self): + ''' + Make the salt client call in the new fasion chunked multi-call way + ''' files, empty_dirs = self._list_files() dest = self.opts['dest'] gzip = self.opts['gzip'] @@ -120,9 +184,10 @@ class SaltCP(object): if gzip \ else salt.utils.itertools.read_file - minions = salt.utils.minions.CkMinions(self.opts).check_minions( + _res = salt.utils.minions.CkMinions(self.opts).check_minions( tgt, tgt_type=selected_target_option or 'glob') + minions = _res['minions'] local = salt.client.get_local_client(self.opts['conf_file']) @@ -152,7 +217,7 @@ class SaltCP(object): index = 1 failed = {} for chunk in reader(fn_, chunk_size=self.opts['salt_cp_chunk_size']): - chunk = base64.b64encode(to_bytes(chunk)) + chunk = base64.b64encode(salt.utils.stringutils.to_bytes(chunk)) append = index > 1 log.debug( 'Copying %s to %starget \'%s\' as %s%s', @@ -166,7 +231,7 @@ class SaltCP(object): ) args = [ tgt, - 'cp.recv', + 'cp.recv_chunked', [remote_path, chunk, append, gzip, mode], timeout, ] @@ -212,14 +277,11 @@ class SaltCP(object): else '', tgt, ) - args = [tgt, 'cp.recv', [remote_path, None], timeout] + args = [tgt, 'cp.recv_chunked', [remote_path, None], timeout] if selected_target_option is not None: args.append(selected_target_option) for minion_id, minion_ret in six.iteritems(local.cmd(*args)): ret.setdefault(minion_id, {})[remote_path] = minion_ret - salt.output.display_output( - ret, - self.opts.get('output', 'nested'), - self.opts) + return ret diff --git a/salt/cli/daemons.py b/salt/cli/daemons.py index a510ad60c1a..82828c47bd0 100644 --- a/salt/cli/daemons.py +++ b/salt/cli/daemons.py @@ -41,10 +41,11 @@ import salt.log.setup # the try block below bypasses an issue at build time so that modules don't # cause the build to fail from salt.utils import migrations -from salt.utils import kinds +import salt.utils.kinds as kinds try: - from salt.utils import parsers, ip_bracket + from salt.utils import ip_bracket + import salt.utils.parsers from salt.utils.verify import check_user, verify_env, verify_socket except ImportError as exc: if exc.args[0] != 'No module named _msgpack': @@ -109,7 +110,7 @@ class DaemonsMixin(object): # pylint: disable=no-init self.shutdown(error) -class Master(parsers.MasterOptionParser, DaemonsMixin): # pylint: disable=no-init +class Master(salt.utils.parsers.MasterOptionParser, DaemonsMixin): # pylint: disable=no-init ''' Creates a master server ''' @@ -220,7 +221,7 @@ class Master(parsers.MasterOptionParser, DaemonsMixin): # pylint: disable=no-in super(Master, self).shutdown(exitcode, exitmsg) -class Minion(parsers.MinionOptionParser, DaemonsMixin): # pylint: disable=no-init +class Minion(salt.utils.parsers.MinionOptionParser, DaemonsMixin): # pylint: disable=no-init ''' Create a minion server ''' @@ -398,7 +399,7 @@ class Minion(parsers.MinionOptionParser, DaemonsMixin): # pylint: disable=no-in # pylint: enable=no-member -class ProxyMinion(parsers.ProxyMinionOptionParser, DaemonsMixin): # pylint: disable=no-init +class ProxyMinion(salt.utils.parsers.ProxyMinionOptionParser, DaemonsMixin): # pylint: disable=no-init ''' Create a proxy minion server ''' @@ -549,7 +550,7 @@ class ProxyMinion(parsers.ProxyMinionOptionParser, DaemonsMixin): # pylint: dis # pylint: enable=no-member -class Syndic(parsers.SyndicOptionParser, DaemonsMixin): # pylint: disable=no-init +class Syndic(salt.utils.parsers.SyndicOptionParser, DaemonsMixin): # pylint: disable=no-init ''' Create a syndic server ''' diff --git a/salt/cli/key.py b/salt/cli/key.py index 1454a2b43c4..42622c40785 100644 --- a/salt/cli/key.py +++ b/salt/cli/key.py @@ -3,11 +3,11 @@ from __future__ import print_function from __future__ import absolute_import -from salt.utils import parsers +import salt.utils.parsers from salt.utils.verify import check_user, verify_log -class SaltKey(parsers.SaltKeyOptionParser): +class SaltKey(salt.utils.parsers.SaltKeyOptionParser): ''' Initialize the Salt key manager ''' diff --git a/salt/cli/run.py b/salt/cli/run.py index ed8cf9324cc..c780681368d 100644 --- a/salt/cli/run.py +++ b/salt/cli/run.py @@ -2,15 +2,14 @@ from __future__ import print_function from __future__ import absolute_import -from salt.utils import parsers -from salt.utils import activate_profile -from salt.utils import output_profile +import salt.utils # Can be removed once activate_profile and output_profile are moved +import salt.utils.parsers from salt.utils.verify import check_user, verify_log from salt.exceptions import SaltClientError import salt.defaults.exitcodes # pylint: disable=W0611 -class SaltRun(parsers.SaltRunOptionParser): +class SaltRun(salt.utils.parsers.SaltRunOptionParser): ''' Used to execute Salt runners ''' @@ -36,7 +35,7 @@ class SaltRun(parsers.SaltRunOptionParser): # someone tries to use the runners via the python API try: if check_user(self.config['user']): - pr = activate_profile(profiling_enabled) + pr = salt.utils.activate_profile(profiling_enabled) try: ret = runner.run() # In older versions ret['data']['retcode'] was used @@ -50,7 +49,7 @@ class SaltRun(parsers.SaltRunOptionParser): elif isinstance(ret, dict) and 'retcode' in ret.get('data', {}): self.exit(ret['data']['retcode']) finally: - output_profile( + salt.utils.output_profile( pr, stats_path=self.options.profiling_path, stop=True) diff --git a/salt/cli/salt.py b/salt/cli/salt.py index 9e35cde576b..24e3a2ecd46 100644 --- a/salt/cli/salt.py +++ b/salt/cli/salt.py @@ -7,8 +7,8 @@ sys.modules['pkg_resources'] = None import os # Import Salt libs -from salt.ext.six import string_types -from salt.utils import parsers, print_cli +import salt.utils # Can be removed once print_cli is moved +import salt.utils.parsers from salt.utils.args import yamlify_arg from salt.utils.verify import verify_log from salt.exceptions import ( @@ -18,10 +18,10 @@ from salt.exceptions import ( ) # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six -class SaltCMD(parsers.SaltCMDOptionParser): +class SaltCMD(salt.utils.parsers.SaltCMDOptionParser): ''' The execution of a salt command happens here ''' @@ -75,8 +75,9 @@ class SaltCMD(parsers.SaltCMDOptionParser): 'show_jid': self.options.show_jid} if 'token' in self.config: + import salt.utils.files try: - with salt.utils.fopen(os.path.join(self.config['cachedir'], '.root_key'), 'r') as fp_: + with salt.utils.files.fopen(os.path.join(self.config['cachedir'], '.root_key'), 'r') as fp_: kwargs['key'] = fp_.readline() except IOError: kwargs['token'] = self.config['token'] @@ -92,7 +93,7 @@ class SaltCMD(parsers.SaltCMDOptionParser): # potentially switch to batch execution if self.options.batch_safe_limit > 1: if len(self._preview_target()) >= self.options.batch_safe_limit: - print_cli('\nNOTICE: Too many minions targeted, switching to batch execution.') + salt.utils.print_cli('\nNOTICE: Too many minions targeted, switching to batch execution.') self.options.batch = self.options.batch_safe_size self._run_batch() return @@ -139,7 +140,7 @@ class SaltCMD(parsers.SaltCMDOptionParser): if self.config['async']: jid = self.local_client.cmd_async(**kwargs) - print_cli('Executed command with job ID: {0}'.format(jid)) + salt.utils.print_cli('Executed command with job ID: {0}'.format(jid)) return # local will be None when there was an error @@ -278,12 +279,12 @@ class SaltCMD(parsers.SaltCMDOptionParser): def _print_errors_summary(self, errors): if errors: - print_cli('\n') - print_cli('---------------------------') - print_cli('Errors') - print_cli('---------------------------') + salt.utils.print_cli('\n') + salt.utils.print_cli('---------------------------') + salt.utils.print_cli('Errors') + salt.utils.print_cli('---------------------------') for error in errors: - print_cli(self._format_error(error)) + salt.utils.print_cli(self._format_error(error)) def _print_returns_summary(self, ret): ''' @@ -296,9 +297,11 @@ class SaltCMD(parsers.SaltCMDOptionParser): not_connected_minions = [] failed_minions = [] for each_minion in ret: - minion_ret = ret[each_minion].get('ret') + minion_ret = ret[each_minion] + if isinstance(minion_ret, dict) and 'ret' in minion_ret: + minion_ret = ret[each_minion].get('ret') if ( - isinstance(minion_ret, string_types) + isinstance(minion_ret, six.string_types) and minion_ret.startswith("Minion did not return") ): if "Not connected" in minion_ret: @@ -311,22 +314,22 @@ class SaltCMD(parsers.SaltCMDOptionParser): return_counter += 1 if self._get_retcode(ret[each_minion]): failed_minions.append(each_minion) - print_cli('\n') - print_cli('-------------------------------------------') - print_cli('Summary') - print_cli('-------------------------------------------') - print_cli('# of minions targeted: {0}'.format(return_counter + not_return_counter)) - print_cli('# of minions returned: {0}'.format(return_counter)) - print_cli('# of minions that did not return: {0}'.format(not_return_counter)) - print_cli('# of minions with errors: {0}'.format(len(failed_minions))) + salt.utils.print_cli('\n') + salt.utils.print_cli('-------------------------------------------') + salt.utils.print_cli('Summary') + salt.utils.print_cli('-------------------------------------------') + salt.utils.print_cli('# of minions targeted: {0}'.format(return_counter + not_return_counter)) + salt.utils.print_cli('# of minions returned: {0}'.format(return_counter)) + salt.utils.print_cli('# of minions that did not return: {0}'.format(not_return_counter)) + salt.utils.print_cli('# of minions with errors: {0}'.format(len(failed_minions))) if self.options.verbose: if not_connected_minions: - print_cli('Minions not connected: {0}'.format(" ".join(not_connected_minions))) + salt.utils.print_cli('Minions not connected: {0}'.format(" ".join(not_connected_minions))) if not_response_minions: - print_cli('Minions not responding: {0}'.format(" ".join(not_response_minions))) + salt.utils.print_cli('Minions not responding: {0}'.format(" ".join(not_response_minions))) if failed_minions: - print_cli('Minions with failures: {0}'.format(" ".join(failed_minions))) - print_cli('-------------------------------------------') + salt.utils.print_cli('Minions with failures: {0}'.format(" ".join(failed_minions))) + salt.utils.print_cli('-------------------------------------------') def _progress_end(self, out): import salt.output @@ -403,10 +406,10 @@ class SaltCMD(parsers.SaltCMDOptionParser): docs = {} if not ret: self.exit(2, 'No minions found to gather docs from\n') - if isinstance(ret, str): + if isinstance(ret, six.string_types): self.exit(2, '{0}\n'.format(ret)) for host in ret: - if isinstance(ret[host], string_types) \ + if isinstance(ret[host], six.string_types) \ and (ret[host].startswith("Minion did not return") or ret[host] == 'VALUE TRIMMED'): continue @@ -418,6 +421,6 @@ class SaltCMD(parsers.SaltCMDOptionParser): salt.output.display_output({fun: docs[fun]}, 'nested', self.config) else: for fun in sorted(docs): - print_cli('{0}:'.format(fun)) - print_cli(docs[fun]) - print_cli('') + salt.utils.print_cli('{0}:'.format(fun)) + salt.utils.print_cli(docs[fun]) + salt.utils.print_cli('') diff --git a/salt/cli/spm.py b/salt/cli/spm.py index 3d347c80a8d..303e5ce65f4 100644 --- a/salt/cli/spm.py +++ b/salt/cli/spm.py @@ -14,7 +14,7 @@ from __future__ import absolute_import # Import Salt libs import salt.spm import salt.utils.parsers as parsers -from salt.utils.verify import verify_log +from salt.utils.verify import verify_log, verify_env class SPM(parsers.SPMParser): @@ -29,6 +29,10 @@ class SPM(parsers.SPMParser): ui = salt.spm.SPMCmdlineInterface() self.parse_args() self.setup_logfile_logger() + v_dirs = [ + self.config['cachedir'], + ] + verify_env(v_dirs, self.config['user'],) verify_log(self.config) client = salt.spm.SPMClient(ui, self.config) client.run(self.args) diff --git a/salt/cli/ssh.py b/salt/cli/ssh.py index 0d6bf6ea7b4..58fb4c6083f 100644 --- a/salt/cli/ssh.py +++ b/salt/cli/ssh.py @@ -2,17 +2,21 @@ from __future__ import print_function from __future__ import absolute_import +import sys import salt.client.ssh -from salt.utils import parsers +import salt.utils.parsers from salt.utils.verify import verify_log -class SaltSSH(parsers.SaltSSHOptionParser): +class SaltSSH(salt.utils.parsers.SaltSSHOptionParser): ''' Used to Execute the salt ssh routine ''' def run(self): + if '-H' in sys.argv or '--hosts' in sys.argv: + sys.argv += ['x', 'x'] # Hack: pass a mandatory two options + # that won't be used anyways with -H or --hosts self.parse_args() self.setup_logfile_logger() verify_log(self.config) diff --git a/salt/client/__init__.py b/salt/client/__init__.py index e7821a56600..b047e599369 100644 --- a/salt/client/__init__.py +++ b/salt/client/__init__.py @@ -35,8 +35,11 @@ import salt.loader import salt.utils import salt.utils.args import salt.utils.event +import salt.utils.files import salt.utils.minions +import salt.utils.platform import salt.utils.verify +import salt.utils.versions import salt.utils.jid import salt.syspaths as syspaths from salt.exceptions import ( @@ -45,7 +48,7 @@ from salt.exceptions import ( ) # Import third party libs -import salt.ext.six as six +from salt.ext import six # pylint: disable=import-error try: import zmq @@ -69,7 +72,7 @@ log = logging.getLogger(__name__) def get_local_client( - c_path=os.path.join(syspaths.CONFIG_DIR, 'master'), + c_path=os.path.join(syspaths.CONFIG_DIR, u'master'), mopts=None, skip_perm_errors=False, io_loop=None, @@ -92,11 +95,11 @@ def get_local_client( # Late import to prevent circular import import salt.config opts = salt.config.client_config(c_path) - if opts['transport'] == 'raet': + if opts[u'transport'] == u'raet': import salt.client.raet return salt.client.raet.LocalClient(mopts=opts) # TODO: AIO core is separate from transport - elif opts['transport'] in ('zeromq', 'tcp'): + elif opts[u'transport'] in (u'zeromq', u'tcp'): return LocalClient( mopts=opts, skip_perm_errors=skip_perm_errors, @@ -133,7 +136,7 @@ class LocalClient(object): local.cmd('*', 'test.fib', [10]) ''' def __init__(self, - c_path=os.path.join(syspaths.CONFIG_DIR, 'master'), + c_path=os.path.join(syspaths.CONFIG_DIR, u'master'), mopts=None, skip_perm_errors=False, io_loop=None, keep_loop=False, auto_reconnect=False): ''' @@ -148,10 +151,9 @@ class LocalClient(object): else: if os.path.isdir(c_path): log.warning( - '{0} expects a file path not a directory path({1}) to ' - 'it\'s \'c_path\' keyword argument'.format( - self.__class__.__name__, c_path - ) + u'%s expects a file path not a directory path(%s) to ' + u'its \'c_path\' keyword argument', + self.__class__.__name__, c_path ) self.opts = salt.config.client_config(c_path) self.serial = salt.payload.Serial(self.opts) @@ -160,9 +162,9 @@ class LocalClient(object): self.key = self.__read_master_key() self.auto_reconnect = auto_reconnect self.event = salt.utils.event.get_event( - 'master', - self.opts['sock_dir'], - self.opts['transport'], + u'master', + self.opts[u'sock_dir'], + self.opts[u'transport'], opts=self.opts, listen=False, io_loop=io_loop, @@ -176,38 +178,37 @@ class LocalClient(object): Read in the rotating master authentication key ''' key_user = self.salt_user - if key_user == 'root': - if self.opts.get('user', 'root') != 'root': - key_user = self.opts.get('user', 'root') - if key_user.startswith('sudo_'): - key_user = self.opts.get('user', 'root') - if salt.utils.is_windows(): + if key_user == u'root': + if self.opts.get(u'user', u'root') != u'root': + key_user = self.opts.get(u'user', u'root') + if key_user.startswith(u'sudo_'): + key_user = self.opts.get(u'user', u'root') + if salt.utils.platform.is_windows(): # The username may contain '\' if it is in Windows # 'DOMAIN\username' format. Fix this for the keyfile path. - key_user = key_user.replace('\\', '_') - keyfile = os.path.join(self.opts['cachedir'], - '.{0}_key'.format(key_user)) - # Make sure all key parent directories are accessible - salt.utils.verify.check_path_traversal(self.opts['cachedir'], - key_user, - self.skip_perm_errors) - + key_user = key_user.replace(u'\\', u'_') + keyfile = os.path.join(self.opts[u'cachedir'], + u'.{0}_key'.format(key_user)) try: - with salt.utils.fopen(keyfile, 'r') as key: + # Make sure all key parent directories are accessible + salt.utils.verify.check_path_traversal(self.opts[u'cachedir'], + key_user, + self.skip_perm_errors) + with salt.utils.files.fopen(keyfile, u'r') as key: return key.read() - except (OSError, IOError): + except (OSError, IOError, SaltClientError): # Fall back to eauth - return '' + return u'' def _convert_range_to_list(self, tgt): ''' convert a seco.range range into a list target ''' - range_ = seco.range.Range(self.opts['range_server']) + range_ = seco.range.Range(self.opts[u'range_server']) try: return range_.expand(tgt) except seco.range.RangeException as err: - print('Range server exception: {0}'.format(err)) + print(u'Range server exception: {0}'.format(err)) return [] def _get_timeout(self, timeout): @@ -215,34 +216,34 @@ class LocalClient(object): Return the timeout to use ''' if timeout is None: - return self.opts['timeout'] + return self.opts[u'timeout'] if isinstance(timeout, int): return timeout if isinstance(timeout, six.string_types): try: return int(timeout) except ValueError: - return self.opts['timeout'] + return self.opts[u'timeout'] # Looks like the timeout is invalid, use config - return self.opts['timeout'] + return self.opts[u'timeout'] def gather_job_info(self, jid, tgt, tgt_type, **kwargs): ''' Return the information about a given job ''' - log.debug('Checking whether jid {0} is still running'.format(jid)) - timeout = int(kwargs.get('gather_job_timeout', self.opts['gather_job_timeout'])) + log.debug(u'Checking whether jid %s is still running', jid) + timeout = int(kwargs.get(u'gather_job_timeout', self.opts[u'gather_job_timeout'])) pub_data = self.run_job(tgt, - 'saltutil.find_job', + u'saltutil.find_job', arg=[jid], tgt_type=tgt_type, timeout=timeout, **kwargs ) - if 'jid' in pub_data: - self.event.subscribe(pub_data['jid']) + if u'jid' in pub_data: + self.event.subscribe(pub_data[u'jid']) return pub_data @@ -250,39 +251,39 @@ class LocalClient(object): ''' Common checks on the pub_data data structure returned from running pub ''' - if pub_data == '': + if pub_data == u'': # Failed to authenticate, this could be a bunch of things raise EauthAuthenticationError( - 'Failed to authenticate! This is most likely because this ' - 'user is not permitted to execute commands, but there is a ' - 'small possibility that a disk error occurred (check ' - 'disk/inode usage).' + u'Failed to authenticate! This is most likely because this ' + u'user is not permitted to execute commands, but there is a ' + u'small possibility that a disk error occurred (check ' + u'disk/inode usage).' ) # Failed to connect to the master and send the pub - if 'error' in pub_data: - print(pub_data['error']) - log.debug('_check_pub_data() error: {0}'.format(pub_data['error'])) + if u'error' in pub_data: + print(pub_data[u'error']) + log.debug(u'_check_pub_data() error: %s', pub_data[u'error']) return {} - elif 'jid' not in pub_data: + elif u'jid' not in pub_data: return {} - if pub_data['jid'] == '0': - print('Failed to connect to the Master, ' - 'is the Salt Master running?') + if pub_data[u'jid'] == u'0': + print(u'Failed to connect to the Master, ' + u'is the Salt Master running?') return {} # If we order masters (via a syndic), don't short circuit if no minions # are found - if not self.opts.get('order_masters'): + if not self.opts.get(u'order_masters'): # Check for no minions - if not pub_data['minions']: - print('No minions matched the target. ' - 'No command was sent, no jid was assigned.') + if not pub_data[u'minions']: + print(u'No minions matched the target. ' + u'No command was sent, no jid was assigned.') return {} else: - self.event.subscribe('syndic/.*/{0}'.format(pub_data['jid']), 'regex') + self.event.subscribe(u'syndic/.*/{0}'.format(pub_data[u'jid']), u'regex') - self.event.subscribe('salt/job/{0}'.format(pub_data['jid'])) + self.event.subscribe(u'salt/job/{0}'.format(pub_data[u'jid'])) return pub_data @@ -291,10 +292,10 @@ class LocalClient(object): tgt, fun, arg=(), - tgt_type='glob', - ret='', + tgt_type=u'glob', + ret=u'', timeout=None, - jid='', + jid=u'', kwarg=None, listen=False, **kwargs): @@ -312,14 +313,14 @@ class LocalClient(object): >>> local.run_job('*', 'test.sleep', [300]) {'jid': '20131219215650131543', 'minions': ['jerry']} ''' - if 'expr_form' in kwargs: - salt.utils.warn_until( - 'Fluorine', - 'The target type should be passed using the \'tgt_type\' ' - 'argument instead of \'expr_form\'. Support for using ' - '\'expr_form\' will be removed in Salt Fluorine.' + if u'expr_form' in kwargs: + salt.utils.versions.warn_until( + u'Fluorine', + u'The target type should be passed using the \'tgt_type\' ' + u'argument instead of \'expr_form\'. Support for using ' + u'\'expr_form\' will be removed in Salt Fluorine.' ) - tgt_type = kwargs.pop('expr_form') + tgt_type = kwargs.pop(u'expr_form') arg = salt.utils.args.condition_input(arg, kwarg) @@ -337,7 +338,7 @@ class LocalClient(object): except SaltClientError: # Re-raise error with specific message raise SaltClientError( - 'The salt master could not be contacted. Is master running?' + u'The salt master could not be contacted. Is master running?' ) except Exception as general_exception: # Convert to generic client error and pass along message @@ -346,7 +347,8 @@ class LocalClient(object): return self._check_pub_data(pub_data) def gather_minions(self, tgt, expr_form): - return salt.utils.minions.CkMinions(self.opts).check_minions(tgt, tgt_type=expr_form) + _res = salt.utils.minions.CkMinions(self.opts).check_minions(tgt, tgt_type=expr_form) + return _res['minions'] @tornado.gen.coroutine def run_job_async( @@ -354,10 +356,10 @@ class LocalClient(object): tgt, fun, arg=(), - tgt_type='glob', - ret='', + tgt_type=u'glob', + ret=u'', timeout=None, - jid='', + jid=u'', kwarg=None, listen=True, io_loop=None, @@ -376,14 +378,14 @@ class LocalClient(object): >>> local.run_job_async('*', 'test.sleep', [300]) {'jid': '20131219215650131543', 'minions': ['jerry']} ''' - if 'expr_form' in kwargs: - salt.utils.warn_until( - 'Fluorine', - 'The target type should be passed using the \'tgt_type\' ' - 'argument instead of \'expr_form\'. Support for using ' - '\'expr_form\' will be removed in Salt Fluorine.' + if u'expr_form' in kwargs: + salt.utils.versions.warn_until( + u'Fluorine', + u'The target type should be passed using the \'tgt_type\' ' + u'argument instead of \'expr_form\'. Support for using ' + u'\'expr_form\' will be removed in Salt Fluorine.' ) - tgt_type = kwargs.pop('expr_form') + tgt_type = kwargs.pop(u'expr_form') arg = salt.utils.args.condition_input(arg, kwarg) @@ -402,7 +404,7 @@ class LocalClient(object): except SaltClientError: # Re-raise error with specific message raise SaltClientError( - 'The salt master could not be contacted. Is master running?' + u'The salt master could not be contacted. Is master running?' ) except Exception as general_exception: # Convert to generic client error and pass along message @@ -415,9 +417,9 @@ class LocalClient(object): tgt, fun, arg=(), - tgt_type='glob', - ret='', - jid='', + tgt_type=u'glob', + ret=u'', + jid=u'', kwarg=None, **kwargs): ''' @@ -433,14 +435,14 @@ class LocalClient(object): >>> local.cmd_async('*', 'test.sleep', [300]) '20131219215921857715' ''' - if 'expr_form' in kwargs: - salt.utils.warn_until( - 'Fluorine', - 'The target type should be passed using the \'tgt_type\' ' - 'argument instead of \'expr_form\'. Support for using ' - '\'expr_form\' will be removed in Salt Fluorine.' + if u'expr_form' in kwargs: + salt.utils.versions.warn_until( + u'Fluorine', + u'The target type should be passed using the \'tgt_type\' ' + u'argument instead of \'expr_form\'. Support for using ' + u'\'expr_form\' will be removed in Salt Fluorine.' ) - tgt_type = kwargs.pop('expr_form') + tgt_type = kwargs.pop(u'expr_form') arg = salt.utils.args.condition_input(arg, kwarg) pub_data = self.run_job(tgt, @@ -451,7 +453,7 @@ class LocalClient(object): jid=jid, **kwargs) try: - return pub_data['jid'] + return pub_data[u'jid'] except KeyError: return 0 @@ -460,8 +462,8 @@ class LocalClient(object): tgt, fun, arg=(), - tgt_type='glob', - ret='', + tgt_type=u'glob', + ret=u'', kwarg=None, sub=3, cli=False, @@ -480,17 +482,17 @@ class LocalClient(object): >>> SLC.cmd_subset('*', 'test.ping', sub=1) {'jerry': True} ''' - if 'expr_form' in kwargs: - salt.utils.warn_until( - 'Fluorine', - 'The target type should be passed using the \'tgt_type\' ' - 'argument instead of \'expr_form\'. Support for using ' - '\'expr_form\' will be removed in Salt Fluorine.' + if u'expr_form' in kwargs: + salt.utils.versions.warn_until( + u'Fluorine', + u'The target type should be passed using the \'tgt_type\' ' + u'argument instead of \'expr_form\'. Support for using ' + u'\'expr_form\' will be removed in Salt Fluorine.' ) - tgt_type = kwargs.pop('expr_form') + tgt_type = kwargs.pop(u'expr_form') minion_ret = self.cmd(tgt, - 'sys.list_functions', + u'sys.list_functions', tgt_type=tgt_type, **kwargs) minions = list(minion_ret) @@ -508,7 +510,7 @@ class LocalClient(object): f_tgt, fun, arg, - tgt_type='list', + tgt_type=u'list', ret=ret, kwarg=kwarg, progress=progress, @@ -519,10 +521,10 @@ class LocalClient(object): tgt, fun, arg=(), - tgt_type='glob', - ret='', + tgt_type=u'glob', + ret=u'', kwarg=None, - batch='10%', + batch=u'10%', **kwargs): ''' Iteratively execute a command on subsets of minions at a time @@ -543,40 +545,42 @@ class LocalClient(object): {'dave': {...}} {'stewart': {...}} ''' - if 'expr_form' in kwargs: - salt.utils.warn_until( - 'Fluorine', - 'The target type should be passed using the \'tgt_type\' ' - 'argument instead of \'expr_form\'. Support for using ' - '\'expr_form\' will be removed in Salt Fluorine.' + if u'expr_form' in kwargs: + salt.utils.versions.warn_until( + u'Fluorine', + u'The target type should be passed using the \'tgt_type\' ' + u'argument instead of \'expr_form\'. Support for using ' + u'\'expr_form\' will be removed in Salt Fluorine.' ) - tgt_type = kwargs.pop('expr_form') + tgt_type = kwargs.pop(u'expr_form') import salt.cli.batch arg = salt.utils.args.condition_input(arg, kwarg) - opts = {'tgt': tgt, - 'fun': fun, - 'arg': arg, - 'tgt_type': tgt_type, - 'ret': ret, - 'batch': batch, - 'failhard': kwargs.get('failhard', False), - 'raw': kwargs.get('raw', False)} + opts = {u'tgt': tgt, + u'fun': fun, + u'arg': arg, + u'tgt_type': tgt_type, + u'ret': ret, + u'batch': batch, + u'failhard': kwargs.get(u'failhard', False), + u'raw': kwargs.get(u'raw', False)} - if 'timeout' in kwargs: - opts['timeout'] = kwargs['timeout'] - if 'gather_job_timeout' in kwargs: - opts['gather_job_timeout'] = kwargs['gather_job_timeout'] + if u'timeout' in kwargs: + opts[u'timeout'] = kwargs[u'timeout'] + if u'gather_job_timeout' in kwargs: + opts[u'gather_job_timeout'] = kwargs[u'gather_job_timeout'] + if u'batch_wait' in kwargs: + opts[u'batch_wait'] = int(kwargs[u'batch_wait']) eauth = {} - if 'eauth' in kwargs: - eauth['eauth'] = kwargs.pop('eauth') - if 'username' in kwargs: - eauth['username'] = kwargs.pop('username') - if 'password' in kwargs: - eauth['password'] = kwargs.pop('password') - if 'token' in kwargs: - eauth['token'] = kwargs.pop('token') + if u'eauth' in kwargs: + eauth[u'eauth'] = kwargs.pop(u'eauth') + if u'username' in kwargs: + eauth[u'username'] = kwargs.pop(u'username') + if u'password' in kwargs: + eauth[u'password'] = kwargs.pop(u'password') + if u'token' in kwargs: + eauth[u'token'] = kwargs.pop(u'token') for key, val in six.iteritems(self.opts): if key not in opts: @@ -590,9 +594,9 @@ class LocalClient(object): fun, arg=(), timeout=None, - tgt_type='glob', - ret='', - jid='', + tgt_type=u'glob', + ret=u'', + jid=u'', full_return=False, kwarg=None, **kwargs): @@ -699,14 +703,14 @@ class LocalClient(object): minion ID. A compound command will return a sub-dictionary keyed by function name. ''' - if 'expr_form' in kwargs: - salt.utils.warn_until( - 'Fluorine', - 'The target type should be passed using the \'tgt_type\' ' - 'argument instead of \'expr_form\'. Support for using ' - '\'expr_form\' will be removed in Salt Fluorine.' + if u'expr_form' in kwargs: + salt.utils.versions.warn_until( + u'Fluorine', + u'The target type should be passed using the \'tgt_type\' ' + u'argument instead of \'expr_form\'. Support for using ' + u'\'expr_form\' will be removed in Salt Fluorine.' ) - tgt_type = kwargs.pop('expr_form') + tgt_type = kwargs.pop(u'expr_form') arg = salt.utils.args.condition_input(arg, kwarg) was_listening = self.event.cpub @@ -727,8 +731,8 @@ class LocalClient(object): ret = {} for fn_ret in self.get_cli_event_returns( - pub_data['jid'], - pub_data['minions'], + pub_data[u'jid'], + pub_data[u'minions'], self._get_timeout(timeout), tgt, tgt_type, @@ -737,9 +741,9 @@ class LocalClient(object): if fn_ret: for mid, data in six.iteritems(fn_ret): ret[mid] = (data if full_return - else data.get('ret', {})) + else data.get(u'ret', {})) - for failed in list(set(pub_data['minions']) ^ set(ret)): + for failed in list(set(pub_data[u'minions']) - set(ret)): ret[failed] = False return ret finally: @@ -752,8 +756,8 @@ class LocalClient(object): fun, arg=(), timeout=None, - tgt_type='glob', - ret='', + tgt_type=u'glob', + ret=u'', verbose=False, kwarg=None, progress=False, @@ -768,14 +772,14 @@ class LocalClient(object): :param verbose: Print extra information about the running command :returns: A generator ''' - if 'expr_form' in kwargs: - salt.utils.warn_until( - 'Fluorine', - 'The target type should be passed using the \'tgt_type\' ' - 'argument instead of \'expr_form\'. Support for using ' - '\'expr_form\' will be removed in Salt Fluorine.' + if u'expr_form' in kwargs: + salt.utils.versions.warn_until( + u'Fluorine', + u'The target type should be passed using the \'tgt_type\' ' + u'argument instead of \'expr_form\'. Support for using ' + u'\'expr_form\' will be removed in Salt Fluorine.' ) - tgt_type = kwargs.pop('expr_form') + tgt_type = kwargs.pop(u'expr_form') arg = salt.utils.args.condition_input(arg, kwarg) was_listening = self.event.cpub @@ -796,8 +800,8 @@ class LocalClient(object): else: try: for fn_ret in self.get_cli_event_returns( - self.pub_data['jid'], - self.pub_data['minions'], + self.pub_data[u'jid'], + self.pub_data[u'minions'], self._get_timeout(timeout), tgt, tgt_type, @@ -811,14 +815,14 @@ class LocalClient(object): yield fn_ret except KeyboardInterrupt: raise SystemExit( - '\n' - 'This job\'s jid is: {0}\n' - 'Exiting gracefully on Ctrl-c\n' - 'The minions may not have all finished running and any ' - 'remaining minions will return upon completion. To look ' - 'up the return data for this job later, run the following ' - 'command:\n\n' - 'salt-run jobs.lookup_jid {0}'.format(self.pub_data['jid']) + u'\n' + u'This job\'s jid is: {0}\n' + u'Exiting gracefully on Ctrl-c\n' + u'The minions may not have all finished running and any ' + u'remaining minions will return upon completion. To look ' + u'up the return data for this job later, run the following ' + u'command:\n\n' + u'salt-run jobs.lookup_jid {0}'.format(self.pub_data[u'jid']) ) finally: if not was_listening: @@ -830,8 +834,8 @@ class LocalClient(object): fun, arg=(), timeout=None, - tgt_type='glob', - ret='', + tgt_type=u'glob', + ret=u'', kwarg=None, **kwargs): ''' @@ -851,14 +855,14 @@ class LocalClient(object): {'dave': {'ret': True}} {'stewart': {'ret': True}} ''' - if 'expr_form' in kwargs: - salt.utils.warn_until( - 'Fluorine', - 'The target type should be passed using the \'tgt_type\' ' - 'argument instead of \'expr_form\'. Support for using ' - '\'expr_form\' will be removed in Salt Fluorine.' + if u'expr_form' in kwargs: + salt.utils.versions.warn_until( + u'Fluorine', + u'The target type should be passed using the \'tgt_type\' ' + u'argument instead of \'expr_form\'. Support for using ' + u'\'expr_form\' will be removed in Salt Fluorine.' ) - tgt_type = kwargs.pop('expr_form') + tgt_type = kwargs.pop(u'expr_form') arg = salt.utils.args.condition_input(arg, kwarg) was_listening = self.event.cpub @@ -877,10 +881,10 @@ class LocalClient(object): if not pub_data: yield pub_data else: - if kwargs.get('yield_pub_data'): + if kwargs.get(u'yield_pub_data'): yield pub_data - for fn_ret in self.get_iter_returns(pub_data['jid'], - pub_data['minions'], + for fn_ret in self.get_iter_returns(pub_data[u'jid'], + pub_data[u'minions'], timeout=self._get_timeout(timeout), tgt=tgt, tgt_type=tgt_type, @@ -888,7 +892,7 @@ class LocalClient(object): if not fn_ret: continue yield fn_ret - self._clean_up_subscriptions(pub_data['jid']) + self._clean_up_subscriptions(pub_data[u'jid']) finally: if not was_listening: self.event.close_pub() @@ -899,8 +903,8 @@ class LocalClient(object): fun, arg=(), timeout=None, - tgt_type='glob', - ret='', + tgt_type=u'glob', + ret=u'', kwarg=None, show_jid=False, verbose=False, @@ -927,14 +931,14 @@ class LocalClient(object): None {'stewart': {'ret': True}} ''' - if 'expr_form' in kwargs: - salt.utils.warn_until( - 'Fluorine', - 'The target type should be passed using the \'tgt_type\' ' - 'argument instead of \'expr_form\'. Support for using ' - '\'expr_form\' will be removed in Salt Fluorine.' + if u'expr_form' in kwargs: + salt.utils.versions.warn_until( + u'Fluorine', + u'The target type should be passed using the \'tgt_type\' ' + u'argument instead of \'expr_form\'. Support for using ' + u'\'expr_form\' will be removed in Salt Fluorine.' ) - tgt_type = kwargs.pop('expr_form') + tgt_type = kwargs.pop(u'expr_form') arg = salt.utils.args.condition_input(arg, kwarg) was_listening = self.event.cpub @@ -953,8 +957,8 @@ class LocalClient(object): if not pub_data: yield pub_data else: - for fn_ret in self.get_iter_returns(pub_data['jid'], - pub_data['minions'], + for fn_ret in self.get_iter_returns(pub_data[u'jid'], + pub_data[u'minions'], timeout=timeout, tgt=tgt, tgt_type=tgt_type, @@ -962,10 +966,10 @@ class LocalClient(object): **kwargs): if fn_ret and any([show_jid, verbose]): for minion in fn_ret: - fn_ret[minion]['jid'] = pub_data['jid'] + fn_ret[minion][u'jid'] = pub_data[u'jid'] yield fn_ret - self._clean_up_subscriptions(pub_data['jid']) + self._clean_up_subscriptions(pub_data[u'jid']) finally: if not was_listening: self.event.close_pub() @@ -976,22 +980,22 @@ class LocalClient(object): fun, arg=(), timeout=None, - tgt_type='glob', - ret='', + tgt_type=u'glob', + ret=u'', verbose=False, kwarg=None, **kwargs): ''' Execute a salt command and return ''' - if 'expr_form' in kwargs: - salt.utils.warn_until( - 'Fluorine', - 'The target type should be passed using the \'tgt_type\' ' - 'argument instead of \'expr_form\'. Support for using ' - '\'expr_form\' will be removed in Salt Fluorine.' + if u'expr_form' in kwargs: + salt.utils.versions.warn_until( + u'Fluorine', + u'The target type should be passed using the \'tgt_type\' ' + u'argument instead of \'expr_form\'. Support for using ' + u'\'expr_form\' will be removed in Salt Fluorine.' ) - tgt_type = kwargs.pop('expr_form') + tgt_type = kwargs.pop(u'expr_form') arg = salt.utils.args.condition_input(arg, kwarg) was_listening = self.event.cpub @@ -1010,8 +1014,8 @@ class LocalClient(object): if not pub_data: return pub_data - return (self.get_cli_static_event_returns(pub_data['jid'], - pub_data['minions'], + return (self.get_cli_static_event_returns(pub_data[u'jid'], + pub_data[u'minions'], timeout, tgt, tgt_type, @@ -1025,8 +1029,8 @@ class LocalClient(object): jid, minions, timeout=None, - tgt='*', - tgt_type='glob', + tgt=u'*', + tgt_type=u'glob', verbose=False, show_jid=False, **kwargs): @@ -1035,23 +1039,23 @@ class LocalClient(object): :returns: all of the information for the JID ''' - if 'expr_form' in kwargs: - salt.utils.warn_until( - 'Fluorine', - 'The target type should be passed using the \'tgt_type\' ' - 'argument instead of \'expr_form\'. Support for using ' - '\'expr_form\' will be removed in Salt Fluorine.' + if u'expr_form' in kwargs: + salt.utils.versions.warn_until( + u'Fluorine', + u'The target type should be passed using the \'tgt_type\' ' + u'argument instead of \'expr_form\'. Support for using ' + u'\'expr_form\' will be removed in Salt Fluorine.' ) - tgt_type = kwargs.pop('expr_form') + tgt_type = kwargs.pop(u'expr_form') if verbose: - msg = 'Executing job with jid {0}'.format(jid) + msg = u'Executing job with jid {0}'.format(jid) print(msg) - print('-' * len(msg) + '\n') + print(u'-' * len(msg) + u'\n') elif show_jid: - print('jid: {0}'.format(jid)) + print(u'jid: {0}'.format(jid)) if timeout is None: - timeout = self.opts['timeout'] + timeout = self.opts[u'timeout'] fret = {} # make sure the minions is a set (since we do set operations on it) minions = set(minions) @@ -1104,8 +1108,8 @@ class LocalClient(object): jid, minions, timeout=None, - tgt='*', - tgt_type='glob', + tgt=u'*', + tgt_type=u'glob', expect_minions=False, block=True, **kwargs): @@ -1114,14 +1118,14 @@ class LocalClient(object): :returns: all of the information for the JID ''' - if 'expr_form' in kwargs: - salt.utils.warn_until( - 'Fluorine', - 'The target type should be passed using the \'tgt_type\' ' - 'argument instead of \'expr_form\'. Support for using ' - '\'expr_form\' will be removed in Salt Fluorine.' + if u'expr_form' in kwargs: + salt.utils.versions.warn_until( + u'Fluorine', + u'The target type should be passed using the \'tgt_type\' ' + u'argument instead of \'expr_form\'. Support for using ' + u'\'expr_form\' will be removed in Salt Fluorine.' ) - tgt_type = kwargs.pop('expr_form') + tgt_type = kwargs.pop(u'expr_form') if not isinstance(minions, set): if isinstance(minions, six.string_types): @@ -1130,44 +1134,44 @@ class LocalClient(object): minions = set(list(minions)) if timeout is None: - timeout = self.opts['timeout'] - gather_job_timeout = int(kwargs.get('gather_job_timeout', self.opts['gather_job_timeout'])) + timeout = self.opts[u'timeout'] + gather_job_timeout = int(kwargs.get(u'gather_job_timeout', self.opts[u'gather_job_timeout'])) start = int(time.time()) # timeouts per minion, id_ -> timeout time minion_timeouts = {} found = set() + missing = [] # Check to see if the jid is real, if not return the empty dict try: - if self.returners['{0}.get_load'.format(self.opts['master_job_cache'])](jid) == {}: - log.warning('jid does not exist') + if self.returners[u'{0}.get_load'.format(self.opts[u'master_job_cache'])](jid) == {}: + log.warning(u'jid does not exist') yield {} # stop the iteration, since the jid is invalid raise StopIteration() except Exception as exc: - log.warning('Returner unavailable: {exc}'.format(exc=exc)) + log.warning(u'Returner unavailable: %s', exc) # Wait for the hosts to check in last_time = False # iterator for this job's return - if self.opts['order_masters']: + if self.opts[u'order_masters']: # If we are a MoM, we need to gather expected minions from downstreams masters. - ret_iter = self.get_returns_no_block('(salt/job|syndic/.*)/{0}'.format(jid), 'regex') + ret_iter = self.get_returns_no_block(u'(salt/job|syndic/.*)/{0}'.format(jid), u'regex') else: - ret_iter = self.get_returns_no_block('salt/job/{0}'.format(jid)) + ret_iter = self.get_returns_no_block(u'salt/job/{0}'.format(jid)) # iterator for the info of this job jinfo_iter = [] # open event jids that need to be un-subscribed from later open_jids = set() timeout_at = time.time() + timeout - gather_syndic_wait = time.time() + self.opts['syndic_wait'] + gather_syndic_wait = time.time() + self.opts[u'syndic_wait'] # are there still minions running the job out there # start as True so that we ping at least once minions_running = True log.debug( - 'get_iter_returns for jid {0} sent to {1} will timeout at {2}'.format( - jid, minions, datetime.fromtimestamp(timeout_at).time() - ) + u'get_iter_returns for jid %s sent to %s will timeout at %s', + jid, minions, datetime.fromtimestamp(timeout_at).time() ) while True: # Process events until timeout is reached or all minions have returned @@ -1175,34 +1179,36 @@ class LocalClient(object): # if we got None, then there were no events if raw is None: break - if 'minions' in raw.get('data', {}): - minions.update(raw['data']['minions']) + if u'minions' in raw.get(u'data', {}): + minions.update(raw[u'data'][u'minions']) + if u'missing' in raw.get(u'data', {}): + missing.extend(raw[u'data'][u'missing']) continue - if 'return' not in raw['data']: + if u'return' not in raw[u'data']: continue - if kwargs.get('raw', False): - found.add(raw['data']['id']) + if kwargs.get(u'raw', False): + found.add(raw[u'data'][u'id']) yield raw else: - found.add(raw['data']['id']) - ret = {raw['data']['id']: {'ret': raw['data']['return']}} - if 'out' in raw['data']: - ret[raw['data']['id']]['out'] = raw['data']['out'] - if 'retcode' in raw['data']: - ret[raw['data']['id']]['retcode'] = raw['data']['retcode'] - if 'jid' in raw['data']: - ret[raw['data']['id']]['jid'] = raw['data']['jid'] - if kwargs.get('_cmd_meta', False): - ret[raw['data']['id']].update(raw['data']) - log.debug('jid {0} return from {1}'.format(jid, raw['data']['id'])) + found.add(raw[u'data'][u'id']) + ret = {raw[u'data'][u'id']: {u'ret': raw[u'data'][u'return']}} + if u'out' in raw[u'data']: + ret[raw[u'data'][u'id']][u'out'] = raw[u'data'][u'out'] + if u'retcode' in raw[u'data']: + ret[raw[u'data'][u'id']][u'retcode'] = raw[u'data'][u'retcode'] + if u'jid' in raw[u'data']: + ret[raw[u'data'][u'id']][u'jid'] = raw[u'data'][u'jid'] + if kwargs.get(u'_cmd_meta', False): + ret[raw[u'data'][u'id']].update(raw[u'data']) + log.debug(u'jid %s return from %s', jid, raw[u'data'][u'id']) yield ret # if we have all of the returns (and we aren't a syndic), no need for anything fancy - if len(found.intersection(minions)) >= len(minions) and not self.opts['order_masters']: + if len(found.intersection(minions)) >= len(minions) and not self.opts[u'order_masters']: # All minions have returned, break out of the loop - log.debug('jid {0} found all minions {1}'.format(jid, found)) + log.debug(u'jid %s found all minions %s', jid, found) break - elif len(found.intersection(minions)) >= len(minions) and self.opts['order_masters']: + elif len(found.intersection(minions)) >= len(minions) and self.opts[u'order_masters']: if len(found) >= len(minions) and len(minions) > 0 and time.time() > gather_syndic_wait: # There were some minions to find and we found them # However, this does not imply that *all* masters have yet responded with expected minion lists. @@ -1224,18 +1230,18 @@ class LocalClient(object): # re-do the ping if time.time() > timeout_at and minions_running: # since this is a new ping, no one has responded yet - jinfo = self.gather_job_info(jid, list(minions - found), 'list', **kwargs) + jinfo = self.gather_job_info(jid, list(minions - found), u'list', **kwargs) minions_running = False # if we weren't assigned any jid that means the master thinks # we have nothing to send - if 'jid' not in jinfo: + if u'jid' not in jinfo: jinfo_iter = [] else: - jinfo_iter = self.get_returns_no_block('salt/job/{0}'.format(jinfo['jid'])) + jinfo_iter = self.get_returns_no_block(u'salt/job/{0}'.format(jinfo[u'jid'])) timeout_at = time.time() + gather_job_timeout # if you are a syndic, wait a little longer - if self.opts['order_masters']: - timeout_at += self.opts.get('syndic_wait', 1) + if self.opts[u'order_masters']: + timeout_at += self.opts.get(u'syndic_wait', 1) # check for minions that are running the job still for raw in jinfo_iter: @@ -1243,48 +1249,48 @@ class LocalClient(object): if raw is None: break try: - if raw['data']['retcode'] > 0: - log.error('saltutil returning errors on minion {0}'.format(raw['data']['id'])) - minions.remove(raw['data']['id']) + if raw[u'data'][u'retcode'] > 0: + log.error(u'saltutil returning errors on minion %s', raw[u'data'][u'id']) + minions.remove(raw[u'data'][u'id']) break except KeyError as exc: # This is a safe pass. We're just using the try/except to # avoid having to deep-check for keys. - missing_key = exc.__str__().strip('\'"') - if missing_key == 'retcode': - log.debug('retcode missing from client return') + missing_key = exc.__str__().strip(u'\'"') + if missing_key == u'retcode': + log.debug(u'retcode missing from client return') else: log.debug( - 'Passing on saltutil error. Key \'%s\' missing ' - 'from client return. This may be an error in ' - 'the client.', missing_key + u'Passing on saltutil error. Key \'%s\' missing ' + u'from client return. This may be an error in ' + u'the client.', missing_key ) # Keep track of the jid events to unsubscribe from later - open_jids.add(jinfo['jid']) + open_jids.add(jinfo[u'jid']) # TODO: move to a library?? - if 'minions' in raw.get('data', {}): - minions.update(raw['data']['minions']) + if u'minions' in raw.get(u'data', {}): + minions.update(raw[u'data'][u'minions']) continue - if 'syndic' in raw.get('data', {}): - minions.update(raw['syndic']) + if u'syndic' in raw.get(u'data', {}): + minions.update(raw[u'syndic']) continue - if 'return' not in raw.get('data', {}): + if u'return' not in raw.get(u'data', {}): continue # if the job isn't running there anymore... don't count - if raw['data']['return'] == {}: + if raw[u'data'][u'return'] == {}: continue - if 'return' in raw['data']['return'] and \ - raw['data']['return']['return'] == {}: + if u'return' in raw[u'data'][u'return'] and \ + raw[u'data'][u'return'][u'return'] == {}: continue # if we didn't originally target the minion, lets add it to the list - if raw['data']['id'] not in minions: - minions.add(raw['data']['id']) + if raw[u'data'][u'id'] not in minions: + minions.add(raw[u'data'][u'id']) # update this minion's timeout, as long as the job is still running - minion_timeouts[raw['data']['id']] = time.time() + timeout + minion_timeouts[raw[u'data'][u'id']] = time.time() + timeout # a minion returned, so we know its running somewhere minions_running = True @@ -1316,6 +1322,10 @@ class LocalClient(object): if expect_minions: for minion in list((minions - found)): + yield {minion: {u'failed': True}} + + if missing: + for minion in missing: yield {minion: {'failed': True}} def get_returns( @@ -1328,49 +1338,47 @@ class LocalClient(object): ''' minions = set(minions) if timeout is None: - timeout = self.opts['timeout'] + timeout = self.opts[u'timeout'] start = int(time.time()) timeout_at = start + timeout log.debug( - 'get_returns for jid {0} sent to {1} will timeout at {2}'.format( - jid, minions, datetime.fromtimestamp(timeout_at).time() - ) + u'get_returns for jid %s sent to %s will timeout at %s', + jid, minions, datetime.fromtimestamp(timeout_at).time() ) found = set() ret = {} # Check to see if the jid is real, if not return the empty dict try: - if self.returners['{0}.get_load'.format(self.opts['master_job_cache'])](jid) == {}: - log.warning('jid does not exist') + if self.returners[u'{0}.get_load'.format(self.opts[u'master_job_cache'])](jid) == {}: + log.warning(u'jid does not exist') return ret except Exception as exc: - raise SaltClientError('Master job cache returner [{0}] failed to verify jid. ' - 'Exception details: {1}'.format(self.opts['master_job_cache'], exc)) + raise SaltClientError(u'Master job cache returner [{0}] failed to verify jid. ' + u'Exception details: {1}'.format(self.opts[u'master_job_cache'], exc)) # Wait for the hosts to check in while True: time_left = timeout_at - int(time.time()) wait = max(1, time_left) raw = self.event.get_event(wait, jid, auto_reconnect=self.auto_reconnect) - if raw is not None and 'return' in raw: - found.add(raw['id']) - ret[raw['id']] = raw['return'] + if raw is not None and u'return' in raw: + found.add(raw[u'id']) + ret[raw[u'id']] = raw[u'return'] if len(found.intersection(minions)) >= len(minions): # All minions have returned, break out of the loop - log.debug('jid {0} found all minions'.format(jid)) + log.debug(u'jid %s found all minions', jid) break continue # Then event system timeout was reached and nothing was returned if len(found.intersection(minions)) >= len(minions): # All minions have returned, break out of the loop - log.debug('jid {0} found all minions'.format(jid)) + log.debug(u'jid %s found all minions', jid) break if int(time.time()) > timeout_at: log.info( - 'jid {0} minions {1} did not return in time'.format( - jid, (minions - found) - ) + u'jid %s minions %s did not return in time', + jid, (minions - found) ) break time.sleep(0.01) @@ -1389,20 +1397,20 @@ class LocalClient(object): event_iter = self.get_event_iter_returns(jid, minions, timeout=timeout) try: - data = self.returners['{0}.get_jid'.format(self.opts['master_job_cache'])](jid) + data = self.returners[u'{0}.get_jid'.format(self.opts[u'master_job_cache'])](jid) except Exception as exc: - raise SaltClientError('Returner {0} could not fetch jid data. ' - 'Exception details: {1}'.format( - self.opts['master_job_cache'], + raise SaltClientError(u'Returner {0} could not fetch jid data. ' + u'Exception details: {1}'.format( + self.opts[u'master_job_cache'], exc)) for minion in data: m_data = {} if u'return' in data[minion]: - m_data['ret'] = data[minion].get(u'return') + m_data[u'ret'] = data[minion].get(u'return') else: - m_data['ret'] = data[minion].get('return') - if 'out' in data[minion]: - m_data['out'] = data[minion]['out'] + m_data[u'ret'] = data[minion].get(u'return') + if u'out' in data[minion]: + m_data[u'out'] = data[minion][u'out'] if minion in ret: ret[minion].update(m_data) else: @@ -1438,20 +1446,20 @@ class LocalClient(object): ret = {} try: - data = self.returners['{0}.get_jid'.format(self.opts['master_job_cache'])](jid) + data = self.returners[u'{0}.get_jid'.format(self.opts[u'master_job_cache'])](jid) except Exception as exc: - raise SaltClientError('Could not examine master job cache. ' - 'Error occurred in {0} returner. ' - 'Exception details: {1}'.format(self.opts['master_job_cache'], + raise SaltClientError(u'Could not examine master job cache. ' + u'Error occurred in {0} returner. ' + u'Exception details: {1}'.format(self.opts[u'master_job_cache'], exc)) for minion in data: m_data = {} if u'return' in data[minion]: - m_data['ret'] = data[minion].get(u'return') + m_data[u'ret'] = data[minion].get(u'return') else: - m_data['ret'] = data[minion].get('return') - if 'out' in data[minion]: - m_data['out'] = data[minion]['out'] + m_data[u'ret'] = data[minion].get(u'return') + if u'out' in data[minion]: + m_data[u'out'] = data[minion][u'out'] if minion in ret: ret[minion].update(m_data) else: @@ -1464,25 +1472,25 @@ class LocalClient(object): jid, minions, timeout=None, - tgt='*', - tgt_type='glob', + tgt=u'*', + tgt_type=u'glob', verbose=False, show_timeout=False, show_jid=False): ''' Get the returns for the command line interface via the event system ''' - log.trace('entered - function get_cli_static_event_returns()') + log.trace(u'entered - function get_cli_static_event_returns()') minions = set(minions) if verbose: - msg = 'Executing job with jid {0}'.format(jid) + msg = u'Executing job with jid {0}'.format(jid) print(msg) - print('-' * len(msg) + '\n') + print(u'-' * len(msg) + u'\n') elif show_jid: - print('jid: {0}'.format(jid)) + print(u'jid: {0}'.format(jid)) if timeout is None: - timeout = self.opts['timeout'] + timeout = self.opts[u'timeout'] start = int(time.time()) timeout_at = start + timeout @@ -1490,13 +1498,13 @@ class LocalClient(object): ret = {} # Check to see if the jid is real, if not return the empty dict try: - if self.returners['{0}.get_load'.format(self.opts['master_job_cache'])](jid) == {}: - log.warning('jid does not exist') + if self.returners[u'{0}.get_load'.format(self.opts[u'master_job_cache'])](jid) == {}: + log.warning(u'jid does not exist') return ret except Exception as exc: - raise SaltClientError('Load could not be retrieved from ' - 'returner {0}. Exception details: {1}'.format( - self.opts['master_job_cache'], + raise SaltClientError(u'Load could not be retrieved from ' + u'returner {0}. Exception details: {1}'.format( + self.opts[u'master_job_cache'], exc)) # Wait for the hosts to check in while True: @@ -1504,17 +1512,17 @@ class LocalClient(object): time_left = timeout_at - int(time.time()) # Wait 0 == forever, use a minimum of 1s wait = max(1, time_left) - jid_tag = 'salt/job/{0}'.format(jid) + jid_tag = u'salt/job/{0}'.format(jid) raw = self.event.get_event(wait, jid_tag, auto_reconnect=self.auto_reconnect) - if raw is not None and 'return' in raw: - if 'minions' in raw.get('data', {}): - minions.update(raw['data']['minions']) + if raw is not None and u'return' in raw: + if u'minions' in raw.get(u'data', {}): + minions.update(raw[u'data'][u'minions']) continue - found.add(raw['id']) - ret[raw['id']] = {'ret': raw['return']} - ret[raw['id']]['success'] = raw.get('success', False) - if 'out' in raw: - ret[raw['id']]['out'] = raw['out'] + found.add(raw[u'id']) + ret[raw[u'id']] = {u'ret': raw[u'return']} + ret[raw[u'id']][u'success'] = raw.get(u'success', False) + if u'out' in raw: + ret[raw[u'id']][u'out'] = raw[u'out'] if len(found.intersection(minions)) >= len(minions): # All minions have returned, break out of the loop break @@ -1525,14 +1533,14 @@ class LocalClient(object): break if int(time.time()) > timeout_at: if verbose or show_timeout: - if self.opts.get('minion_data_cache', False) \ - or tgt_type in ('glob', 'pcre', 'list'): + if self.opts.get(u'minion_data_cache', False) \ + or tgt_type in (u'glob', u'pcre', u'list'): if len(found) < len(minions): fail = sorted(list(minions.difference(found))) for minion in fail: ret[minion] = { - 'out': 'no_return', - 'ret': 'Minion did not return' + u'out': u'no_return', + u'ret': u'Minion did not return' } break time.sleep(0.01) @@ -1545,8 +1553,8 @@ class LocalClient(object): jid, minions, timeout=None, - tgt='*', - tgt_type='glob', + tgt=u'*', + tgt_type=u'glob', verbose=False, progress=False, show_timeout=False, @@ -1555,23 +1563,23 @@ class LocalClient(object): ''' Get the returns for the command line interface via the event system ''' - log.trace('func get_cli_event_returns()') + log.trace(u'func get_cli_event_returns()') - if 'expr_form' in kwargs: - salt.utils.warn_until( - 'Fluorine', - 'The target type should be passed using the \'tgt_type\' ' - 'argument instead of \'expr_form\'. Support for using ' - '\'expr_form\' will be removed in Salt Fluorine.' + if u'expr_form' in kwargs: + salt.utils.versions.warn_until( + u'Fluorine', + u'The target type should be passed using the \'tgt_type\' ' + u'argument instead of \'expr_form\'. Support for using ' + u'\'expr_form\' will be removed in Salt Fluorine.' ) - tgt_type = kwargs.pop('expr_form') + tgt_type = kwargs.pop(u'expr_form') if verbose: - msg = 'Executing job with jid {0}'.format(jid) + msg = u'Executing job with jid {0}'.format(jid) print(msg) - print('-' * len(msg) + '\n') + print(u'-' * len(msg) + u'\n') elif show_jid: - print('jid: {0}'.format(jid)) + print(u'jid: {0}'.format(jid)) # lazy load the connected minions connected_minions = None @@ -1585,29 +1593,29 @@ class LocalClient(object): expect_minions=(verbose or show_timeout), **kwargs ): - log.debug('return event: %s', ret) + log.debug(u'return event: %s', ret) return_count = return_count + 1 if progress: for id_, min_ret in six.iteritems(ret): - if not min_ret.get('failed') is True: - yield {'minion_count': len(minions), 'return_count': return_count} + if not min_ret.get(u'failed') is True: + yield {u'minion_count': len(minions), u'return_count': return_count} # replace the return structure for missing minions for id_, min_ret in six.iteritems(ret): - if min_ret.get('failed') is True: + if min_ret.get(u'failed') is True: if connected_minions is None: connected_minions = salt.utils.minions.CkMinions(self.opts).connected_ids() - if self.opts['minion_data_cache'] \ - and salt.cache.factory(self.opts).contains('minions/{0}'.format(id_), 'data') \ + if self.opts[u'minion_data_cache'] \ + and salt.cache.factory(self.opts).contains(u'minions/{0}'.format(id_), u'data') \ and connected_minions \ and id_ not in connected_minions: - yield {id_: {'out': 'no_return', - 'ret': 'Minion did not return. [Not connected]'}} + yield {id_: {u'out': u'no_return', + u'ret': u'Minion did not return. [Not connected]'}} else: # don't report syndics as unresponsive minions - if not os.path.exists(os.path.join(self.opts['syndic_dir'], id_)): - yield {id_: {'out': 'no_return', - 'ret': 'Minion did not return. [No response]'}} + if not os.path.exists(os.path.join(self.opts[u'syndic_dir'], id_)): + yield {id_: {u'out': u'no_return', + u'ret': u'Minion did not return. [No response]'}} else: yield {id_: min_ret} @@ -1618,16 +1626,16 @@ class LocalClient(object): Gather the return data from the event system, break hard when timeout is reached. ''' - log.trace('entered - function get_event_iter_returns()') + log.trace(u'entered - function get_event_iter_returns()') if timeout is None: - timeout = self.opts['timeout'] + timeout = self.opts[u'timeout'] timeout_at = time.time() + timeout found = set() # Check to see if the jid is real, if not return the empty dict - if self.returners['{0}.get_load'.format(self.opts['master_job_cache'])](jid) == {}: - log.warning('jid does not exist') + if self.returners[u'{0}.get_load'.format(self.opts[u'master_job_cache'])](jid) == {}: + log.warning(u'jid does not exist') yield {} # stop the iteration, since the jid is invalid raise StopIteration() @@ -1637,16 +1645,16 @@ class LocalClient(object): if raw is None or time.time() > timeout_at: # Timeout reached break - if 'minions' in raw.get('data', {}): + if u'minions' in raw.get(u'data', {}): continue try: - found.add(raw['id']) - ret = {raw['id']: {'ret': raw['return']}} + found.add(raw[u'id']) + ret = {raw[u'id']: {u'ret': raw[u'return']}} except KeyError: # Ignore other erroneous messages continue - if 'out' in raw: - ret[raw['id']]['out'] = raw['out'] + if u'out' in raw: + ret[raw[u'id']][u'out'] = raw[u'out'] yield ret time.sleep(0.02) @@ -1662,59 +1670,59 @@ class LocalClient(object): ''' Set up the payload_kwargs to be sent down to the master ''' - if tgt_type == 'nodegroup': - if tgt not in self.opts['nodegroups']: + if tgt_type == u'nodegroup': + if tgt not in self.opts[u'nodegroups']: conf_file = self.opts.get( - 'conf_file', 'the master config file' + u'conf_file', u'the master config file' ) raise SaltInvocationError( - 'Node group {0} unavailable in {1}'.format( + u'Node group {0} unavailable in {1}'.format( tgt, conf_file ) ) tgt = salt.utils.minions.nodegroup_comp(tgt, - self.opts['nodegroups']) - tgt_type = 'compound' + self.opts[u'nodegroups']) + tgt_type = u'compound' # Convert a range expression to a list of nodes and change expression # form to list - if tgt_type == 'range' and HAS_RANGE: + if tgt_type == u'range' and HAS_RANGE: tgt = self._convert_range_to_list(tgt) - tgt_type = 'list' + tgt_type = u'list' # If an external job cache is specified add it to the ret list - if self.opts.get('ext_job_cache'): + if self.opts.get(u'ext_job_cache'): if ret: - ret += ',{0}'.format(self.opts['ext_job_cache']) + ret += u',{0}'.format(self.opts[u'ext_job_cache']) else: - ret = self.opts['ext_job_cache'] + ret = self.opts[u'ext_job_cache'] # format the payload - make a function that does this in the payload # module # Generate the standard keyword args to feed to format_payload - payload_kwargs = {'cmd': 'publish', - 'tgt': tgt, - 'fun': fun, - 'arg': arg, - 'key': self.key, - 'tgt_type': tgt_type, - 'ret': ret, - 'jid': jid} + payload_kwargs = {u'cmd': u'publish', + u'tgt': tgt, + u'fun': fun, + u'arg': arg, + u'key': self.key, + u'tgt_type': tgt_type, + u'ret': ret, + u'jid': jid} # if kwargs are passed, pack them. if kwargs: - payload_kwargs['kwargs'] = kwargs + payload_kwargs[u'kwargs'] = kwargs # If we have a salt user, add it to the payload - if self.opts['syndic_master'] and 'user' in kwargs: - payload_kwargs['user'] = kwargs['user'] + if self.opts[u'syndic_master'] and u'user' in kwargs: + payload_kwargs[u'user'] = kwargs[u'user'] elif self.salt_user: - payload_kwargs['user'] = self.salt_user + payload_kwargs[u'user'] = self.salt_user # If we're a syndication master, pass the timeout - if self.opts['order_masters']: - payload_kwargs['to'] = timeout + if self.opts[u'order_masters']: + payload_kwargs[u'to'] = timeout return payload_kwargs @@ -1722,9 +1730,9 @@ class LocalClient(object): tgt, fun, arg=(), - tgt_type='glob', - ret='', - jid='', + tgt_type=u'glob', + ret=u'', + jid=u'', timeout=5, listen=False, **kwargs): @@ -1749,22 +1757,22 @@ class LocalClient(object): minions: A set, the targets that the tgt passed should match. ''' - if 'expr_form' in kwargs: - salt.utils.warn_until( - 'Fluorine', - 'The target type should be passed using the \'tgt_type\' ' - 'argument instead of \'expr_form\'. Support for using ' - '\'expr_form\' will be removed in Salt Fluorine.' + if u'expr_form' in kwargs: + salt.utils.versions.warn_until( + u'Fluorine', + u'The target type should be passed using the \'tgt_type\' ' + u'argument instead of \'expr_form\'. Support for using ' + u'\'expr_form\' will be removed in Salt Fluorine.' ) - tgt_type = kwargs.pop('expr_form') + tgt_type = kwargs.pop(u'expr_form') # Make sure the publisher is running by checking the unix socket - if (self.opts.get('ipc_mode', '') != 'tcp' and - not os.path.exists(os.path.join(self.opts['sock_dir'], - 'publish_pull.ipc'))): + if (self.opts.get(u'ipc_mode', u'') != u'tcp' and + not os.path.exists(os.path.join(self.opts[u'sock_dir'], + u'publish_pull.ipc'))): log.error( - 'Unable to connect to the salt master publisher at ' - '{0}'.format(self.opts['sock_dir']) + u'Unable to connect to the salt master publisher at %s', + self.opts[u'sock_dir'] ) raise SaltClientError @@ -1778,10 +1786,10 @@ class LocalClient(object): timeout, **kwargs) - master_uri = 'tcp://' + salt.utils.ip_bracket(self.opts['interface']) + \ - ':' + str(self.opts['ret_port']) + master_uri = u'tcp://' + salt.utils.ip_bracket(self.opts[u'interface']) + \ + u':' + str(self.opts[u'ret_port']) channel = salt.transport.Channel.factory(self.opts, - crypt='clear', + crypt=u'clear', master_uri=master_uri) try: @@ -1792,13 +1800,13 @@ class LocalClient(object): payload = channel.send(payload_kwargs, timeout=timeout) except SaltReqTimeoutError: raise SaltReqTimeoutError( - 'Salt request timed out. The master is not responding. You ' - 'may need to run your command with `--async` in order to ' - 'bypass the congested event bus. With `--async`, the CLI tool ' - 'will print the job id (jid) and exit immediately without ' - 'listening for responses. You can then use ' - '`salt-run jobs.lookup_jid` to look up the results of the job ' - 'in the job cache later.' + u'Salt request timed out. The master is not responding. You ' + u'may need to run your command with `--async` in order to ' + u'bypass the congested event bus. With `--async`, the CLI tool ' + u'will print the job id (jid) and exit immediately without ' + u'listening for responses. You can then use ' + u'`salt-run jobs.lookup_jid` to look up the results of the job ' + u'in the job cache later.' ) if not payload: @@ -1808,10 +1816,10 @@ class LocalClient(object): if key == self.key: return payload self.key = key - payload_kwargs['key'] = self.key + payload_kwargs[u'key'] = self.key payload = channel.send(payload_kwargs) - error = payload.pop('error', None) + error = payload.pop(u'error', None) if error is not None: raise PublishError(error) @@ -1821,17 +1829,17 @@ class LocalClient(object): # We have the payload, let's get rid of the channel fast(GC'ed faster) del channel - return {'jid': payload['load']['jid'], - 'minions': payload['load']['minions']} + return {u'jid': payload[u'load'][u'jid'], + u'minions': payload[u'load'][u'minions']} @tornado.gen.coroutine def pub_async(self, tgt, fun, arg=(), - tgt_type='glob', - ret='', - jid='', + tgt_type=u'glob', + ret=u'', + jid=u'', timeout=5, io_loop=None, listen=True, @@ -1857,22 +1865,22 @@ class LocalClient(object): minions: A set, the targets that the tgt passed should match. ''' - if 'expr_form' in kwargs: - salt.utils.warn_until( - 'Fluorine', - 'The target type should be passed using the \'tgt_type\' ' - 'argument instead of \'expr_form\'. Support for using ' - '\'expr_form\' will be removed in Salt Fluorine.' + if u'expr_form' in kwargs: + salt.utils.versions.warn_until( + u'Fluorine', + u'The target type should be passed using the \'tgt_type\' ' + u'argument instead of \'expr_form\'. Support for using ' + u'\'expr_form\' will be removed in Salt Fluorine.' ) - tgt_type = kwargs.pop('expr_form') + tgt_type = kwargs.pop(u'expr_form') # Make sure the publisher is running by checking the unix socket - if (self.opts.get('ipc_mode', '') != 'tcp' and - not os.path.exists(os.path.join(self.opts['sock_dir'], - 'publish_pull.ipc'))): + if (self.opts.get(u'ipc_mode', u'') != u'tcp' and + not os.path.exists(os.path.join(self.opts[u'sock_dir'], + u'publish_pull.ipc'))): log.error( - 'Unable to connect to the salt master publisher at ' - '{0}'.format(self.opts['sock_dir']) + u'Unable to connect to the salt master publisher at %s', + self.opts[u'sock_dir'] ) raise SaltClientError @@ -1886,11 +1894,11 @@ class LocalClient(object): timeout, **kwargs) - master_uri = 'tcp://' + salt.utils.ip_bracket(self.opts['interface']) + \ - ':' + str(self.opts['ret_port']) + master_uri = u'tcp://' + salt.utils.ip_bracket(self.opts[u'interface']) + \ + u':' + str(self.opts[u'ret_port']) channel = salt.transport.client.AsyncReqChannel.factory(self.opts, io_loop=io_loop, - crypt='clear', + crypt=u'clear', master_uri=master_uri) try: @@ -1901,13 +1909,13 @@ class LocalClient(object): payload = yield channel.send(payload_kwargs, timeout=timeout) except SaltReqTimeoutError: raise SaltReqTimeoutError( - 'Salt request timed out. The master is not responding. You ' - 'may need to run your command with `--async` in order to ' - 'bypass the congested event bus. With `--async`, the CLI tool ' - 'will print the job id (jid) and exit immediately without ' - 'listening for responses. You can then use ' - '`salt-run jobs.lookup_jid` to look up the results of the job ' - 'in the job cache later.' + u'Salt request timed out. The master is not responding. You ' + u'may need to run your command with `--async` in order to ' + u'bypass the congested event bus. With `--async`, the CLI tool ' + u'will print the job id (jid) and exit immediately without ' + u'listening for responses. You can then use ' + u'`salt-run jobs.lookup_jid` to look up the results of the job ' + u'in the job cache later.' ) if not payload: @@ -1917,10 +1925,10 @@ class LocalClient(object): if key == self.key: raise tornado.gen.Return(payload) self.key = key - payload_kwargs['key'] = self.key + payload_kwargs[u'key'] = self.key payload = yield channel.send(payload_kwargs) - error = payload.pop('error', None) + error = payload.pop(u'error', None) if error is not None: raise PublishError(error) @@ -1930,21 +1938,21 @@ class LocalClient(object): # We have the payload, let's get rid of the channel fast(GC'ed faster) del channel - raise tornado.gen.Return({'jid': payload['load']['jid'], - 'minions': payload['load']['minions']}) + raise tornado.gen.Return({u'jid': payload[u'load'][u'jid'], + u'minions': payload[u'load'][u'minions']}) def __del__(self): # This IS really necessary! # When running tests, if self.events is not destroyed, we leak 2 # threads per test case which uses self.client - if hasattr(self, 'event'): + if hasattr(self, u'event'): # The call below will take care of calling 'self.event.destroy()' del self.event def _clean_up_subscriptions(self, job_id): - if self.opts.get('order_masters'): - self.event.unsubscribe('syndic/.*/{0}'.format(job_id), 'regex') - self.event.unsubscribe('salt/job/{0}'.format(job_id)) + if self.opts.get(u'order_masters'): + self.event.unsubscribe(u'syndic/.*/{0}'.format(job_id), u'regex') + self.event.unsubscribe(u'salt/job/{0}'.format(job_id)) class FunctionWrapper(dict): @@ -1959,7 +1967,7 @@ class FunctionWrapper(dict): super(FunctionWrapper, self).__init__() self.opts = opts self.minion = minion - self.local = LocalClient(self.opts['conf_file']) + self.local = LocalClient(self.opts[u'conf_file']) self.functions = self.__load_functions() def __missing__(self, key): @@ -1976,7 +1984,7 @@ class FunctionWrapper(dict): Find out what functions are available on the minion ''' return set(self.local.cmd(self.minion, - 'sys.list_functions').get(self.minion, [])) + u'sys.list_functions').get(self.minion, [])) def run_key(self, key): ''' @@ -1989,7 +1997,7 @@ class FunctionWrapper(dict): ''' args = list(args) for _key, _val in kwargs: - args.append('{0}={1}'.format(_key, _val)) + args.append(u'{0}={1}'.format(_key, _val)) return self.local.cmd(self.minion, key, args) return func @@ -2032,7 +2040,7 @@ class Caller(object): __opts__['file_client'] = 'local' caller = salt.client.Caller(mopts=__opts__) ''' - def __init__(self, c_path=os.path.join(syspaths.CONFIG_DIR, 'minion'), + def __init__(self, c_path=os.path.join(syspaths.CONFIG_DIR, u'minion'), mopts=None): # Late-import of the minion module to keep the CLI as light as possible import salt.minion diff --git a/salt/client/api.py b/salt/client/api.py index 5e18b50b32d..27ad6a46f34 100644 --- a/salt/client/api.py +++ b/salt/client/api.py @@ -14,8 +14,9 @@ client applications. http://docs.saltstack.com/ref/clients/index.html ''' -from __future__ import absolute_import + # Import Python libs +from __future__ import absolute_import import os # Import Salt libs @@ -24,9 +25,9 @@ import salt.auth import salt.client import salt.runner import salt.wheel -import salt.utils +import salt.utils.args +import salt.utils.event import salt.syspaths as syspaths -from salt.utils.event import tagify from salt.exceptions import EauthAuthenticationError @@ -37,7 +38,7 @@ def tokenify(cmd, token=None): Otherwise return cmd ''' if token is not None: - cmd['token'] = token + cmd[u'token'] = token return cmd @@ -50,19 +51,19 @@ class APIClient(object): if not opts: opts = salt.config.client_config( os.environ.get( - 'SALT_MASTER_CONFIG', - os.path.join(syspaths.CONFIG_DIR, 'master') + u'SALT_MASTER_CONFIG', + os.path.join(syspaths.CONFIG_DIR, u'master') ) ) self.opts = opts - self.localClient = salt.client.get_local_client(self.opts['conf_file']) + self.localClient = salt.client.get_local_client(self.opts[u'conf_file']) self.runnerClient = salt.runner.RunnerClient(self.opts) self.wheelClient = salt.wheel.Wheel(self.opts) self.resolver = salt.auth.Resolver(self.opts) self.event = salt.utils.event.get_event( - 'master', - self.opts['sock_dir'], - self.opts['transport'], + u'master', + self.opts[u'sock_dir'], + self.opts[u'transport'], opts=self.opts, listen=listen) @@ -118,20 +119,20 @@ class APIClient(object): ''' cmd = dict(cmd) # make copy - client = 'minion' # default to local minion client - mode = cmd.get('mode', 'async') # default to 'async' + client = u'minion' # default to local minion client + mode = cmd.get(u'mode', u'async') # default to 'async' # check for wheel or runner prefix to fun name to use wheel or runner client - funparts = cmd.get('fun', '').split('.') - if len(funparts) > 2 and funparts[0] in ['wheel', 'runner']: # master + funparts = cmd.get(u'fun', u'').split(u'.') + if len(funparts) > 2 and funparts[0] in [u'wheel', u'runner']: # master client = funparts[0] - cmd['fun'] = '.'.join(funparts[1:]) # strip prefix + cmd[u'fun'] = u'.'.join(funparts[1:]) # strip prefix - if not ('token' in cmd or - ('eauth' in cmd and 'password' in cmd and 'username' in cmd)): - raise EauthAuthenticationError('No authentication credentials given') + if not (u'token' in cmd or + (u'eauth' in cmd and u'password' in cmd and u'username' in cmd)): + raise EauthAuthenticationError(u'No authentication credentials given') - executor = getattr(self, '{0}_{1}'.format(client, mode)) + executor = getattr(self, u'{0}_{1}'.format(client, mode)) result = executor(**cmd) return result @@ -204,9 +205,9 @@ class APIClient(object): Adds client per the command. ''' - cmd['client'] = 'minion' - if len(cmd['module'].split('.')) > 2 and cmd['module'].split('.')[0] in ['runner', 'wheel']: - cmd['client'] = 'master' + cmd[u'client'] = u'minion' + if len(cmd[u'module'].split(u'.')) > 2 and cmd[u'module'].split(u'.')[0] in [u'runner', u'wheel']: + cmd[u'client'] = u'master' return self._signature(cmd) def _signature(self, cmd): @@ -216,20 +217,20 @@ class APIClient(object): ''' result = {} - client = cmd.get('client', 'minion') - if client == 'minion': - cmd['fun'] = 'sys.argspec' - cmd['kwarg'] = dict(module=cmd['module']) + client = cmd.get(u'client', u'minion') + if client == u'minion': + cmd[u'fun'] = u'sys.argspec' + cmd[u'kwarg'] = dict(module=cmd[u'module']) result = self.run(cmd) - elif client == 'master': - parts = cmd['module'].split('.') + elif client == u'master': + parts = cmd[u'module'].split(u'.') client = parts[0] - module = '.'.join(parts[1:]) # strip prefix - if client == 'wheel': + module = u'.'.join(parts[1:]) # strip prefix + if client == u'wheel': functions = self.wheelClient.functions - elif client == 'runner': + elif client == u'runner': functions = self.runnerClient.functions - result = {'master': salt.utils.argspec_report(functions, module)} + result = {u'master': salt.utils.args.argspec_report(functions, module)} return result def create_token(self, creds): @@ -274,20 +275,20 @@ class APIClient(object): tokenage = self.resolver.mk_token(creds) except Exception as ex: raise EauthAuthenticationError( - "Authentication failed with {0}.".format(repr(ex))) + u"Authentication failed with {0}.".format(repr(ex))) - if 'token' not in tokenage: - raise EauthAuthenticationError("Authentication failed with provided credentials.") + if u'token' not in tokenage: + raise EauthAuthenticationError(u"Authentication failed with provided credentials.") # Grab eauth config for the current backend for the current user - tokenage_eauth = self.opts['external_auth'][tokenage['eauth']] - if tokenage['name'] in tokenage_eauth: - tokenage['perms'] = tokenage_eauth[tokenage['name']] + tokenage_eauth = self.opts[u'external_auth'][tokenage[u'eauth']] + if tokenage[u'name'] in tokenage_eauth: + tokenage[u'perms'] = tokenage_eauth[tokenage[u'name']] else: - tokenage['perms'] = tokenage_eauth['*'] + tokenage[u'perms'] = tokenage_eauth[u'*'] - tokenage['user'] = tokenage['name'] - tokenage['username'] = tokenage['name'] + tokenage[u'user'] = tokenage[u'name'] + tokenage[u'username'] = tokenage[u'name'] return tokenage @@ -300,11 +301,11 @@ class APIClient(object): result = self.resolver.get_token(token) except Exception as ex: raise EauthAuthenticationError( - "Token validation failed with {0}.".format(repr(ex))) + u"Token validation failed with {0}.".format(repr(ex))) return result - def get_event(self, wait=0.25, tag='', full=False): + def get_event(self, wait=0.25, tag=u'', full=False): ''' Get a single salt event. If no events are available, then block for up to ``wait`` seconds. @@ -322,4 +323,4 @@ class APIClient(object): Need to convert this to a master call with appropriate authentication ''' - return self.event.fire_event(data, tagify(tag, 'wui')) + return self.event.fire_event(data, salt.utils.event.tagify(tag, u'wui')) diff --git a/salt/client/mixins.py b/salt/client/mixins.py index 09680a0561d..7ddaa4d97bc 100644 --- a/salt/client/mixins.py +++ b/salt/client/mixins.py @@ -5,6 +5,7 @@ A collection of mixins useful for the various *Client interfaces # Import Python libs from __future__ import absolute_import, print_function, with_statement +import fnmatch import signal import logging import weakref @@ -15,17 +16,21 @@ import copy as pycopy # Import Salt libs import salt.exceptions import salt.minion -import salt.utils +import salt.utils # Can be removed once daemonize, get_specific_user, format_call are moved +import salt.utils.args import salt.utils.doc import salt.utils.error import salt.utils.event import salt.utils.jid import salt.utils.job import salt.utils.lazy +import salt.utils.platform import salt.utils.process +import salt.utils.state +import salt.utils.versions import salt.transport import salt.log.setup -import salt.ext.six as six +from salt.ext import six # Import 3rd-party libs import tornado.stack_context @@ -33,18 +38,18 @@ import tornado.stack_context log = logging.getLogger(__name__) CLIENT_INTERNAL_KEYWORDS = frozenset([ - 'client', - 'cmd', - 'eauth', - 'fun', - 'kwarg', - 'match', - 'token', - '__jid__', - '__tag__', - '__user__', - 'username', - 'password' + u'client', + u'cmd', + u'eauth', + u'fun', + u'kwarg', + u'match', + u'token', + u'__jid__', + u'__tag__', + u'__user__', + u'username', + u'password' ]) @@ -76,9 +81,9 @@ class ClientFuncsDict(collections.MutableMapping): raise KeyError def wrapper(*args, **kwargs): - low = {'fun': key, - 'args': args, - 'kwargs': kwargs, + low = {u'fun': key, + u'args': args, + u'kwargs': kwargs, } pub_data = {} # Copy kwargs keys so we can iterate over and pop the pub data @@ -86,18 +91,18 @@ class ClientFuncsDict(collections.MutableMapping): # pull out pub_data if you have it for kwargs_key in kwargs_keys: - if kwargs_key.startswith('__pub_'): + if kwargs_key.startswith(u'__pub_'): pub_data[kwargs_key] = kwargs.pop(kwargs_key) - async_pub = self.client._gen_async_pub(pub_data.get('__pub_jid')) + async_pub = self.client._gen_async_pub(pub_data.get(u'__pub_jid')) user = salt.utils.get_specific_user() return self.client._proc_function( key, low, user, - async_pub['tag'], # TODO: fix - async_pub['jid'], # TODO: fix + async_pub[u'tag'], # TODO: fix + async_pub[u'jid'], # TODO: fix False, # Don't daemonize ) return wrapper @@ -128,14 +133,14 @@ class SyncClientMixin(object): Execute a function through the master network interface. ''' load = kwargs - load['cmd'] = self.client + load[u'cmd'] = self.client channel = salt.transport.Channel.factory(self.opts, - crypt='clear', - usage='master_call') + crypt=u'clear', + usage=u'master_call') ret = channel.send(load) if isinstance(ret, collections.Mapping): - if 'error' in ret: - salt.utils.error.raise_error(**ret['error']) + if u'error' in ret: + salt.utils.error.raise_error(**ret[u'error']) return ret def cmd_sync(self, low, timeout=None, full_return=False): @@ -154,19 +159,19 @@ class SyncClientMixin(object): 'eauth': 'pam', }) ''' - event = salt.utils.event.get_master_event(self.opts, self.opts['sock_dir'], listen=True) + event = salt.utils.event.get_master_event(self.opts, self.opts[u'sock_dir'], listen=True) job = self.master_call(**low) - ret_tag = salt.utils.event.tagify('ret', base=job['tag']) + ret_tag = salt.utils.event.tagify(u'ret', base=job[u'tag']) if timeout is None: - timeout = self.opts.get('rest_timeout', 300) + timeout = self.opts.get(u'rest_timeout', 300) ret = event.get_event(tag=ret_tag, full=True, wait=timeout, auto_reconnect=True) if ret is None: raise salt.exceptions.SaltClientTimeout( - "RunnerClient job '{0}' timed out".format(job['jid']), - jid=job['jid']) + u"RunnerClient job '{0}' timed out".format(job[u'jid']), + jid=job[u'jid']) - return ret if full_return else ret['data']['return'] + return ret if full_return else ret[u'data'][u'return'] def cmd(self, fun, arg=None, pub_data=None, kwarg=None, print_event=True, full_return=False): ''' @@ -201,40 +206,40 @@ class SyncClientMixin(object): arg = tuple() if not isinstance(arg, list) and not isinstance(arg, tuple): raise salt.exceptions.SaltInvocationError( - 'arg must be formatted as a list/tuple' + u'arg must be formatted as a list/tuple' ) if pub_data is None: pub_data = {} if not isinstance(pub_data, dict): raise salt.exceptions.SaltInvocationError( - 'pub_data must be formatted as a dictionary' + u'pub_data must be formatted as a dictionary' ) if kwarg is None: kwarg = {} if not isinstance(kwarg, dict): raise salt.exceptions.SaltInvocationError( - 'kwarg must be formatted as a dictionary' + u'kwarg must be formatted as a dictionary' ) arglist = salt.utils.args.parse_input( arg, - no_parse=self.opts.get('no_parse', [])) + no_parse=self.opts.get(u'no_parse', [])) # if you were passed kwarg, add it to arglist if kwarg: - kwarg['__kwarg__'] = True + kwarg[u'__kwarg__'] = True arglist.append(kwarg) args, kwargs = salt.minion.load_args_and_kwargs( self.functions[fun], arglist, pub_data ) - low = {'fun': fun, - 'arg': args, - 'kwarg': kwargs} + low = {u'fun': fun, + u'arg': args, + u'kwarg': kwargs} return self.low(fun, low, print_event=print_event, full_return=full_return) @property def mminion(self): - if not hasattr(self, '_mminion'): + if not hasattr(self, u'_mminion'): self._mminion = salt.minion.MasterMinion(self.opts, states=False, rend=False) return self._mminion @@ -243,15 +248,15 @@ class SyncClientMixin(object): Check for deprecated usage and allow until Salt Oxygen. ''' msg = [] - if 'args' in low: - msg.append('call with arg instead') - low['arg'] = low.pop('args') - if 'kwargs' in low: - msg.append('call with kwarg instead') - low['kwarg'] = low.pop('kwargs') + if u'args' in low: + msg.append(u'call with arg instead') + low[u'arg'] = low.pop(u'args') + if u'kwargs' in low: + msg.append(u'call with kwarg instead') + low[u'kwarg'] = low.pop(u'kwargs') if msg: - salt.utils.warn_until('Oxygen', ' '.join(msg)) + salt.utils.versions.warn_until(u'Oxygen', u' '.join(msg)) return self._low(fun, low, print_event=print_event, full_return=full_return) @@ -265,13 +270,13 @@ class SyncClientMixin(object): class_name = self.__class__.__name__.lower() except AttributeError: log.warning( - 'Unable to determine class name', + u'Unable to determine class name', exc_info_on_loglevel=logging.DEBUG ) return True try: - return self.opts['{0}_returns'.format(class_name)] + return self.opts[u'{0}_returns'.format(class_name)] except KeyError: # No such option, assume this isn't one we care about gating and # just return True. @@ -294,24 +299,24 @@ class SyncClientMixin(object): # this is not to clutter the output with the module loading # if we have a high debug level. self.mminion # pylint: disable=W0104 - jid = low.get('__jid__', salt.utils.jid.gen_jid()) - tag = low.get('__tag__', salt.utils.event.tagify(jid, prefix=self.tag_prefix)) + jid = low.get(u'__jid__', salt.utils.jid.gen_jid(self.opts)) + tag = low.get(u'__tag__', salt.utils.event.tagify(jid, prefix=self.tag_prefix)) - data = {'fun': '{0}.{1}'.format(self.client, fun), - 'jid': jid, - 'user': low.get('__user__', 'UNKNOWN'), + data = {u'fun': u'{0}.{1}'.format(self.client, fun), + u'jid': jid, + u'user': low.get(u'__user__', u'UNKNOWN'), } event = salt.utils.event.get_event( - 'master', - self.opts['sock_dir'], - self.opts['transport'], + u'master', + self.opts[u'sock_dir'], + self.opts[u'transport'], opts=self.opts, listen=False) if print_event: print_func = self.print_async_event \ - if hasattr(self, 'print_async_event') \ + if hasattr(self, u'print_async_event') \ else None else: # Suppress printing of return event (this keeps us from printing @@ -326,12 +331,12 @@ class SyncClientMixin(object): # TODO: document these, and test that they exist # TODO: Other things to inject?? - func_globals = {'__jid__': jid, - '__user__': data['user'], - '__tag__': tag, + func_globals = {u'__jid__': jid, + u'__user__': data[u'user'], + u'__tag__': tag, # weak ref to avoid the Exception in interpreter # teardown of event - '__jid_event__': weakref.proxy(namespaced_event), + u'__jid_event__': weakref.proxy(namespaced_event), } try: @@ -343,9 +348,9 @@ class SyncClientMixin(object): completed_funcs = [] for mod_name in six.iterkeys(self_functions): - if '.' not in mod_name: + if u'.' not in mod_name: continue - mod, _ = mod_name.split('.', 1) + mod, _ = mod_name.split(u'.', 1) if mod in completed_funcs: continue completed_funcs.append(mod) @@ -361,85 +366,95 @@ class SyncClientMixin(object): # we make the transition we will load "kwargs" using format_call if # there are no kwargs in the low object passed in f_call = None - if 'arg' not in low: + if u'arg' not in low: f_call = salt.utils.format_call( self.functions[fun], low, expected_extra_kws=CLIENT_INTERNAL_KEYWORDS ) - args = f_call.get('args', ()) + args = f_call.get(u'args', ()) else: - args = low['arg'] + args = low[u'arg'] - if 'kwarg' not in low: + if u'kwarg' not in low: log.critical( - 'kwargs must be passed inside the low data within the ' - '\'kwarg\' key. See usage of ' - 'salt.utils.args.parse_input() and ' - 'salt.minion.load_args_and_kwargs() elsewhere in the ' - 'codebase.' + u'kwargs must be passed inside the low data within the ' + u'\'kwarg\' key. See usage of ' + u'salt.utils.args.parse_input() and ' + u'salt.minion.load_args_and_kwargs() elsewhere in the ' + u'codebase.' ) kwargs = {} else: - kwargs = low['kwarg'] + kwargs = low[u'kwarg'] # Update the event data with loaded args and kwargs - data['fun_args'] = list(args) + ([kwargs] if kwargs else []) - func_globals['__jid_event__'].fire_event(data, 'new') + data[u'fun_args'] = list(args) + ([kwargs] if kwargs else []) + func_globals[u'__jid_event__'].fire_event(data, u'new') # Initialize a context for executing the method. with tornado.stack_context.StackContext(self.functions.context_dict.clone): - data['return'] = self.functions[fun](*args, **kwargs) - data['success'] = True - if isinstance(data['return'], dict) and 'data' in data['return']: + data[u'return'] = self.functions[fun](*args, **kwargs) + data[u'success'] = True + if isinstance(data[u'return'], dict) and u'data' in data[u'return']: # some functions can return boolean values - data['success'] = salt.utils.check_state_result(data['return']['data']) + data[u'success'] = salt.utils.state.check_result(data[u'return'][u'data']) except (Exception, SystemExit) as ex: if isinstance(ex, salt.exceptions.NotImplemented): - data['return'] = str(ex) + data[u'return'] = str(ex) else: - data['return'] = 'Exception occurred in {0} {1}: {2}'.format( + data[u'return'] = u'Exception occurred in {0} {1}: {2}'.format( self.client, fun, traceback.format_exc(), ) - data['success'] = False - - namespaced_event.fire_event(data, 'ret') + data[u'success'] = False if self.store_job: try: salt.utils.job.store_job( self.opts, { - 'id': self.opts['id'], - 'tgt': self.opts['id'], - 'jid': data['jid'], - 'return': data, + u'id': self.opts[u'id'], + u'tgt': self.opts[u'id'], + u'jid': data[u'jid'], + u'return': data, }, event=None, mminion=self.mminion, ) except salt.exceptions.SaltCacheError: - log.error('Could not store job cache info. ' - 'Job details for this run may be unavailable.') + log.error(u'Could not store job cache info. ' + u'Job details for this run may be unavailable.') + + # Outputters _can_ mutate data so write to the job cache first! + namespaced_event.fire_event(data, u'ret') # if we fired an event, make sure to delete the event object. # This will ensure that we call destroy, which will do the 0MQ linger - log.info('Runner completed: {0}'.format(data['jid'])) + log.info(u'Runner completed: %s', data[u'jid']) del event del namespaced_event - return data if full_return else data['return'] + return data if full_return else data[u'return'] def get_docs(self, arg=None): ''' Return a dictionary of functions and the inline documentation for each ''' if arg: - target_mod = arg + '.' if not arg.endswith('.') else arg - docs = [(fun, self.functions[fun].__doc__) - for fun in sorted(self.functions) - if fun == arg or fun.startswith(target_mod)] + if u'*' in arg: + target_mod = arg + _use_fnmatch = True + else: + target_mod = arg + u'.' if not arg.endswith(u'.') else arg + _use_fnmatch = False + if _use_fnmatch: + docs = [(fun, self.functions[fun].__doc__) + for fun in fnmatch.filter(self.functions, target_mod)] + else: + docs = [(fun, self.functions[fun].__doc__) + for fun in sorted(self.functions) + if fun == arg or fun.startswith(target_mod)] else: docs = [(fun, self.functions[fun].__doc__) for fun in sorted(self.functions)] @@ -459,7 +474,7 @@ class AsyncClientMixin(object): Run this method in a multiprocess target to execute the function in a multiprocess and fire the return data on the event bus ''' - if daemonize and not salt.utils.is_windows(): + if daemonize and not salt.utils.platform.is_windows(): # Shutdown the multiprocessing before daemonizing salt.log.setup.shutdown_multiprocessing_logging() @@ -469,9 +484,9 @@ class AsyncClientMixin(object): salt.log.setup.setup_multiprocessing_logging() # pack a few things into low - low['__jid__'] = jid - low['__user__'] = user - low['__tag__'] = tag + low[u'__jid__'] = jid + low[u'__user__'] = user + low[u'__tag__'] = tag return self.low(fun, low, full_return=False) @@ -497,11 +512,11 @@ class AsyncClientMixin(object): def _gen_async_pub(self, jid=None): if jid is None: - jid = salt.utils.jid.gen_jid() + jid = salt.utils.jid.gen_jid(self.opts) tag = salt.utils.event.tagify(jid, prefix=self.tag_prefix) - return {'tag': tag, 'jid': jid} + return {u'tag': tag, u'jid': jid} - def async(self, fun, low, user='UNKNOWN', pub=None): + def async(self, fun, low, user=u'UNKNOWN', pub=None): ''' Execute the function in a multiprocess and return the event tag to use to watch for the return @@ -510,7 +525,7 @@ class AsyncClientMixin(object): proc = salt.utils.process.SignalHandlingMultiprocessingProcess( target=self._proc_function, - args=(fun, low, user, async_pub['tag'], async_pub['jid'])) + args=(fun, low, user, async_pub[u'tag'], async_pub[u'jid'])) with salt.utils.process.default_signals(signal.SIGINT, signal.SIGTERM): # Reset current signals before starting the process in # order not to inherit the current signal handlers @@ -526,29 +541,29 @@ class AsyncClientMixin(object): return # if we are "quiet", don't print - if self.opts.get('quiet', False): + if self.opts.get(u'quiet', False): return # some suffixes we don't want to print - if suffix in ('new',): + if suffix in (u'new',): return try: - outputter = self.opts.get('output', event.get('outputter', None) or event.get('return').get('outputter')) + outputter = self.opts.get(u'output', event.get(u'outputter', None) or event.get(u'return').get(u'outputter')) except AttributeError: outputter = None # if this is a ret, we have our own set of rules - if suffix == 'ret': + if suffix == u'ret': # Check if outputter was passed in the return data. If this is the case, # then the return data will be a dict two keys: 'data' and 'outputter' - if isinstance(event.get('return'), dict) \ - and set(event['return']) == set(('data', 'outputter')): - event_data = event['return']['data'] - outputter = event['return']['outputter'] + if isinstance(event.get(u'return'), dict) \ + and set(event[u'return']) == set((u'data', u'outputter')): + event_data = event[u'return'][u'data'] + outputter = event[u'return'][u'outputter'] else: - event_data = event['return'] + event_data = event[u'return'] else: - event_data = {'suffix': suffix, 'event': event} + event_data = {u'suffix': suffix, u'event': event} salt.output.display_output(event_data, outputter, self.opts) diff --git a/salt/client/netapi.py b/salt/client/netapi.py index 4c4967770c2..a5e07193184 100644 --- a/salt/client/netapi.py +++ b/salt/client/netapi.py @@ -20,7 +20,7 @@ class NetapiClient(object): ''' def __init__(self, opts): self.opts = opts - self.process_manager = salt.utils.process.ProcessManager(name='NetAPIProcessManager') + self.process_manager = salt.utils.process.ProcessManager(name=u'NetAPIProcessManager') self.netapi = salt.loader.netapi(self.opts) def run(self): @@ -28,11 +28,11 @@ class NetapiClient(object): Load and start all available api modules ''' if not len(self.netapi): - log.error("Did not find any netapi configurations, nothing to start") + log.error(u"Did not find any netapi configurations, nothing to start") for fun in self.netapi: - if fun.endswith('.start'): - log.info('Starting {0} netapi module'.format(fun)) + if fun.endswith(u'.start'): + log.info(u'Starting %s netapi module', fun) self.process_manager.add_process(self.netapi[fun]) # Install the SIGINT/SIGTERM handlers if not done so far diff --git a/salt/client/raet/__init__.py b/salt/client/raet/__init__.py index efce87088da..36546efaf0d 100644 --- a/salt/client/raet/__init__.py +++ b/salt/client/raet/__init__.py @@ -12,9 +12,9 @@ import logging # Import Salt libs import salt.config import salt.client -import salt.utils +import salt.utils.kinds as kinds +import salt.utils.versions import salt.syspaths as syspaths -from salt.utils import kinds try: from raet import raeting, nacling @@ -32,7 +32,7 @@ class LocalClient(salt.client.LocalClient): The RAET LocalClient ''' def __init__(self, - c_path=os.path.join(syspaths.CONFIG_DIR, 'master'), + c_path=os.path.join(syspaths.CONFIG_DIR, u'master'), mopts=None): salt.client.LocalClient.__init__(self, c_path, mopts) @@ -41,22 +41,22 @@ class LocalClient(salt.client.LocalClient): tgt, fun, arg=(), - tgt_type='glob', - ret='', - jid='', + tgt_type=u'glob', + ret=u'', + jid=u'', timeout=5, **kwargs): ''' Publish the command! ''' - if 'expr_form' in kwargs: - salt.utils.warn_until( - 'Fluorine', - 'The target type should be passed using the \'tgt_type\' ' - 'argument instead of \'expr_form\'. Support for using ' - '\'expr_form\' will be removed in Salt Fluorine.' + if u'expr_form' in kwargs: + salt.utils.versions.warn_until( + u'Fluorine', + u'The target type should be passed using the \'tgt_type\' ' + u'argument instead of \'expr_form\'. Support for using ' + u'\'expr_form\' will be removed in Salt Fluorine.' ) - tgt_type = kwargs.pop('expr_form') + tgt_type = kwargs.pop(u'expr_form') payload_kwargs = self._prep_pub( tgt, @@ -68,21 +68,21 @@ class LocalClient(salt.client.LocalClient): timeout=timeout, **kwargs) - kind = self.opts['__role'] + kind = self.opts[u'__role'] if kind not in kinds.APPL_KINDS: - emsg = ("Invalid application kind = '{0}' for Raet LocalClient.".format(kind)) - log.error(emsg + "\n") + emsg = (u"Invalid application kind = '{0}' for Raet LocalClient.".format(kind)) + log.error(emsg + u"\n") raise ValueError(emsg) if kind in [kinds.APPL_KIND_NAMES[kinds.applKinds.master], kinds.APPL_KIND_NAMES[kinds.applKinds.syndic]]: - lanename = 'master' + lanename = u'master' else: - emsg = ("Unsupported application kind '{0}' for Raet LocalClient.".format(kind)) - log.error(emsg + '\n') + emsg = (u"Unsupported application kind '{0}' for Raet LocalClient.".format(kind)) + log.error(emsg + u'\n') raise ValueError(emsg) - sockdirpath = self.opts['sock_dir'] - name = 'client' + nacling.uuid(size=18) + sockdirpath = self.opts[u'sock_dir'] + name = u'client' + nacling.uuid(size=18) stack = LaneStack( name=name, lanename=lanename, @@ -91,12 +91,12 @@ class LocalClient(salt.client.LocalClient): manor_yard = RemoteYard( stack=stack, lanename=lanename, - name='manor', + name=u'manor', dirpath=sockdirpath) stack.addRemote(manor_yard) - route = {'dst': (None, manor_yard.name, 'local_cmd'), - 'src': (None, stack.local.name, None)} - msg = {'route': route, 'load': payload_kwargs} + route = {u'dst': (None, manor_yard.name, u'local_cmd'), + u'src': (None, stack.local.name, None)} + msg = {u'route': route, u'load': payload_kwargs} stack.transmit(msg) stack.serviceAll() while True: @@ -104,9 +104,9 @@ class LocalClient(salt.client.LocalClient): stack.serviceAll() while stack.rxMsgs: msg, sender = stack.rxMsgs.popleft() - ret = msg.get('return', {}) - if 'ret' in ret: + ret = msg.get(u'return', {}) + if u'ret' in ret: stack.server.close() - return ret['ret'] + return ret[u'ret'] stack.server.close() return ret diff --git a/salt/client/ssh/__init__.py b/salt/client/ssh/__init__.py index 83226e984ce..cd3e53f0ea2 100644 --- a/salt/client/ssh/__init__.py +++ b/salt/client/ssh/__init__.py @@ -17,12 +17,11 @@ import os import re import sys import time -import yaml import uuid import tempfile import binascii import sys -import ntpath +import datetime # Import salt libs import salt.output @@ -39,17 +38,23 @@ import salt.serializers.yaml import salt.state import salt.utils import salt.utils.args -import salt.utils.event import salt.utils.atomicfile +import salt.utils.event +import salt.utils.files +import salt.utils.network +import salt.utils.path +import salt.utils.stringutils import salt.utils.thin import salt.utils.url import salt.utils.verify -import salt.utils.network -from salt.utils import is_windows +from salt.utils.locales import sdecode +from salt.utils.platform import is_windows from salt.utils.process import MultiprocessingProcess +import salt.roster +from salt.template import compile_template # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import input # pylint: disable=import-error,redefined-builtin try: import saltwinshell @@ -63,7 +68,7 @@ except ImportError: HAS_ZMQ = False # The directory where salt thin is deployed -DEFAULT_THIN_DIR = '/var/tmp/.%%USER%%_%%FQDNUUID%%_salt' +DEFAULT_THIN_DIR = u'/var/tmp/.%%USER%%_%%FQDNUUID%%_salt' # RSTR is just a delimiter to distinguish the beginning of salt STDOUT # and STDERR. There is no special meaning. Messages prior to RSTR in @@ -78,11 +83,11 @@ DEFAULT_THIN_DIR = '/var/tmp/.%%USER%%_%%FQDNUUID%%_salt' # Failure in SHIM # RSTR in stderr, No RSTR in stdout: # Undefined behavior -RSTR = '_edbc7885e4f9aac9b83b35999b68d015148caf467b78fa39c05f669c0ff89878' +RSTR = u'_edbc7885e4f9aac9b83b35999b68d015148caf467b78fa39c05f669c0ff89878' # The regex to find RSTR in output - Must be on an output line by itself # NOTE - must use non-grouping match groups or output splitting will fail. -RSTR_RE = r'(?:^|\r?\n)' + RSTR + '(?:\r?\n|$)' +RSTR_RE = sdecode(r'(?:^|\r?\n)' + RSTR + r'(?:\r?\n|$)') # future lint: disable=non-unicode-string # METHODOLOGY: # @@ -127,8 +132,9 @@ RSTR_RE = r'(?:^|\r?\n)' + RSTR + '(?:\r?\n|$)' # to be able to define the string with indentation for readability but # still strip the white space for compactness and to avoid issues with # some multi-line embedded python code having indentation errors +# future lint: disable=non-unicode-string SSH_SH_SHIM = \ - '\n'.join( + u'\n'.join( [s.strip() for s in r'''/bin/sh << 'EOF' set -e set -u @@ -166,7 +172,7 @@ do "from __future__ import print_function; import sys; import os; - map(sys.stdout.write, ['{{{{0}}}}={{{{1}}}} ' \ + map(sys.stdout.write, [u'{{{{0}}}}={{{{1}}}} ' \ .format(x, os.environ[x]) for x in [$ex_vars]])") exec $SUDO PATH=$PATH LD_LIBRARY_PATH=$LD_LIBRARY_PATH \ MANPATH=$MANPATH XDG_DATA_DIRS=$XDG_DATA_DIRS \ @@ -188,14 +194,15 @@ echo "ERROR: Unable to locate appropriate python command" >&2 exit $EX_PYTHON_INVALID EOF'''.format( EX_THIN_PYTHON_INVALID=salt.defaults.exitcodes.EX_THIN_PYTHON_INVALID, - ).split('\n')]) + ).split(u'\n')]) +# future lint: enable=non-unicode-string if not is_windows(): - shim_file = os.path.join(os.path.dirname(__file__), 'ssh_py_shim.py') + shim_file = os.path.join(os.path.dirname(__file__), u'ssh_py_shim.py') if not os.path.exists(shim_file): # On esky builds we only have the .pyc file - shim_file += "c" - with salt.utils.fopen(shim_file) as ssh_py_shim: + shim_file += u'c' + with salt.utils.files.fopen(shim_file) as ssh_py_shim: SSH_PY_SHIM = ssh_py_shim.read() log = logging.getLogger(__name__) @@ -205,13 +212,16 @@ class SSH(object): ''' Create an SSH execution system ''' + ROSTER_UPDATE_FLAG = '#__needs_update' + def __init__(self, opts): + self.__parsed_rosters = {SSH.ROSTER_UPDATE_FLAG: True} pull_sock = os.path.join(opts['sock_dir'], 'master_event_pull.ipc') if os.path.isfile(pull_sock) and HAS_ZMQ: self.event = salt.utils.event.get_event( - 'master', - opts['sock_dir'], - opts['transport'], + u'master', + opts[u'sock_dir'], + opts[u'transport'], opts=opts, listen=False) else: @@ -219,146 +229,234 @@ class SSH(object): self.opts = opts if self.opts['regen_thin']: self.opts['ssh_wipe'] = True - if not salt.utils.which('ssh'): - raise salt.exceptions.SaltSystemExit('No ssh binary found in path -- ssh must be installed for salt-ssh to run. Exiting.') + if not salt.utils.path.which('ssh'): + raise salt.exceptions.SaltSystemExit('No ssh binary found in path -- ssh must be ' + 'installed for salt-ssh to run. Exiting.') self.opts['_ssh_version'] = ssh_version() self.tgt_type = self.opts['selected_target_option'] \ if self.opts['selected_target_option'] else 'glob' - self.roster = salt.roster.Roster(opts, opts.get('roster', 'flat')) + self._expand_target() + self.roster = salt.roster.Roster(self.opts, self.opts.get('roster', 'flat')) self.targets = self.roster.targets( - self.opts['tgt'], + self.opts[u'tgt'], self.tgt_type) + if not self.targets: + self._update_targets() # If we're in a wfunc, we need to get the ssh key location from the # top level opts, stored in __master_opts__ - if '__master_opts__' in self.opts: - if self.opts['__master_opts__'].get('ssh_use_home_key') and \ - os.path.isfile(os.path.expanduser('~/.ssh/id_rsa')): - priv = os.path.expanduser('~/.ssh/id_rsa') + if u'__master_opts__' in self.opts: + if self.opts[u'__master_opts__'].get(u'ssh_use_home_key') and \ + os.path.isfile(os.path.expanduser(u'~/.ssh/id_rsa')): + priv = os.path.expanduser(u'~/.ssh/id_rsa') else: - priv = self.opts['__master_opts__'].get( - 'ssh_priv', + priv = self.opts[u'__master_opts__'].get( + u'ssh_priv', os.path.join( - self.opts['__master_opts__']['pki_dir'], - 'ssh', - 'salt-ssh.rsa' + self.opts[u'__master_opts__'][u'pki_dir'], + u'ssh', + u'salt-ssh.rsa' ) ) else: priv = self.opts.get( - 'ssh_priv', + u'ssh_priv', os.path.join( - self.opts['pki_dir'], - 'ssh', - 'salt-ssh.rsa' + self.opts[u'pki_dir'], + u'ssh', + u'salt-ssh.rsa' ) ) - if priv != 'agent-forwarding': + if priv != u'agent-forwarding': if not os.path.isfile(priv): try: salt.client.ssh.shell.gen_key(priv) except OSError: raise salt.exceptions.SaltClientError( - 'salt-ssh could not be run because it could not generate keys.\n\n' - 'You can probably resolve this by executing this script with ' - 'increased permissions via sudo or by running as root.\n' - 'You could also use the \'-c\' option to supply a configuration ' - 'directory that you have permissions to read and write to.' + u'salt-ssh could not be run because it could not generate keys.\n\n' + u'You can probably resolve this by executing this script with ' + u'increased permissions via sudo or by running as root.\n' + u'You could also use the \'-c\' option to supply a configuration ' + u'directory that you have permissions to read and write to.' ) self.defaults = { - 'user': self.opts.get( - 'ssh_user', - salt.config.DEFAULT_MASTER_OPTS['ssh_user'] + u'user': self.opts.get( + u'ssh_user', + salt.config.DEFAULT_MASTER_OPTS[u'ssh_user'] ), - 'port': self.opts.get( - 'ssh_port', - salt.config.DEFAULT_MASTER_OPTS['ssh_port'] + u'port': self.opts.get( + u'ssh_port', + salt.config.DEFAULT_MASTER_OPTS[u'ssh_port'] ), - 'passwd': self.opts.get( - 'ssh_passwd', - salt.config.DEFAULT_MASTER_OPTS['ssh_passwd'] + u'passwd': self.opts.get( + u'ssh_passwd', + salt.config.DEFAULT_MASTER_OPTS[u'ssh_passwd'] ), - 'priv': priv, - 'timeout': self.opts.get( - 'ssh_timeout', - salt.config.DEFAULT_MASTER_OPTS['ssh_timeout'] + u'priv': priv, + u'timeout': self.opts.get( + u'ssh_timeout', + salt.config.DEFAULT_MASTER_OPTS[u'ssh_timeout'] ) + self.opts.get( - 'timeout', - salt.config.DEFAULT_MASTER_OPTS['timeout'] + u'timeout', + salt.config.DEFAULT_MASTER_OPTS[u'timeout'] ), - 'sudo': self.opts.get( - 'ssh_sudo', - salt.config.DEFAULT_MASTER_OPTS['ssh_sudo'] + u'sudo': self.opts.get( + u'ssh_sudo', + salt.config.DEFAULT_MASTER_OPTS[u'ssh_sudo'] ), - 'sudo_user': self.opts.get( - 'ssh_sudo_user', - salt.config.DEFAULT_MASTER_OPTS['ssh_sudo_user'] + u'sudo_user': self.opts.get( + u'ssh_sudo_user', + salt.config.DEFAULT_MASTER_OPTS[u'ssh_sudo_user'] ), - 'identities_only': self.opts.get( - 'ssh_identities_only', - salt.config.DEFAULT_MASTER_OPTS['ssh_identities_only'] + u'identities_only': self.opts.get( + u'ssh_identities_only', + salt.config.DEFAULT_MASTER_OPTS[u'ssh_identities_only'] ), - 'remote_port_forwards': self.opts.get( - 'ssh_remote_port_forwards' + u'remote_port_forwards': self.opts.get( + u'ssh_remote_port_forwards' ), - 'ssh_options': self.opts.get( - 'ssh_options' + u'ssh_options': self.opts.get( + u'ssh_options' ) } - if self.opts.get('rand_thin_dir'): - self.defaults['thin_dir'] = os.path.join( - '/var/tmp', - '.{0}'.format(uuid.uuid4().hex[:6])) - self.opts['ssh_wipe'] = 'True' + if self.opts.get(u'rand_thin_dir'): + self.defaults[u'thin_dir'] = os.path.join( + u'/var/tmp', + u'.{0}'.format(uuid.uuid4().hex[:6])) + self.opts[u'ssh_wipe'] = u'True' self.serial = salt.payload.Serial(opts) self.returners = salt.loader.returners(self.opts, {}) self.fsclient = salt.fileclient.FSClient(self.opts) - self.thin = salt.utils.thin.gen_thin(self.opts['cachedir'], - extra_mods=self.opts.get('thin_extra_mods'), - overwrite=self.opts['regen_thin'], - python2_bin=self.opts['python2_bin'], - python3_bin=self.opts['python3_bin']) + self.thin = salt.utils.thin.gen_thin(self.opts[u'cachedir'], + extra_mods=self.opts.get(u'thin_extra_mods'), + overwrite=self.opts[u'regen_thin'], + python2_bin=self.opts[u'python2_bin'], + python3_bin=self.opts[u'python3_bin']) self.mods = mod_data(self.fsclient) + def _get_roster(self): + ''' + Read roster filename as a key to the data. + :return: + ''' + roster_file = salt.roster.get_roster_file(self.opts) + if roster_file not in self.__parsed_rosters: + roster_data = compile_template(roster_file, salt.loader.render(self.opts, {}), + self.opts['renderer'], self.opts['renderer_blacklist'], + self.opts['renderer_whitelist']) + self.__parsed_rosters[roster_file] = roster_data + return roster_file + + def _expand_target(self): + ''' + Figures out if the target is a reachable host without wildcards, expands if any. + :return: + ''' + # TODO: Support -L + target = self.opts['tgt'] + if isinstance(target, list): + return + + hostname = self.opts['tgt'].split('@')[-1] + needs_expansion = '*' not in hostname and salt.utils.network.is_reachable_host(hostname) + if needs_expansion: + hostname = salt.utils.network.ip_to_host(hostname) + self._get_roster() + for roster_filename in self.__parsed_rosters: + roster_data = self.__parsed_rosters[roster_filename] + if not isinstance(roster_data, bool): + for host_id in roster_data: + if hostname in [host_id, roster_data.get('host')]: + if hostname != self.opts['tgt']: + self.opts['tgt'] = hostname + self.__parsed_rosters[self.ROSTER_UPDATE_FLAG] = False + return + + def _update_roster(self): + ''' + Update default flat roster with the passed in information. + :return: + ''' + roster_file = self._get_roster() + if os.access(roster_file, os.W_OK): + if self.__parsed_rosters[self.ROSTER_UPDATE_FLAG]: + with salt.utils.files.fopen(roster_file, 'a') as roster_fp: + roster_fp.write('# Automatically added by "{s_user}" at {s_time}\n{hostname}:\n host: ' + '{hostname}\n user: {user}' + '\n passwd: {passwd}\n'.format(s_user=getpass.getuser(), + s_time=datetime.datetime.utcnow().isoformat(), + hostname=self.opts.get('tgt', ''), + user=self.opts.get('ssh_user', ''), + passwd=self.opts.get('ssh_passwd', ''))) + log.info('The host {0} has been added to the roster {1}'.format(self.opts.get('tgt', ''), + roster_file)) + else: + log.error('Unable to update roster {0}: access denied'.format(roster_file)) + + def _update_targets(self): + ''' + Uptade targets in case hostname was directly passed without the roster. + :return: + ''' + + hostname = self.opts.get('tgt', '') + if '@' in hostname: + user, hostname = hostname.split('@', 1) + else: + user = self.opts.get('ssh_user') + if hostname == '*': + hostname = '' + + if salt.utils.network.is_reachable_host(hostname): + hostname = salt.utils.network.ip_to_host(hostname) + self.opts['tgt'] = hostname + self.targets[hostname] = { + 'passwd': self.opts.get('ssh_passwd', ''), + 'host': hostname, + 'user': user, + } + if not self.opts.get('ssh_skip_roster'): + self._update_roster() + def get_pubkey(self): ''' Return the key string for the SSH public key ''' - if '__master_opts__' in self.opts and \ - self.opts['__master_opts__'].get('ssh_use_home_key') and \ - os.path.isfile(os.path.expanduser('~/.ssh/id_rsa')): - priv = os.path.expanduser('~/.ssh/id_rsa') + if u'__master_opts__' in self.opts and \ + self.opts[u'__master_opts__'].get(u'ssh_use_home_key') and \ + os.path.isfile(os.path.expanduser(u'~/.ssh/id_rsa')): + priv = os.path.expanduser(u'~/.ssh/id_rsa') else: priv = self.opts.get( - 'ssh_priv', + u'ssh_priv', os.path.join( - self.opts['pki_dir'], - 'ssh', - 'salt-ssh.rsa' + self.opts[u'pki_dir'], + u'ssh', + u'salt-ssh.rsa' ) ) - pub = '{0}.pub'.format(priv) - with salt.utils.fopen(pub, 'r') as fp_: - return '{0} rsa root@master'.format(fp_.read().split()[1]) + pub = u'{0}.pub'.format(priv) + with salt.utils.files.fopen(pub, u'r') as fp_: + return u'{0} rsa root@master'.format(fp_.read().split()[1]) def key_deploy(self, host, ret): ''' Deploy the SSH key if the minions don't auth ''' - if not isinstance(ret[host], dict) or self.opts.get('ssh_key_deploy'): + if not isinstance(ret[host], dict) or self.opts.get(u'ssh_key_deploy'): target = self.targets[host] - if target.get('passwd', False) or self.opts['ssh_passwd']: + if target.get(u'passwd', False) or self.opts[u'ssh_passwd']: self._key_deploy_run(host, target, False) return ret - if ret[host].get('stderr', '').count('Permission denied'): + if ret[host].get(u'stderr', u'').count(u'Permission denied'): target = self.targets[host] # permission denied, attempt to auto deploy ssh key - print(('Permission denied for host {0}, do you want to deploy ' - 'the salt-ssh key? (password required):').format(host)) - deploy = input('[Y/n] ') - if deploy.startswith(('n', 'N')): + print((u'Permission denied for host {0}, do you want to deploy ' + u'the salt-ssh key? (password required):').format(host)) + deploy = input(u'[Y/n] ') + if deploy.startswith((u'n', u'N')): return ret - target['passwd'] = getpass.getpass( - 'Password for {0}@{1}: '.format(target['user'], host) + target[u'passwd'] = getpass.getpass( + u'Password for {0}@{1}: '.format(target[u'user'], host) ) return self._key_deploy_run(host, target, True) return ret @@ -368,8 +466,8 @@ class SSH(object): The ssh-copy-id routine ''' argv = [ - 'ssh.set_auth_key', - target.get('user', 'root'), + u'ssh.set_auth_key', + target.get(u'user', u'root'), self.get_pubkey(), ] @@ -381,16 +479,16 @@ class SSH(object): fsclient=self.fsclient, thin=self.thin, **target) - if salt.utils.which('ssh-copy-id'): + if salt.utils.path.which(u'ssh-copy-id'): # we have ssh-copy-id, use it! stdout, stderr, retcode = single.shell.copy_id() else: stdout, stderr, retcode = single.run() if re_run: - target.pop('passwd') + target.pop(u'passwd') single = Single( self.opts, - self.opts['argv'], + self.opts[u'argv'], host, mods=self.mods, fsclient=self.fsclient, @@ -399,11 +497,11 @@ class SSH(object): stdout, stderr, retcode = single.cmd_block() try: data = salt.utils.find_json(stdout) - return {host: data.get('local', data)} + return {host: data.get(u'local', data)} except Exception: if stderr: return {host: stderr} - return {host: 'Bad Return'} + return {host: u'Bad Return'} if salt.defaults.exitcodes.EX_OK != retcode: return {host: stderr} return {host: stdout} @@ -415,31 +513,31 @@ class SSH(object): opts = copy.deepcopy(opts) single = Single( opts, - opts['argv'], + opts[u'argv'], host, mods=self.mods, fsclient=self.fsclient, thin=self.thin, mine=mine, **target) - ret = {'id': single.id} + ret = {u'id': single.id} stdout, stderr, retcode = single.run() # This job is done, yield try: data = salt.utils.find_json(stdout) - if len(data) < 2 and 'local' in data: - ret['ret'] = data['local'] + if len(data) < 2 and u'local' in data: + ret[u'ret'] = data[u'local'] else: - ret['ret'] = { - 'stdout': stdout, - 'stderr': stderr, - 'retcode': retcode, + ret[u'ret'] = { + u'stdout': stdout, + u'stderr': stderr, + u'retcode': retcode, } except Exception: - ret['ret'] = { - 'stdout': stdout, - 'stderr': stderr, - 'retcode': retcode, + ret[u'ret'] = { + u'stdout': stdout, + u'stderr': stderr, + u'retcode': retcode, } que.put(ret) @@ -456,9 +554,9 @@ class SSH(object): init = False while True: if not self.targets: - log.error('No matching targets found in roster.') + log.error(u'No matching targets found in roster.') break - if len(running) < self.opts.get('ssh_max_procs', 25) and not init: + if len(running) < self.opts.get(u'ssh_max_procs', 25) and not init: try: host = next(target_iter) except StopIteration: @@ -467,6 +565,8 @@ class SSH(object): for default in self.defaults: if default not in self.targets[host]: self.targets[host][default] = self.defaults[default] + if 'host' not in self.targets[host]: + self.targets[host]['host'] = host args = ( que, self.opts, @@ -478,14 +578,14 @@ class SSH(object): target=self.handle_routine, args=args) routine.start() - running[host] = {'thread': routine} + running[host] = {u'thread': routine} continue ret = {} try: ret = que.get(False) - if 'id' in ret: - returned.add(ret['id']) - yield {ret['id']: ret['ret']} + if u'id' in ret: + returned.add(ret[u'id']) + yield {ret[u'id']: ret[u'ret']} except Exception: # This bare exception is here to catch spurious exceptions # thrown by que.get during healthy operation. Please do not @@ -493,27 +593,27 @@ class SSH(object): # control program flow. pass for host in running: - if not running[host]['thread'].is_alive(): + if not running[host][u'thread'].is_alive(): if host not in returned: # Try to get any returns that came through since we # last checked try: while True: ret = que.get(False) - if 'id' in ret: - returned.add(ret['id']) - yield {ret['id']: ret['ret']} + if u'id' in ret: + returned.add(ret[u'id']) + yield {ret[u'id']: ret[u'ret']} except Exception: pass if host not in returned: - error = ('Target \'{0}\' did not return any data, ' - 'probably due to an error.').format(host) - ret = {'id': host, - 'ret': error} + error = (u'Target \'{0}\' did not return any data, ' + u'probably due to an error.').format(host) + ret = {u'id': host, + u'ret': error} log.error(error) - yield {ret['id']: ret['ret']} - running[host]['thread'].join() + yield {ret[u'id']: ret[u'ret']} + running[host][u'thread'].join() rets.add(host) for host in rets: if host in running: @@ -521,7 +621,7 @@ class SSH(object): if len(rets) >= len(self.targets): break # Sleep when limit or all threads started - if len(running) >= self.opts.get('ssh_max_procs', 25) or len(self.targets) >= len(running): + if len(running) >= self.opts.get(u'ssh_max_procs', 25) or len(self.targets) >= len(running): time.sleep(0.1) def run_iter(self, mine=False, jid=None): @@ -533,33 +633,33 @@ class SSH(object): pillar, or master config (they will be checked in that order) and will modify the argv with the arguments from mine_functions ''' - fstr = '{0}.prep_jid'.format(self.opts['master_job_cache']) - jid = self.returners[fstr](passed_jid=jid or self.opts.get('jid', None)) + fstr = u'{0}.prep_jid'.format(self.opts[u'master_job_cache']) + jid = self.returners[fstr](passed_jid=jid or self.opts.get(u'jid', None)) # Save the invocation information - argv = self.opts['argv'] + argv = self.opts[u'argv'] - if self.opts.get('raw_shell', False): - fun = 'ssh._raw' + if self.opts.get(u'raw_shell', False): + fun = u'ssh._raw' args = argv else: - fun = argv[0] if argv else '' + fun = argv[0] if argv else u'' args = argv[1:] job_load = { - 'jid': jid, - 'tgt_type': self.tgt_type, - 'tgt': self.opts['tgt'], - 'user': self.opts['user'], - 'fun': fun, - 'arg': args, + u'jid': jid, + u'tgt_type': self.tgt_type, + u'tgt': self.opts[u'tgt'], + u'user': self.opts[u'user'], + u'fun': fun, + u'arg': args, } # save load to the master job cache - if self.opts['master_job_cache'] == 'local_cache': - self.returners['{0}.save_load'.format(self.opts['master_job_cache'])](jid, job_load, minions=self.targets.keys()) + if self.opts[u'master_job_cache'] == u'local_cache': + self.returners[u'{0}.save_load'.format(self.opts[u'master_job_cache'])](jid, job_load, minions=self.targets.keys()) else: - self.returners['{0}.save_load'.format(self.opts['master_job_cache'])](jid, job_load) + self.returners[u'{0}.save_load'.format(self.opts[u'master_job_cache'])](jid, job_load) for ret in self.handle_ssh(mine=mine): host = next(six.iterkeys(ret)) @@ -568,69 +668,85 @@ class SSH(object): self.event.fire_event( ret, salt.utils.event.tagify( - [jid, 'ret', host], - 'job')) + [jid, u'ret', host], + u'job')) yield ret def cache_job(self, jid, id_, ret, fun): ''' Cache the job information ''' - self.returners['{0}.returner'.format(self.opts['master_job_cache'])]({'jid': jid, - 'id': id_, - 'return': ret, - 'fun': fun}) + self.returners[u'{0}.returner'.format(self.opts[u'master_job_cache'])]({u'jid': jid, + u'id': id_, + u'return': ret, + u'fun': fun}) def run(self, jid=None): ''' Execute the overall routine, print results via outputters ''' + if self.opts['list_hosts']: + self._get_roster() + ret = {} + for roster_file in self.__parsed_rosters: + if roster_file.startswith('#'): + continue + ret[roster_file] = {} + for host_id in self.__parsed_rosters[roster_file]: + hostname = self.__parsed_rosters[roster_file][host_id]['host'] + ret[roster_file][host_id] = hostname + salt.output.display_output(ret, 'nested', self.opts) + sys.exit() + fstr = '{0}.prep_jid'.format(self.opts['master_job_cache']) jid = self.returners[fstr](passed_jid=jid or self.opts.get('jid', None)) # Save the invocation information - argv = self.opts['argv'] + argv = self.opts[u'argv'] - if self.opts.get('raw_shell', False): - fun = 'ssh._raw' + if self.opts.get(u'raw_shell', False): + fun = u'ssh._raw' args = argv else: - fun = argv[0] if argv else '' + fun = argv[0] if argv else u'' args = argv[1:] job_load = { - 'jid': jid, - 'tgt_type': self.tgt_type, - 'tgt': self.opts['tgt'], - 'user': self.opts['user'], - 'fun': fun, - 'arg': args, + u'jid': jid, + u'tgt_type': self.tgt_type, + u'tgt': self.opts[u'tgt'], + u'user': self.opts[u'user'], + u'fun': fun, + u'arg': args, } # save load to the master job cache try: if isinstance(jid, bytes): - jid = jid.decode('utf-8') - if self.opts['master_job_cache'] == 'local_cache': - self.returners['{0}.save_load'.format(self.opts['master_job_cache'])](jid, job_load, minions=self.targets.keys()) + jid = jid.decode(u'utf-8') + if self.opts[u'master_job_cache'] == u'local_cache': + self.returners[u'{0}.save_load'.format(self.opts[u'master_job_cache'])](jid, job_load, minions=self.targets.keys()) else: - self.returners['{0}.save_load'.format(self.opts['master_job_cache'])](jid, job_load) + self.returners[u'{0}.save_load'.format(self.opts[u'master_job_cache'])](jid, job_load) except Exception as exc: log.exception(exc) - log.error('Could not save load with returner {0}: {1}'.format(self.opts['master_job_cache'], exc)) + log.error( + u'Could not save load with returner %s: %s', + self.opts[u'master_job_cache'], exc + ) - if self.opts.get('verbose'): - msg = 'Executing job with jid {0}'.format(jid) + if self.opts.get(u'verbose'): + msg = u'Executing job with jid {0}'.format(jid) print(msg) - print('-' * len(msg) + '\n') - print('') + print(u'-' * len(msg) + u'\n') + print(u'') sret = {} - outputter = self.opts.get('output', 'nested') + outputter = self.opts.get(u'output', u'nested') final_exit = 0 for ret in self.handle_ssh(): host = next(six.iterkeys(ret)) if isinstance(ret[host], dict): - host_ret = ret[host].get('retcode', 0) + host_ret = ret[host].get(u'retcode', 0) if host_ret != 0: final_exit = 1 else: @@ -640,17 +756,17 @@ class SSH(object): self.cache_job(jid, host, ret[host], fun) ret = self.key_deploy(host, ret) - if isinstance(ret[host], dict) and ret[host].get('stderr', '').startswith('ssh:'): - ret[host] = ret[host]['stderr'] + if isinstance(ret[host], dict) and ret[host].get(u'stderr', u'').startswith(u'ssh:'): + ret[host] = ret[host][u'stderr'] if not isinstance(ret[host], dict): p_data = {host: ret[host]} - elif 'return' not in ret[host]: + elif u'return' not in ret[host]: p_data = ret else: - outputter = ret[host].get('out', self.opts.get('output', 'nested')) - p_data = {host: ret[host].get('return', {})} - if self.opts.get('static'): + outputter = ret[host].get(u'out', self.opts.get(u'output', u'nested')) + p_data = {host: ret[host].get(u'return', {})} + if self.opts.get(u'static'): sret.update(p_data) else: salt.output.display_output( @@ -661,9 +777,9 @@ class SSH(object): self.event.fire_event( ret, salt.utils.event.tagify( - [jid, 'ret', host], - 'job')) - if self.opts.get('static'): + [jid, u'ret', host], + u'job')) + if self.opts.get(u'static'): salt.output.display_output( sret, outputter, @@ -706,35 +822,35 @@ class Single(object): **kwargs): # Get mine setting and mine_functions if defined in kwargs (from roster) self.mine = mine - self.mine_functions = kwargs.get('mine_functions') - self.cmd_umask = kwargs.get('cmd_umask', None) + self.mine_functions = kwargs.get(u'mine_functions') + self.cmd_umask = kwargs.get(u'cmd_umask', None) self.winrm = winrm self.opts = opts self.tty = tty - if kwargs.get('wipe'): - self.wipe = 'False' + if kwargs.get(u'wipe'): + self.wipe = u'False' else: - self.wipe = 'True' if self.opts.get('ssh_wipe') else 'False' - if kwargs.get('thin_dir'): - self.thin_dir = kwargs['thin_dir'] + self.wipe = u'True' if self.opts.get(u'ssh_wipe') else u'False' + if kwargs.get(u'thin_dir'): + self.thin_dir = kwargs[u'thin_dir'] elif self.winrm: saltwinshell.set_winvars(self) else: if user: - thin_dir = DEFAULT_THIN_DIR.replace('%%USER%%', user) + thin_dir = DEFAULT_THIN_DIR.replace(u'%%USER%%', user) else: - thin_dir = DEFAULT_THIN_DIR.replace('%%USER%%', 'root') + thin_dir = DEFAULT_THIN_DIR.replace(u'%%USER%%', u'root') self.thin_dir = thin_dir.replace( - '%%FQDNUUID%%', + u'%%FQDNUUID%%', uuid.uuid3(uuid.NAMESPACE_DNS, salt.utils.network.get_fqhostname()).hex[:6] ) - self.opts['thin_dir'] = self.thin_dir + self.opts[u'thin_dir'] = self.thin_dir self.fsclient = fsclient - self.context = {'master_opts': self.opts, - 'fileclient': self.fsclient} + self.context = {u'master_opts': self.opts, + u'fileclient': self.fsclient} if isinstance(argv, six.string_types): self.argv = [argv] @@ -745,34 +861,34 @@ class Single(object): self.id = id_ self.mods = mods if isinstance(mods, dict) else {} - args = {'host': host, - 'user': user, - 'port': port, - 'passwd': passwd, - 'priv': priv, - 'timeout': timeout, - 'sudo': sudo, - 'tty': tty, - 'mods': self.mods, - 'identities_only': identities_only, - 'sudo_user': sudo_user, - 'remote_port_forwards': remote_port_forwards, - 'winrm': winrm, - 'ssh_options': ssh_options} + args = {u'host': host, + u'user': user, + u'port': port, + u'passwd': passwd, + u'priv': priv, + u'timeout': timeout, + u'sudo': sudo, + u'tty': tty, + u'mods': self.mods, + u'identities_only': identities_only, + u'sudo_user': sudo_user, + u'remote_port_forwards': remote_port_forwards, + u'winrm': winrm, + u'ssh_options': ssh_options} # Pre apply changeable defaults self.minion_opts = { - 'grains_cache': True, + u'grains_cache': True, } - self.minion_opts.update(opts.get('ssh_minion_opts', {})) + self.minion_opts.update(opts.get(u'ssh_minion_opts', {})) if minion_opts is not None: self.minion_opts.update(minion_opts) # Post apply system needed defaults self.minion_opts.update({ - 'root_dir': os.path.join(self.thin_dir, 'running_data'), - 'id': self.id, - 'sock_dir': '/', - 'log_file': 'salt-call.log', - 'fileserver_list_cache_time': 3, + u'root_dir': os.path.join(self.thin_dir, u'running_data'), + u'id': self.id, + u'sock_dir': u'/', + u'log_file': u'salt-call.log', + u'fileserver_list_cache_time': 3, }) self.minion_config = salt.serializers.yaml.serialize(self.minion_opts) self.target = kwargs @@ -780,17 +896,17 @@ class Single(object): self.serial = salt.payload.Serial(opts) self.wfuncs = salt.loader.ssh_wrapper(opts, None, self.context) self.shell = salt.client.ssh.shell.gen_shell(opts, **args) - self.thin = thin if thin else salt.utils.thin.thin_path(opts['cachedir']) + self.thin = thin if thin else salt.utils.thin.thin_path(opts[u'cachedir']) def __arg_comps(self): ''' Return the function name and the arg list ''' - fun = self.argv[0] if self.argv else '' + fun = self.argv[0] if self.argv else u'' parsed = salt.utils.args.parse_input( self.argv[1:], condition=False, - no_parse=self.opts.get('no_parse', [])) + no_parse=self.opts.get(u'no_parse', [])) args = parsed[0] kws = parsed[1] return fun, args, kws @@ -805,7 +921,7 @@ class Single(object): ''' if self.winrm: return arg - return ''.join(['\\' + char if re.match(r'\W', char) else char for char in arg]) + return u''.join([u'\\' + char if re.match(sdecode(r'\W'), char) else char for char in arg]) # future lint: disable=non-unicode-string def deploy(self): ''' @@ -813,7 +929,7 @@ class Single(object): ''' self.shell.send( self.thin, - os.path.join(self.thin_dir, 'salt-thin.tgz'), + os.path.join(self.thin_dir, u'salt-thin.tgz'), ) self.deploy_ext() return True @@ -822,10 +938,10 @@ class Single(object): ''' Deploy the ext_mods tarball ''' - if self.mods.get('file'): + if self.mods.get(u'file'): self.shell.send( - self.mods['file'], - os.path.join(self.thin_dir, 'salt-ext_mods.tgz'), + self.mods[u'file'], + os.path.join(self.thin_dir, u'salt-ext_mods.tgz'), ) return True @@ -843,8 +959,8 @@ class Single(object): ''' stdout = stderr = retcode = None - if self.opts.get('raw_shell', False): - cmd_str = ' '.join([self._escape_arg(arg) for arg in self.argv]) + if self.opts.get(u'raw_shell', False): + cmd_str = u' '.join([self._escape_arg(arg) for arg in self.argv]) stdout, stderr, retcode = self.shell.exec_cmd(cmd_str) elif self.fun in self.wfuncs or self.mine: @@ -865,24 +981,24 @@ class Single(object): # Execute routine data_cache = False data = None - cdir = os.path.join(self.opts['cachedir'], 'minions', self.id) + cdir = os.path.join(self.opts[u'cachedir'], u'minions', self.id) if not os.path.isdir(cdir): os.makedirs(cdir) - datap = os.path.join(cdir, 'ssh_data.p') + datap = os.path.join(cdir, u'ssh_data.p') refresh = False if not os.path.isfile(datap): refresh = True else: passed_time = (time.time() - os.stat(datap).st_mtime) / 60 - if passed_time > self.opts.get('cache_life', 60): + if passed_time > self.opts.get(u'cache_life', 60): refresh = True - if self.opts.get('refresh_cache'): + if self.opts.get(u'refresh_cache'): refresh = True conf_grains = {} # Save conf file grains before they get clobbered - if 'ssh_grains' in self.opts: - conf_grains = self.opts['ssh_grains'] + if u'ssh_grains' in self.opts: + conf_grains = self.opts[u'ssh_grains'] if not data_cache: refresh = True if refresh: @@ -894,72 +1010,71 @@ class Single(object): fsclient=self.fsclient, minion_opts=self.minion_opts, **self.target) - opts_pkg = pre_wrapper['test.opts_pkg']() # pylint: disable=E1102 - if '_error' in opts_pkg: + opts_pkg = pre_wrapper[u'test.opts_pkg']() # pylint: disable=E1102 + if u'_error' in opts_pkg: #Refresh failed - retcode = opts_pkg['retcode'] - ret = json.dumps({'local': opts_pkg}) + retcode = opts_pkg[u'retcode'] + ret = json.dumps({u'local': opts_pkg}) return ret, retcode - opts_pkg['file_roots'] = self.opts['file_roots'] - opts_pkg['pillar_roots'] = self.opts['pillar_roots'] - opts_pkg['ext_pillar'] = self.opts['ext_pillar'] - opts_pkg['extension_modules'] = self.opts['extension_modules'] - opts_pkg['_ssh_version'] = self.opts['_ssh_version'] - opts_pkg['__master_opts__'] = self.context['master_opts'] - if '_caller_cachedir' in self.opts: - opts_pkg['_caller_cachedir'] = self.opts['_caller_cachedir'] + opts_pkg[u'file_roots'] = self.opts[u'file_roots'] + opts_pkg[u'pillar_roots'] = self.opts[u'pillar_roots'] + opts_pkg[u'ext_pillar'] = self.opts[u'ext_pillar'] + opts_pkg[u'extension_modules'] = self.opts[u'extension_modules'] + opts_pkg[u'_ssh_version'] = self.opts[u'_ssh_version'] + opts_pkg[u'__master_opts__'] = self.context[u'master_opts'] + if u'_caller_cachedir' in self.opts: + opts_pkg[u'_caller_cachedir'] = self.opts[u'_caller_cachedir'] else: - opts_pkg['_caller_cachedir'] = self.opts['cachedir'] + opts_pkg[u'_caller_cachedir'] = self.opts[u'cachedir'] # Use the ID defined in the roster file - opts_pkg['id'] = self.id + opts_pkg[u'id'] = self.id retcode = 0 # Restore master grains for grain in conf_grains: - opts_pkg['grains'][grain] = conf_grains[grain] + opts_pkg[u'grains'][grain] = conf_grains[grain] # Enable roster grains support - if 'grains' in self.target: - for grain in self.target['grains']: - opts_pkg['grains'][grain] = self.target['grains'][grain] + if u'grains' in self.target: + for grain in self.target[u'grains']: + opts_pkg[u'grains'][grain] = self.target[u'grains'][grain] popts = {} - popts.update(opts_pkg['__master_opts__']) + popts.update(opts_pkg[u'__master_opts__']) popts.update(opts_pkg) pillar = salt.pillar.Pillar( popts, - opts_pkg['grains'], - opts_pkg['id'], - opts_pkg.get('environment', 'base') + opts_pkg[u'grains'], + opts_pkg[u'id'], + opts_pkg.get(u'environment', u'base') ) - pillar_dirs = {} - pillar_data = pillar.compile_pillar(pillar_dirs=pillar_dirs) + pillar_data = pillar.compile_pillar() # TODO: cache minion opts in datap in master.py - data = {'opts': opts_pkg, - 'grains': opts_pkg['grains'], - 'pillar': pillar_data} + data = {u'opts': opts_pkg, + u'grains': opts_pkg[u'grains'], + u'pillar': pillar_data} if data_cache: - with salt.utils.fopen(datap, 'w+b') as fp_: + with salt.utils.files.fopen(datap, u'w+b') as fp_: fp_.write( self.serial.dumps(data) ) if not data and data_cache: - with salt.utils.fopen(datap, 'rb') as fp_: + with salt.utils.files.fopen(datap, u'rb') as fp_: data = self.serial.load(fp_) - opts = data.get('opts', {}) - opts['grains'] = data.get('grains') + opts = data.get(u'opts', {}) + opts[u'grains'] = data.get(u'grains') # Restore master grains for grain in conf_grains: - opts['grains'][grain] = conf_grains[grain] + opts[u'grains'][grain] = conf_grains[grain] # Enable roster grains support - if 'grains' in self.target: - for grain in self.target['grains']: - opts['grains'][grain] = self.target['grains'][grain] + if u'grains' in self.target: + for grain in self.target[u'grains']: + opts[u'grains'][grain] = self.target[u'grains'][grain] - opts['pillar'] = data.get('pillar') + opts[u'pillar'] = data.get(u'pillar') wrapper = salt.client.ssh.wrapper.FunctionWrapper( opts, self.id, @@ -978,18 +1093,18 @@ class Single(object): if self.mine_functions and self.fun in self.mine_functions: mine_fun_data = self.mine_functions[self.fun] - elif opts['pillar'] and self.fun in opts['pillar'].get('mine_functions', {}): - mine_fun_data = opts['pillar']['mine_functions'][self.fun] - elif self.fun in self.context['master_opts'].get('mine_functions', {}): - mine_fun_data = self.context['master_opts']['mine_functions'][self.fun] + elif opts[u'pillar'] and self.fun in opts[u'pillar'].get(u'mine_functions', {}): + mine_fun_data = opts[u'pillar'][u'mine_functions'][self.fun] + elif self.fun in self.context[u'master_opts'].get(u'mine_functions', {}): + mine_fun_data = self.context[u'master_opts'][u'mine_functions'][self.fun] if isinstance(mine_fun_data, dict): - mine_fun = mine_fun_data.pop('mine_function', mine_fun) + mine_fun = mine_fun_data.pop(u'mine_function', mine_fun) mine_args = mine_fun_data elif isinstance(mine_fun_data, list): for item in mine_fun_data[:]: - if isinstance(item, dict) and 'mine_function' in item: - mine_fun = item['mine_function'] + if isinstance(item, dict) and u'mine_function' in item: + mine_fun = item[u'mine_function'] mine_fun_data.pop(mine_fun_data.index(item)) mine_args = mine_fun_data else: @@ -1009,49 +1124,49 @@ class Single(object): else: result = self.wfuncs[self.fun](*self.args, **self.kwargs) except TypeError as exc: - result = 'TypeError encountered executing {0}: {1}'.format(self.fun, exc) + result = u'TypeError encountered executing {0}: {1}'.format(self.fun, exc) log.error(result, exc_info_on_loglevel=logging.DEBUG) retcode = 1 except Exception as exc: - result = 'An Exception occurred while executing {0}: {1}'.format(self.fun, exc) + result = u'An Exception occurred while executing {0}: {1}'.format(self.fun, exc) log.error(result, exc_info_on_loglevel=logging.DEBUG) retcode = 1 # Mimic the json data-structure that "salt-call --local" will # emit (as seen in ssh_py_shim.py) - if isinstance(result, dict) and 'local' in result: - ret = json.dumps({'local': result['local']}) + if isinstance(result, dict) and u'local' in result: + ret = json.dumps({u'local': result[u'local']}) else: - ret = json.dumps({'local': {'return': result}}) + ret = json.dumps({u'local': {u'return': result}}) return ret, retcode def _cmd_str(self): ''' Prepare the command string ''' - sudo = 'sudo' if self.target['sudo'] else '' - sudo_user = self.target['sudo_user'] - if '_caller_cachedir' in self.opts: - cachedir = self.opts['_caller_cachedir'] + sudo = u'sudo' if self.target[u'sudo'] else u'' + sudo_user = self.target[u'sudo_user'] + if u'_caller_cachedir' in self.opts: + cachedir = self.opts[u'_caller_cachedir'] else: - cachedir = self.opts['cachedir'] - thin_sum = salt.utils.thin.thin_sum(cachedir, 'sha1') - debug = '' - if not self.opts.get('log_level'): - self.opts['log_level'] = 'info' - if salt.log.LOG_LEVELS['debug'] >= salt.log.LOG_LEVELS[self.opts.get('log_level', 'info')]: - debug = '1' - arg_str = ''' + cachedir = self.opts[u'cachedir'] + thin_sum = salt.utils.thin.thin_sum(cachedir, u'sha1') + debug = u'' + if not self.opts.get(u'log_level'): + self.opts[u'log_level'] = u'info' + if salt.log.LOG_LEVELS[u'debug'] >= salt.log.LOG_LEVELS[self.opts.get(u'log_level', u'info')]: + debug = u'1' + arg_str = u''' OPTIONS = OBJ() OPTIONS.config = \ """ {0} """ -OPTIONS.delimiter = '{1}' -OPTIONS.saltdir = '{2}' -OPTIONS.checksum = '{3}' -OPTIONS.hashfunc = '{4}' -OPTIONS.version = '{5}' -OPTIONS.ext_mods = '{6}' +OPTIONS.delimiter = u'{1}' +OPTIONS.saltdir = u'{2}' +OPTIONS.checksum = u'{3}' +OPTIONS.hashfunc = u'{4}' +OPTIONS.version = u'{5}' +OPTIONS.ext_mods = u'{6}' OPTIONS.wipe = {7} OPTIONS.tty = {8} OPTIONS.cmd_umask = {9} @@ -1059,18 +1174,18 @@ ARGS = {10}\n'''.format(self.minion_config, RSTR, self.thin_dir, thin_sum, - 'sha1', + u'sha1', salt.version.__version__, - self.mods.get('version', ''), + self.mods.get(u'version', u''), self.wipe, self.tty, self.cmd_umask, self.argv) - py_code = SSH_PY_SHIM.replace('#%%OPTS', arg_str) + py_code = SSH_PY_SHIM.replace(u'#%%OPTS', arg_str) if six.PY2: - py_code_enc = py_code.encode('base64') + py_code_enc = py_code.encode(u'base64') else: - py_code_enc = base64.encodebytes(py_code.encode('utf-8')).decode('utf-8') + py_code_enc = base64.encodebytes(py_code.encode(u'utf-8')).decode(u'utf-8') if not self.winrm: cmd = SSH_SH_SHIM.format( DEBUG=debug, @@ -1084,7 +1199,7 @@ ARGS = {10}\n'''.format(self.minion_config, return cmd - def shim_cmd(self, cmd_str, extension='py'): + def shim_cmd(self, cmd_str, extension=u'py'): ''' Run a shim command. @@ -1095,15 +1210,15 @@ ARGS = {10}\n'''.format(self.minion_config, return self.shell.exec_cmd(cmd_str) # Write the shim to a temporary file in the default temp directory - with tempfile.NamedTemporaryFile(mode='w+b', - prefix='shim_', + with tempfile.NamedTemporaryFile(mode=u'w+b', + prefix=u'shim_', delete=False) as shim_tmp_file: - shim_tmp_file.write(salt.utils.to_bytes(cmd_str)) + shim_tmp_file.write(salt.utils.stringutils.to_bytes(cmd_str)) # Copy shim to target system, under $HOME/. - target_shim_file = '.{0}.{1}'.format(binascii.hexlify(os.urandom(6)), extension) + target_shim_file = u'.{0}.{1}'.format(binascii.hexlify(os.urandom(6)), extension) if self.winrm: - target_shim_file = saltwinshell.get_target_shim_file(self) + target_shim_file = saltwinshell.get_target_shim_file(self, target_shim_file) self.shell.send(shim_tmp_file.name, target_shim_file, makedirs=True) # Remove our shim file @@ -1113,19 +1228,19 @@ ARGS = {10}\n'''.format(self.minion_config, pass # Execute shim - if extension == 'ps1': - ret = self.shell.exec_cmd('"powershell {0}"'.format(target_shim_file)) + if extension == u'ps1': + ret = self.shell.exec_cmd(u'"powershell {0}"'.format(target_shim_file)) else: if not self.winrm: - ret = self.shell.exec_cmd('/bin/sh \'$HOME/{0}\''.format(target_shim_file)) + ret = self.shell.exec_cmd(u'/bin/sh \'$HOME/{0}\''.format(target_shim_file)) else: - ret = saltwinshell.call_python(self) + ret = saltwinshell.call_python(self, target_shim_file) # Remove shim from target system if not self.winrm: - self.shell.exec_cmd('rm \'$HOME/{0}\''.format(target_shim_file)) + self.shell.exec_cmd(u'rm \'$HOME/{0}\''.format(target_shim_file)) else: - self.shell.exec_cmd('del {0}'.format(target_shim_file)) + self.shell.exec_cmd(u'del {0}'.format(target_shim_file)) return ret @@ -1141,36 +1256,39 @@ ARGS = {10}\n'''.format(self.minion_config, 6. return command results ''' self.argv = _convert_args(self.argv) - log.debug('Performing shimmed, blocking command as follows:\n{0}'.format(' '.join(self.argv))) + log.debug( + u'Performing shimmed, blocking command as follows:\n%s', + u' '.join(self.argv) + ) cmd_str = self._cmd_str() stdout, stderr, retcode = self.shim_cmd(cmd_str) - log.trace('STDOUT {1}\n{0}'.format(stdout, self.target['host'])) - log.trace('STDERR {1}\n{0}'.format(stderr, self.target['host'])) - log.debug('RETCODE {1}: {0}'.format(retcode, self.target['host'])) + log.trace(u'STDOUT %s\n%s', self.target[u'host'], stdout) + log.trace(u'STDERR %s\n%s', self.target[u'host'], stderr) + log.debug(u'RETCODE %s: %s', self.target[u'host'], retcode) error = self.categorize_shim_errors(stdout, stderr, retcode) if error: - if error == 'Python environment not found on Windows system': + if error == u'Python environment not found on Windows system': saltwinshell.deploy_python(self) stdout, stderr, retcode = self.shim_cmd(cmd_str) while re.search(RSTR_RE, stdout): stdout = re.split(RSTR_RE, stdout, 1)[1].strip() while re.search(RSTR_RE, stderr): stderr = re.split(RSTR_RE, stderr, 1)[1].strip() - elif error == 'Undefined SHIM state': + elif error == u'Undefined SHIM state': self.deploy() stdout, stderr, retcode = self.shim_cmd(cmd_str) if not re.search(RSTR_RE, stdout) or not re.search(RSTR_RE, stderr): # If RSTR is not seen in both stdout and stderr then there # was a thin deployment problem. - return 'ERROR: Failure deploying thin, undefined state: {0}'.format(stdout), stderr, retcode + return u'ERROR: Failure deploying thin, undefined state: {0}'.format(stdout), stderr, retcode while re.search(RSTR_RE, stdout): stdout = re.split(RSTR_RE, stdout, 1)[1].strip() while re.search(RSTR_RE, stderr): stderr = re.split(RSTR_RE, stderr, 1)[1].strip() else: - return 'ERROR: {0}'.format(error), stderr, retcode + return u'ERROR: {0}'.format(error), stderr, retcode # FIXME: this discards output from ssh_shim if the shim succeeds. It should # always save the shim output regardless of shim success or failure. @@ -1186,35 +1304,35 @@ ARGS = {10}\n'''.format(self.minion_config, else: # RSTR was found in stdout but not stderr - which means there # is a SHIM command for the master. - shim_command = re.split(r'\r?\n', stdout, 1)[0].strip() - log.debug('SHIM retcode({0}) and command: {1}'.format(retcode, shim_command)) - if 'deploy' == shim_command and retcode == salt.defaults.exitcodes.EX_THIN_DEPLOY: + shim_command = re.split(sdecode(r'\r?\n'), stdout, 1)[0].strip() # future lint: disable=non-unicode-string + log.debug(u'SHIM retcode(%s) and command: %s', retcode, shim_command) + if u'deploy' == shim_command and retcode == salt.defaults.exitcodes.EX_THIN_DEPLOY: self.deploy() stdout, stderr, retcode = self.shim_cmd(cmd_str) if not re.search(RSTR_RE, stdout) or not re.search(RSTR_RE, stderr): if not self.tty: # If RSTR is not seen in both stdout and stderr then there # was a thin deployment problem. - log.error('ERROR: Failure deploying thin, retrying: {0}\n{1}'.format(stdout, stderr)) + log.error(u'ERROR: Failure deploying thin, retrying: %s\n%s', stdout, stderr) return self.cmd_block() elif not re.search(RSTR_RE, stdout): # If RSTR is not seen in stdout with tty, then there # was a thin deployment problem. - log.error('ERROR: Failure deploying thin, retrying: {0}\n{1}'.format(stdout, stderr)) + log.error(u'ERROR: Failure deploying thin, retrying: %s\n%s', stdout, stderr) while re.search(RSTR_RE, stdout): stdout = re.split(RSTR_RE, stdout, 1)[1].strip() if self.tty: - stderr = '' + stderr = u'' else: while re.search(RSTR_RE, stderr): stderr = re.split(RSTR_RE, stderr, 1)[1].strip() - elif 'ext_mods' == shim_command: + elif u'ext_mods' == shim_command: self.deploy_ext() stdout, stderr, retcode = self.shim_cmd(cmd_str) if not re.search(RSTR_RE, stdout) or not re.search(RSTR_RE, stderr): # If RSTR is not seen in both stdout and stderr then there # was a thin deployment problem. - return 'ERROR: Failure deploying ext_mods: {0}'.format(stdout), stderr, retcode + return u'ERROR: Failure deploying ext_mods: {0}'.format(stdout), stderr, retcode while re.search(RSTR_RE, stdout): stdout = re.split(RSTR_RE, stdout, 1)[1].strip() while re.search(RSTR_RE, stderr): @@ -1223,7 +1341,7 @@ ARGS = {10}\n'''.format(self.minion_config, return stdout, stderr, retcode def categorize_shim_errors(self, stdout, stderr, retcode): - if re.search(RSTR_RE, stdout) and stdout != RSTR+'\n': + if re.search(RSTR_RE, stdout) and stdout != RSTR+u'\n': # RSTR was found in stdout which means that the shim # functioned without *errors* . . . but there may be shim # commands, unless the only thing we found is RSTR @@ -1231,75 +1349,75 @@ ARGS = {10}\n'''.format(self.minion_config, if re.search(RSTR_RE, stderr): # Undefined state - return 'Undefined SHIM state' + return u'Undefined SHIM state' - if stderr.startswith('Permission denied'): + if stderr.startswith(u'Permission denied'): # SHIM was not even reached return None - perm_error_fmt = 'Permissions problem, target user may need '\ - 'to be root or use sudo:\n {0}' + perm_error_fmt = u'Permissions problem, target user may need '\ + u'to be root or use sudo:\n {0}' errors = [ ( (), - 'sudo: no tty present and no askpass program specified', - 'sudo expected a password, NOPASSWD required' + u'sudo: no tty present and no askpass program specified', + u'sudo expected a password, NOPASSWD required' ), ( (salt.defaults.exitcodes.EX_THIN_PYTHON_INVALID,), - 'Python interpreter is too old', - 'salt requires python 2.6 or newer on target hosts, must have same major version as origin host' + u'Python interpreter is too old', + u'salt requires python 2.6 or newer on target hosts, must have same major version as origin host' ), ( (salt.defaults.exitcodes.EX_THIN_CHECKSUM,), - 'checksum mismatched', - 'The salt thin transfer was corrupted' + u'checksum mismatched', + u'The salt thin transfer was corrupted' ), ( (salt.defaults.exitcodes.EX_SCP_NOT_FOUND,), - 'scp not found', - 'No scp binary. openssh-clients package required' + u'scp not found', + u'No scp binary. openssh-clients package required' ), ( (salt.defaults.exitcodes.EX_CANTCREAT,), - 'salt path .* exists but is not a directory', - 'A necessary path for salt thin unexpectedly exists:\n ' + stderr, + u'salt path .* exists but is not a directory', + u'A necessary path for salt thin unexpectedly exists:\n ' + stderr, ), ( (), - 'sudo: sorry, you must have a tty to run sudo', - 'sudo is configured with requiretty' + u'sudo: sorry, you must have a tty to run sudo', + u'sudo is configured with requiretty' ), ( (), - 'Failed to open log file', + u'Failed to open log file', perm_error_fmt.format(stderr) ), ( (), - 'Permission denied:.*/salt', + u'Permission denied:.*/salt', perm_error_fmt.format(stderr) ), ( (), - 'Failed to create directory path.*/salt', + u'Failed to create directory path.*/salt', perm_error_fmt.format(stderr) ), ( (salt.defaults.exitcodes.EX_SOFTWARE,), - 'exists but is not', - 'An internal error occurred with the shim, please investigate:\n ' + stderr, + u'exists but is not', + u'An internal error occurred with the shim, please investigate:\n ' + stderr, ), ( (), - 'The system cannot find the path specified', - 'Python environment not found on Windows system', + u'The system cannot find the path specified', + u'Python environment not found on Windows system', ), ( (), - 'is not recognized', - 'Python environment not found on Windows system', + u'is not recognized', + u'Python environment not found on Windows system', ), ] @@ -1327,14 +1445,14 @@ def lowstate_file_refs(chunks): ''' refs = {} for chunk in chunks: - saltenv = 'base' + saltenv = u'base' crefs = [] for state in chunk: - if state == '__env__': + if state == u'__env__': saltenv = chunk[state] - elif state == 'saltenv': + elif state == u'saltenv': saltenv = chunk[state] - elif state.startswith('__'): + elif state.startswith(u'__'): continue crefs.extend(salt_refs(chunk[state])) if crefs: @@ -1348,14 +1466,14 @@ def salt_refs(data): ''' Pull salt file references out of the states ''' - proto = 'salt://' + proto = u'salt://' ret = [] - if isinstance(data, str): + if isinstance(data, six.string_types): if data.startswith(proto): return [data] if isinstance(data, list): for comp in data: - if isinstance(comp, str): + if isinstance(comp, six.string_types): if comp.startswith(proto): ret.append(comp) return ret @@ -1367,23 +1485,23 @@ def mod_data(fsclient): ''' # TODO, change out for a fileserver backend sync_refs = [ - 'modules', - 'states', - 'grains', - 'renderers', - 'returners', + u'modules', + u'states', + u'grains', + u'renderers', + u'returners', ] ret = {} envs = fsclient.envs() - ver_base = '' + ver_base = u'' for env in envs: files = fsclient.file_list(env) for ref in sync_refs: mods_data = {} - pref = '_{0}'.format(ref) + pref = u'_{0}'.format(ref) for fn_ in sorted(files): if fn_.startswith(pref): - if fn_.endswith(('.py', '.so', '.pyx')): + if fn_.endswith((u'.py', u'.so', u'.pyx')): full = salt.utils.url.create(fn_) mod_path = fsclient.cache_file(full, env) if not os.path.isfile(mod_path): @@ -1400,21 +1518,21 @@ def mod_data(fsclient): return {} if six.PY3: - ver_base = salt.utils.to_bytes(ver_base) + ver_base = salt.utils.stringutils.to_bytes(ver_base) ver = hashlib.sha1(ver_base).hexdigest() ext_tar_path = os.path.join( - fsclient.opts['cachedir'], - 'ext_mods.{0}.tgz'.format(ver)) - mods = {'version': ver, - 'file': ext_tar_path} + fsclient.opts[u'cachedir'], + u'ext_mods.{0}.tgz'.format(ver)) + mods = {u'version': ver, + u'file': ext_tar_path} if os.path.isfile(ext_tar_path): return mods - tfp = tarfile.open(ext_tar_path, 'w:gz') - verfile = os.path.join(fsclient.opts['cachedir'], 'ext_mods.ver') - with salt.utils.fopen(verfile, 'w+') as fp_: + tfp = tarfile.open(ext_tar_path, u'w:gz') + verfile = os.path.join(fsclient.opts[u'cachedir'], u'ext_mods.ver') + with salt.utils.files.fopen(verfile, u'w+') as fp_: fp_.write(ver) - tfp.add(verfile, 'ext_version') + tfp.add(verfile, u'ext_version') for ref in ret: for fn_ in ret[ref]: tfp.add(ret[ref][fn_], os.path.join(ref, fn_)) @@ -1429,11 +1547,11 @@ def ssh_version(): # This function needs more granular checks and to be validated against # older versions of ssh ret = subprocess.Popen( - ['ssh', '-V'], + [u'ssh', u'-V'], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() try: - version_parts = ret[1].split(b',')[0].split(b'_')[1] + version_parts = ret[1].split(b',')[0].split(b'_')[1] # future lint: disable=non-unicode-string parts = [] for part in version_parts: try: @@ -1454,9 +1572,9 @@ def _convert_args(args): for arg in args: if isinstance(arg, dict): for key in list(arg.keys()): - if key == '__kwarg__': + if key == u'__kwarg__': continue - converted.append('{0}={1}'.format(key, arg[key])) + converted.append(u'{0}={1}'.format(key, arg[key])) else: converted.append(arg) return converted diff --git a/salt/client/ssh/client.py b/salt/client/ssh/client.py index 46ad336e44c..e41a5245ed3 100644 --- a/salt/client/ssh/client.py +++ b/salt/client/ssh/client.py @@ -9,6 +9,7 @@ import random # Import Salt libs import salt.config +import salt.utils.versions import salt.syspaths as syspaths from salt.exceptions import SaltClientError # Temporary @@ -22,7 +23,7 @@ class SSHClient(object): .. versionadded:: 2015.5.0 ''' def __init__(self, - c_path=os.path.join(syspaths.CONFIG_DIR, 'master'), + c_path=os.path.join(syspaths.CONFIG_DIR, u'master'), mopts=None, disable_custom_roster=False): if mopts: @@ -30,15 +31,14 @@ class SSHClient(object): else: if os.path.isdir(c_path): log.warning( - '{0} expects a file path not a directory path({1}) to ' - 'it\'s \'c_path\' keyword argument'.format( - self.__class__.__name__, c_path - ) + u'%s expects a file path not a directory path(%s) to ' + u'its \'c_path\' keyword argument', + self.__class__.__name__, c_path ) self.opts = salt.config.client_config(c_path) # Salt API should never offer a custom roster! - self.opts['__disable_custom_roster'] = disable_custom_roster + self.opts[u'__disable_custom_roster'] = disable_custom_roster def _prep_ssh( self, @@ -46,30 +46,30 @@ class SSHClient(object): fun, arg=(), timeout=None, - tgt_type='glob', + tgt_type=u'glob', kwarg=None, **kwargs): ''' Prepare the arguments ''' - if 'expr_form' in kwargs: - salt.utils.warn_until( - 'Fluorine', - 'The target type should be passed using the \'tgt_type\' ' - 'argument instead of \'expr_form\'. Support for using ' - '\'expr_form\' will be removed in Salt Fluorine.' + if u'expr_form' in kwargs: + salt.utils.versions.warn_until( + u'Fluorine', + u'The target type should be passed using the \'tgt_type\' ' + u'argument instead of \'expr_form\'. Support for using ' + u'\'expr_form\' will be removed in Salt Fluorine.' ) - tgt_type = kwargs.pop('expr_form') + tgt_type = kwargs.pop(u'expr_form') opts = copy.deepcopy(self.opts) opts.update(kwargs) if timeout: - opts['timeout'] = timeout + opts[u'timeout'] = timeout arg = salt.utils.args.condition_input(arg, kwarg) - opts['argv'] = [fun] + arg - opts['selected_target_option'] = tgt_type - opts['tgt'] = tgt - opts['arg'] = arg + opts[u'argv'] = [fun] + arg + opts[u'selected_target_option'] = tgt_type + opts[u'tgt'] = tgt + opts[u'arg'] = arg return salt.client.ssh.SSH(opts) def cmd_iter( @@ -78,8 +78,8 @@ class SSHClient(object): fun, arg=(), timeout=None, - tgt_type='glob', - ret='', + tgt_type=u'glob', + ret=u'', kwarg=None, **kwargs): ''' @@ -88,14 +88,14 @@ class SSHClient(object): .. versionadded:: 2015.5.0 ''' - if 'expr_form' in kwargs: - salt.utils.warn_until( - 'Fluorine', - 'The target type should be passed using the \'tgt_type\' ' - 'argument instead of \'expr_form\'. Support for using ' - '\'expr_form\' will be removed in Salt Fluorine.' + if u'expr_form' in kwargs: + salt.utils.versions.warn_until( + u'Fluorine', + u'The target type should be passed using the \'tgt_type\' ' + u'argument instead of \'expr_form\'. Support for using ' + u'\'expr_form\' will be removed in Salt Fluorine.' ) - tgt_type = kwargs.pop('expr_form') + tgt_type = kwargs.pop(u'expr_form') ssh = self._prep_ssh( tgt, @@ -105,7 +105,7 @@ class SSHClient(object): tgt_type, kwarg, **kwargs) - for ret in ssh.run_iter(jid=kwargs.get('jid', None)): + for ret in ssh.run_iter(jid=kwargs.get(u'jid', None)): yield ret def cmd(self, @@ -113,7 +113,7 @@ class SSHClient(object): fun, arg=(), timeout=None, - tgt_type='glob', + tgt_type=u'glob', kwarg=None, **kwargs): ''' @@ -122,14 +122,14 @@ class SSHClient(object): .. versionadded:: 2015.5.0 ''' - if 'expr_form' in kwargs: - salt.utils.warn_until( - 'Fluorine', - 'The target type should be passed using the \'tgt_type\' ' - 'argument instead of \'expr_form\'. Support for using ' - '\'expr_form\' will be removed in Salt Fluorine.' + if u'expr_form' in kwargs: + salt.utils.versions.warn_until( + u'Fluorine', + u'The target type should be passed using the \'tgt_type\' ' + u'argument instead of \'expr_form\'. Support for using ' + u'\'expr_form\' will be removed in Salt Fluorine.' ) - tgt_type = kwargs.pop('expr_form') + tgt_type = kwargs.pop(u'expr_form') ssh = self._prep_ssh( tgt, @@ -140,7 +140,7 @@ class SSHClient(object): kwarg, **kwargs) final = {} - for ret in ssh.run_iter(jid=kwargs.get('jid', None)): + for ret in ssh.run_iter(jid=kwargs.get(u'jid', None)): final.update(ret) return final @@ -166,16 +166,16 @@ class SSHClient(object): kwargs = copy.deepcopy(low) - for ignore in ['tgt', 'fun', 'arg', 'timeout', 'tgt_type', 'kwarg']: + for ignore in [u'tgt', u'fun', u'arg', u'timeout', u'tgt_type', u'kwarg']: if ignore in kwargs: del kwargs[ignore] - return self.cmd(low['tgt'], - low['fun'], - low.get('arg', []), - low.get('timeout'), - low.get('tgt_type'), - low.get('kwarg'), + return self.cmd(low[u'tgt'], + low[u'fun'], + low.get(u'arg', []), + low.get(u'timeout'), + low.get(u'tgt_type'), + low.get(u'kwarg'), **kwargs) def cmd_async(self, low, timeout=None): @@ -204,8 +204,8 @@ class SSHClient(object): fun, arg=(), timeout=None, - tgt_type='glob', - ret='', + tgt_type=u'glob', + ret=u'', kwarg=None, sub=3, **kwargs): @@ -226,24 +226,24 @@ class SSHClient(object): .. versionadded:: 2017.7.0 ''' - if 'expr_form' in kwargs: - salt.utils.warn_until( - 'Fluorine', - 'The target type should be passed using the \'tgt_type\' ' - 'argument instead of \'expr_form\'. Support for using ' - '\'expr_form\' will be removed in Salt Fluorine.' + if u'expr_form' in kwargs: + salt.utils.versions.warn_until( + u'Fluorine', + u'The target type should be passed using the \'tgt_type\' ' + u'argument instead of \'expr_form\'. Support for using ' + u'\'expr_form\' will be removed in Salt Fluorine.' ) - tgt_type = kwargs.pop('expr_form') + tgt_type = kwargs.pop(u'expr_form') minion_ret = self.cmd(tgt, - 'sys.list_functions', + u'sys.list_functions', tgt_type=tgt_type, **kwargs) minions = list(minion_ret) random.shuffle(minions) f_tgt = [] for minion in minions: - if fun in minion_ret[minion]['return']: + if fun in minion_ret[minion][u'return']: f_tgt.append(minion) if len(f_tgt) >= sub: break - return self.cmd_iter(f_tgt, fun, arg, timeout, tgt_type='list', ret=ret, kwarg=kwarg, **kwargs) + return self.cmd_iter(f_tgt, fun, arg, timeout, tgt_type=u'list', ret=ret, kwarg=kwarg, **kwargs) diff --git a/salt/client/ssh/shell.py b/salt/client/ssh/shell.py index 73c0b4aead7..a3b59abe95c 100644 --- a/salt/client/ssh/shell.py +++ b/salt/client/ssh/shell.py @@ -21,12 +21,12 @@ import salt.utils.vt log = logging.getLogger(__name__) -SSH_PASSWORD_PROMPT_RE = re.compile(r'(?:.*)[Pp]assword(?: for .*)?:', re.M) -KEY_VALID_RE = re.compile(r'.*\(yes\/no\).*') +SSH_PASSWORD_PROMPT_RE = re.compile(r'(?:.*)[Pp]assword(?: for .*)?:', re.M) # future lint: disable=non-unicode-string +KEY_VALID_RE = re.compile(r'.*\(yes\/no\).*') # future lint: disable=non-unicode-string # Keep these in sync with ./__init__.py -RSTR = '_edbc7885e4f9aac9b83b35999b68d015148caf467b78fa39c05f669c0ff89878' -RSTR_RE = re.compile(r'(?:^|\r?\n)' + RSTR + r'(?:\r?\n|$)') +RSTR = u'_edbc7885e4f9aac9b83b35999b68d015148caf467b78fa39c05f669c0ff89878' +RSTR_RE = re.compile(r'(?:^|\r?\n)' + RSTR + r'(?:\r?\n|$)') # future lint: disable=non-unicode-string class NoPasswdError(Exception): @@ -41,7 +41,7 @@ def gen_key(path): ''' Generate a key for use with salt-ssh ''' - cmd = 'ssh-keygen -P "" -f {0} -t rsa -q'.format(path) + cmd = u'ssh-keygen -P "" -f {0} -t rsa -q'.format(path) if not os.path.isdir(os.path.dirname(path)): os.makedirs(os.path.dirname(path)) subprocess.call(cmd, shell=True) @@ -51,12 +51,12 @@ def gen_shell(opts, **kwargs): ''' Return the correct shell interface for the target system ''' - if kwargs['winrm']: + if kwargs[u'winrm']: try: import saltwinshell shell = saltwinshell.Shell(opts, **kwargs) except ImportError: - log.error('The saltwinshell library is not available') + log.error(u'The saltwinshell library is not available') sys.exit(salt.defaults.exitcodes.EX_GENERIC) else: shell = Shell(opts, **kwargs) @@ -86,7 +86,7 @@ class Shell(object): ssh_options=None): self.opts = opts # ssh , but scp [ (4, 9): - options.append('GSSAPIAuthentication=no') - options.append('ConnectTimeout={0}'.format(self.timeout)) - if self.opts.get('ignore_host_keys'): - options.append('StrictHostKeyChecking=no') - if self.opts.get('no_host_keys'): - options.extend(['StrictHostKeyChecking=no', - 'UserKnownHostsFile=/dev/null']) - known_hosts = self.opts.get('known_hosts_file') + options.append(u'PasswordAuthentication=no') + if self.opts.get(u'_ssh_version', (0,)) > (4, 9): + options.append(u'GSSAPIAuthentication=no') + options.append(u'ConnectTimeout={0}'.format(self.timeout)) + if self.opts.get(u'ignore_host_keys'): + options.append(u'StrictHostKeyChecking=no') + if self.opts.get(u'no_host_keys'): + options.extend([u'StrictHostKeyChecking=no', + u'UserKnownHostsFile=/dev/null']) + known_hosts = self.opts.get(u'known_hosts_file') if known_hosts and os.path.isfile(known_hosts): - options.append('UserKnownHostsFile={0}'.format(known_hosts)) + options.append(u'UserKnownHostsFile={0}'.format(known_hosts)) if self.port: - options.append('Port={0}'.format(self.port)) + options.append(u'Port={0}'.format(self.port)) if self.priv: - options.append('IdentityFile={0}'.format(self.priv)) + options.append(u'IdentityFile={0}'.format(self.priv)) if self.user: - options.append('User={0}'.format(self.user)) + options.append(u'User={0}'.format(self.user)) if self.identities_only: - options.append('IdentitiesOnly=yes') + options.append(u'IdentitiesOnly=yes') ret = [] for option in options: - ret.append('-o {0} '.format(option)) - return ''.join(ret) + ret.append(u'-o {0} '.format(option)) + return u''.join(ret) def _passwd_opts(self): ''' @@ -156,42 +156,42 @@ class Shell(object): # TODO ControlMaster does not work without ControlPath # user could take advantage of it if they set ControlPath in their # ssh config. Also, ControlPersist not widely available. - options = ['ControlMaster=auto', - 'StrictHostKeyChecking=no', + options = [u'ControlMaster=auto', + u'StrictHostKeyChecking=no', ] - if self.opts['_ssh_version'] > (4, 9): - options.append('GSSAPIAuthentication=no') - options.append('ConnectTimeout={0}'.format(self.timeout)) - if self.opts.get('ignore_host_keys'): - options.append('StrictHostKeyChecking=no') - if self.opts.get('no_host_keys'): - options.extend(['StrictHostKeyChecking=no', - 'UserKnownHostsFile=/dev/null']) + if self.opts[u'_ssh_version'] > (4, 9): + options.append(u'GSSAPIAuthentication=no') + options.append(u'ConnectTimeout={0}'.format(self.timeout)) + if self.opts.get(u'ignore_host_keys'): + options.append(u'StrictHostKeyChecking=no') + if self.opts.get(u'no_host_keys'): + options.extend([u'StrictHostKeyChecking=no', + u'UserKnownHostsFile=/dev/null']) if self.passwd: - options.extend(['PasswordAuthentication=yes', - 'PubkeyAuthentication=yes']) + options.extend([u'PasswordAuthentication=yes', + u'PubkeyAuthentication=yes']) else: - options.extend(['PasswordAuthentication=no', - 'PubkeyAuthentication=yes', - 'KbdInteractiveAuthentication=no', - 'ChallengeResponseAuthentication=no', - 'BatchMode=yes']) + options.extend([u'PasswordAuthentication=no', + u'PubkeyAuthentication=yes', + u'KbdInteractiveAuthentication=no', + u'ChallengeResponseAuthentication=no', + u'BatchMode=yes']) if self.port: - options.append('Port={0}'.format(self.port)) + options.append(u'Port={0}'.format(self.port)) if self.user: - options.append('User={0}'.format(self.user)) + options.append(u'User={0}'.format(self.user)) if self.identities_only: - options.append('IdentitiesOnly=yes') + options.append(u'IdentitiesOnly=yes') ret = [] for option in options: - ret.append('-o {0} '.format(option)) - return ''.join(ret) + ret.append(u'-o {0} '.format(option)) + return u''.join(ret) def _ssh_opts(self): - return ' '.join(['-o {0}'.format(opt) - for opt in self.ssh_options]) + return u' '.join([u'-o {0}'.format(opt) + for opt in self.ssh_options]) def _copy_id_str_old(self): ''' @@ -200,9 +200,9 @@ class Shell(object): if self.passwd: # Using single quotes prevents shell expansion and # passwords containing '$' - return "{0} {1} '{2} -p {3} {4} {5}@{6}'".format( - 'ssh-copy-id', - '-i {0}.pub'.format(self.priv), + return u"{0} {1} '{2} -p {3} {4} {5}@{6}'".format( + u'ssh-copy-id', + u'-i {0}.pub'.format(self.priv), self._passwd_opts(), self.port, self._ssh_opts(), @@ -218,9 +218,9 @@ class Shell(object): if self.passwd: # Using single quotes prevents shell expansion and # passwords containing '$' - return "{0} {1} {2} -p {3} {4} {5}@{6}".format( - 'ssh-copy-id', - '-i {0}.pub'.format(self.priv), + return u"{0} {1} {2} -p {3} {4} {5}@{6}".format( + u'ssh-copy-id', + u'-i {0}.pub'.format(self.priv), self._passwd_opts(), self.port, self._ssh_opts(), @@ -233,11 +233,11 @@ class Shell(object): Execute ssh-copy-id to plant the id file on the target ''' stdout, stderr, retcode = self._run_cmd(self._copy_id_str_old()) - if salt.defaults.exitcodes.EX_OK != retcode and 'Usage' in stderr: + if salt.defaults.exitcodes.EX_OK != retcode and u'Usage' in stderr: stdout, stderr, retcode = self._run_cmd(self._copy_id_str_new()) return stdout, stderr, retcode - def _cmd_str(self, cmd, ssh='ssh'): + def _cmd_str(self, cmd, ssh=u'ssh'): ''' Return the cmd string to execute ''' @@ -246,21 +246,21 @@ class Shell(object): # need to deliver the SHIM to the remote host and execute it there command = [ssh] - if ssh != 'scp': + if ssh != u'scp': command.append(self.host) - if self.tty and ssh == 'ssh': - command.append('-t -t') + if self.tty and ssh == u'ssh': + command.append(u'-t -t') if self.passwd or self.priv: command.append(self.priv and self._key_opts() or self._passwd_opts()) - if ssh != 'scp' and self.remote_port_forwards: - command.append(' '.join(['-R {0}'.format(item) - for item in self.remote_port_forwards.split(',')])) + if ssh != u'scp' and self.remote_port_forwards: + command.append(u' '.join([u'-R {0}'.format(item) + for item in self.remote_port_forwards.split(u',')])) if self.ssh_options: command.append(self._ssh_opts()) command.append(cmd) - return ' '.join(command) + return u' '.join(command) def _old_run_cmd(self, cmd): ''' @@ -277,7 +277,7 @@ class Shell(object): data = proc.communicate() return data[0], data[1], proc.returncode except Exception: - return ('local', 'Unknown Error', None) + return (u'local', u'Unknown Error', None) def _run_nb_cmd(self, cmd): ''' @@ -301,7 +301,7 @@ class Shell(object): err = self.get_error(err) yield out, err, rcode except Exception: - yield ('', 'Unknown Error', None) + yield (u'', u'Unknown Error', None) def exec_nb_cmd(self, cmd): ''' @@ -312,9 +312,9 @@ class Shell(object): rcode = None cmd = self._cmd_str(cmd) - logmsg = 'Executing non-blocking command: {0}'.format(cmd) + logmsg = u'Executing non-blocking command: {0}'.format(cmd) if self.passwd: - logmsg = logmsg.replace(self.passwd, ('*' * 6)) + logmsg = logmsg.replace(self.passwd, (u'*' * 6)) log.debug(logmsg) for out, err, rcode in self._run_nb_cmd(cmd): @@ -323,7 +323,7 @@ class Shell(object): if err is not None: r_err.append(err) yield None, None, None - yield ''.join(r_out), ''.join(r_err), rcode + yield u''.join(r_out), u''.join(r_err), rcode def exec_cmd(self, cmd): ''' @@ -331,11 +331,11 @@ class Shell(object): ''' cmd = self._cmd_str(cmd) - logmsg = 'Executing command: {0}'.format(cmd) + logmsg = u'Executing command: {0}'.format(cmd) if self.passwd: - logmsg = logmsg.replace(self.passwd, ('*' * 6)) - if 'decode("base64")' in logmsg or 'base64.b64decode(' in logmsg: - log.debug('Executed SHIM command. Command logged to TRACE') + logmsg = logmsg.replace(self.passwd, (u'*' * 6)) + if u'decode("base64")' in logmsg or u'base64.b64decode(' in logmsg: + log.debug(u'Executed SHIM command. Command logged to TRACE') log.trace(logmsg) else: log.debug(logmsg) @@ -348,19 +348,19 @@ class Shell(object): scp a file or files to a remote system ''' if makedirs: - self.exec_cmd('mkdir -p {0}'.format(os.path.dirname(remote))) + self.exec_cmd(u'mkdir -p {0}'.format(os.path.dirname(remote))) # scp needs [ function mappings for sanitizing grain output. This # is used when the 'sanitize' flag is given. _SANITIZERS = { - 'serialnumber': _serial_sanitizer, - 'domain': _DOMAINNAME_SANITIZER, - 'fqdn': _FQDN_SANITIZER, - 'id': _FQDN_SANITIZER, - 'host': _HOSTNAME_SANITIZER, - 'localhost': _HOSTNAME_SANITIZER, - 'nodename': _HOSTNAME_SANITIZER, + u'serialnumber': _serial_sanitizer, + u'domain': _DOMAINNAME_SANITIZER, + u'fqdn': _FQDN_SANITIZER, + u'id': _FQDN_SANITIZER, + u'host': _HOSTNAME_SANITIZER, + u'localhost': _HOSTNAME_SANITIZER, + u'nodename': _HOSTNAME_SANITIZER, } -def get(key, default='', delimiter=DEFAULT_TARGET_DELIM, ordered=True): +def get(key, default=u'', delimiter=DEFAULT_TARGET_DELIM, ordered=True): ''' Attempt to retrieve the named value from grains, if the named value is not available return the passed default. The default return is an empty string. @@ -149,7 +151,7 @@ def item(*args, **kwargs): ret[arg] = __grains__[arg] except KeyError: pass - if salt.utils.is_true(kwargs.get('sanitize')): + if salt.utils.is_true(kwargs.get(u'sanitize')): for arg, func in six.iteritems(_SANITIZERS): if arg in ret: ret[arg] = func(ret[arg]) @@ -170,9 +172,9 @@ def ls(): # pylint: disable=C0103 def filter_by(lookup_dict, - grain='os_family', + grain=u'os_family', merge=None, - default='default', + default=u'default', base=None): ''' .. versionadded:: 0.17.0 @@ -266,12 +268,12 @@ def filter_by(lookup_dict, elif isinstance(base_values, collections.Mapping): if not isinstance(ret, collections.Mapping): - raise SaltException('filter_by default and look-up values must both be dictionaries.') + raise SaltException(u'filter_by default and look-up values must both be dictionaries.') ret = salt.utils.dictupdate.update(copy.deepcopy(base_values), ret) if merge: if not isinstance(merge, collections.Mapping): - raise SaltException('filter_by merge argument must be a dictionary.') + raise SaltException(u'filter_by merge argument must be a dictionary.') else: if ret is None: ret = merge diff --git a/salt/client/ssh/wrapper/mine.py b/salt/client/ssh/wrapper/mine.py index 32d7a2e3fdc..71f6f05a3cd 100644 --- a/salt/client/ssh/wrapper/mine.py +++ b/salt/client/ssh/wrapper/mine.py @@ -14,7 +14,7 @@ import copy import salt.client.ssh -def get(tgt, fun, tgt_type='glob', roster='flat'): +def get(tgt, fun, tgt_type=u'glob', roster=u'flat'): ''' Get data from the mine based on the target, function and tgt_type @@ -36,15 +36,15 @@ def get(tgt, fun, tgt_type='glob', roster='flat'): salt-ssh '*' mine.get '192.168.5.0' network.ipaddrs roster=scan ''' # Set up opts for the SSH object - opts = copy.deepcopy(__context__['master_opts']) + opts = copy.deepcopy(__context__[u'master_opts']) minopts = copy.deepcopy(__opts__) opts.update(minopts) if roster: - opts['roster'] = roster - opts['argv'] = [fun] - opts['selected_target_option'] = tgt_type - opts['tgt'] = tgt - opts['arg'] = [] + opts[u'roster'] = roster + opts[u'argv'] = [fun] + opts[u'selected_target_option'] = tgt_type + opts[u'tgt'] = tgt + opts[u'arg'] = [] # Create the SSH object to handle the actual call ssh = salt.client.ssh.SSH(opts) @@ -56,8 +56,8 @@ def get(tgt, fun, tgt_type='glob', roster='flat'): cret = {} for host in rets: - if 'return' in rets[host]: - cret[host] = rets[host]['return'] + if u'return' in rets[host]: + cret[host] = rets[host][u'return'] else: cret[host] = rets[host] return cret diff --git a/salt/client/ssh/wrapper/pillar.py b/salt/client/ssh/wrapper/pillar.py index 68c671aec88..8b71558e4a7 100644 --- a/salt/client/ssh/wrapper/pillar.py +++ b/salt/client/ssh/wrapper/pillar.py @@ -13,7 +13,7 @@ import salt.utils from salt.defaults import DEFAULT_TARGET_DELIM -def get(key, default='', merge=False, delimiter=DEFAULT_TARGET_DELIM): +def get(key, default=u'', merge=False, delimiter=DEFAULT_TARGET_DELIM): ''' .. versionadded:: 0.14 @@ -130,10 +130,10 @@ def keys(key, delimiter=DEFAULT_TARGET_DELIM): __pillar__, key, KeyError, delimiter) if ret is KeyError: - raise KeyError("Pillar key not found: {0}".format(key)) + raise KeyError(u"Pillar key not found: {0}".format(key)) if not isinstance(ret, dict): - raise ValueError("Pillar value in key {0} is not a dict".format(key)) + raise ValueError(u"Pillar value in key {0} is not a dict".format(key)) return ret.keys() diff --git a/salt/client/ssh/wrapper/publish.py b/salt/client/ssh/wrapper/publish.py index 62a803e9ea3..478275256b3 100644 --- a/salt/client/ssh/wrapper/publish.py +++ b/salt/client/ssh/wrapper/publish.py @@ -17,6 +17,8 @@ import logging # Import salt libs import salt.client.ssh import salt.runner +import salt.utils.args +import salt.utils.versions log = logging.getLogger(__name__) @@ -24,10 +26,10 @@ log = logging.getLogger(__name__) def _publish(tgt, fun, arg=None, - tgt_type='glob', - returner='', + tgt_type=u'glob', + returner=u'', timeout=None, - form='clean', + form=u'clean', roster=None): ''' Publish a command "from the minion out to other minions". In reality, the @@ -53,13 +55,13 @@ def _publish(tgt, salt-ssh system.example.com publish.publish '*' cmd.run 'ls -la /tmp' ''' - if fun.startswith('publish.'): - log.info('Cannot publish publish calls. Returning {}') + if fun.startswith(u'publish.'): + log.info(u'Cannot publish publish calls. Returning {}') return {} # TODO: implement returners? Do they make sense for salt-ssh calls? if returner: - log.warning('Returners currently not supported in salt-ssh publish') + log.warning(u'Returners currently not supported in salt-ssh publish') # Make sure args have been processed if arg is None: @@ -72,17 +74,17 @@ def _publish(tgt, arg = [] # Set up opts for the SSH object - opts = copy.deepcopy(__context__['master_opts']) + opts = copy.deepcopy(__context__[u'master_opts']) minopts = copy.deepcopy(__opts__) opts.update(minopts) if roster: - opts['roster'] = roster + opts[u'roster'] = roster if timeout: - opts['timeout'] = timeout - opts['argv'] = [fun] + arg - opts['selected_target_option'] = tgt_type - opts['tgt'] = tgt - opts['arg'] = arg + opts[u'timeout'] = timeout + opts[u'argv'] = [fun] + arg + opts[u'selected_target_option'] = tgt_type + opts[u'tgt'] = tgt + opts[u'arg'] = arg # Create the SSH object to handle the actual call ssh = salt.client.ssh.SSH(opts) @@ -92,11 +94,11 @@ def _publish(tgt, for ret in ssh.run_iter(): rets.update(ret) - if form == 'clean': + if form == u'clean': cret = {} for host in rets: - if 'return' in rets[host]: - cret[host] = rets[host]['return'] + if u'return' in rets[host]: + cret[host] = rets[host][u'return'] else: cret[host] = rets[host] return cret @@ -107,8 +109,8 @@ def _publish(tgt, def publish(tgt, fun, arg=None, - tgt_type='glob', - returner='', + tgt_type=u'glob', + returner=u'', timeout=5, roster=None, expr_form=None): @@ -173,11 +175,11 @@ def publish(tgt, # remember to remove the expr_form argument from this function when # performing the cleanup on this deprecation. if expr_form is not None: - salt.utils.warn_until( - 'Fluorine', - 'the target type should be passed using the \'tgt_type\' ' - 'argument instead of \'expr_form\'. Support for using ' - '\'expr_form\' will be removed in Salt Fluorine.' + salt.utils.versions.warn_until( + u'Fluorine', + u'the target type should be passed using the \'tgt_type\' ' + u'argument instead of \'expr_form\'. Support for using ' + u'\'expr_form\' will be removed in Salt Fluorine.' ) tgt_type = expr_form @@ -187,15 +189,15 @@ def publish(tgt, tgt_type=tgt_type, returner=returner, timeout=timeout, - form='clean', + form=u'clean', roster=roster) def full_data(tgt, fun, arg=None, - tgt_type='glob', - returner='', + tgt_type=u'glob', + returner=u'', timeout=5, roster=None, expr_form=None): @@ -223,11 +225,11 @@ def full_data(tgt, # remember to remove the expr_form argument from this function when # performing the cleanup on this deprecation. if expr_form is not None: - salt.utils.warn_until( - 'Fluorine', - 'the target type should be passed using the \'tgt_type\' ' - 'argument instead of \'expr_form\'. Support for using ' - '\'expr_form\' will be removed in Salt Fluorine.' + salt.utils.versions.warn_until( + u'Fluorine', + u'the target type should be passed using the \'tgt_type\' ' + u'argument instead of \'expr_form\'. Support for using ' + u'\'expr_form\' will be removed in Salt Fluorine.' ) tgt_type = expr_form @@ -237,7 +239,7 @@ def full_data(tgt, tgt_type=tgt_type, returner=returner, timeout=timeout, - form='full', + form=u'full', roster=roster) @@ -260,5 +262,5 @@ def runner(fun, arg=None, timeout=5): arg = [] # Create and run the runner - runner = salt.runner.RunnerClient(__opts__['__master_opts__']) + runner = salt.runner.RunnerClient(__opts__[u'__master_opts__']) return runner.cmd(fun, arg) diff --git a/salt/client/ssh/wrapper/state.py b/salt/client/ssh/wrapper/state.py index 2710d12ff84..d927ce2e558 100644 --- a/salt/client/ssh/wrapper/state.py +++ b/salt/client/ssh/wrapper/state.py @@ -20,10 +20,12 @@ import salt.state import salt.loader import salt.minion import salt.log -from salt.ext.six import string_types + +# Import 3rd-party libs +from salt.ext import six __func_alias__ = { - 'apply_': 'apply' + u'apply_': u'apply' } log = logging.getLogger(__name__) @@ -34,37 +36,37 @@ def _merge_extra_filerefs(*args): ''' ret = [] for arg in args: - if isinstance(arg, string_types): + if isinstance(arg, six.string_types): if arg: - ret.extend(arg.split(',')) + ret.extend(arg.split(u',')) elif isinstance(arg, list): if arg: ret.extend(arg) - return ','.join(ret) + return u','.join(ret) -def sls(mods, saltenv='base', test=None, exclude=None, **kwargs): +def sls(mods, saltenv=u'base', test=None, exclude=None, **kwargs): ''' Create the seed file for a state.sls run ''' st_kwargs = __salt__.kwargs - __opts__['grains'] = __grains__ - __pillar__.update(kwargs.get('pillar', {})) + __opts__[u'grains'] = __grains__ + __pillar__.update(kwargs.get(u'pillar', {})) st_ = salt.client.ssh.state.SSHHighState( __opts__, __pillar__, __salt__, - __context__['fileclient']) - if isinstance(mods, str): - mods = mods.split(',') + __context__[u'fileclient']) + if isinstance(mods, six.string_types): + mods = mods.split(u',') high_data, errors = st_.render_highstate({saltenv: mods}) if exclude: - if isinstance(exclude, str): - exclude = exclude.split(',') - if '__exclude__' in high_data: - high_data['__exclude__'].extend(exclude) + if isinstance(exclude, six.string_types): + exclude = exclude.split(u',') + if u'__exclude__' in high_data: + high_data[u'__exclude__'].extend(exclude) else: - high_data['__exclude__'] = exclude + high_data[u'__exclude__'] = exclude high_data, ext_errors = st_.state.reconcile_extend(high_data) errors += ext_errors errors += st_.state.verify_high(high_data) @@ -81,38 +83,38 @@ def sls(mods, saltenv='base', test=None, exclude=None, **kwargs): file_refs = salt.client.ssh.state.lowstate_file_refs( chunks, _merge_extra_filerefs( - kwargs.get('extra_filerefs', ''), - __opts__.get('extra_filerefs', '') + kwargs.get(u'extra_filerefs', u''), + __opts__.get(u'extra_filerefs', u'') ) ) - roster = salt.roster.Roster(__opts__, __opts__.get('roster', 'flat')) - roster_grains = roster.opts['grains'] + roster = salt.roster.Roster(__opts__, __opts__.get(u'roster', u'flat')) + roster_grains = roster.opts[u'grains'] # Create the tar containing the state pkg and relevant files. trans_tar = salt.client.ssh.state.prep_trans_tar( __opts__, - __context__['fileclient'], + __context__[u'fileclient'], chunks, file_refs, __pillar__, - st_kwargs['id_'], + st_kwargs[u'id_'], roster_grains) - trans_tar_sum = salt.utils.get_hash(trans_tar, __opts__['hash_type']) - cmd = 'state.pkg {0}/salt_state.tgz test={1} pkg_sum={2} hash_type={3}'.format( - __opts__['thin_dir'], + trans_tar_sum = salt.utils.get_hash(trans_tar, __opts__[u'hash_type']) + cmd = u'state.pkg {0}/salt_state.tgz test={1} pkg_sum={2} hash_type={3}'.format( + __opts__[u'thin_dir'], test, trans_tar_sum, - __opts__['hash_type']) + __opts__[u'hash_type']) single = salt.client.ssh.Single( __opts__, cmd, - fsclient=__context__['fileclient'], + fsclient=__context__[u'fileclient'], minion_opts=__salt__.minion_opts, **st_kwargs) single.shell.send( trans_tar, - '{0}/salt_state.tgz'.format(__opts__['thin_dir'])) + u'{0}/salt_state.tgz'.format(__opts__[u'thin_dir'])) stdout, stderr, _ = single.cmd_block() # Clean up our tar @@ -125,7 +127,7 @@ def sls(mods, saltenv='base', test=None, exclude=None, **kwargs): try: return json.loads(stdout, object_hook=salt.utils.decode_dict) except Exception as e: - log.error("JSON Render failed for: {0}\n{1}".format(stdout, stderr)) + log.error(u"JSON Render failed for: %s\n%s", stdout, stderr) log.error(str(e)) # If for some reason the json load fails, return the stdout @@ -144,51 +146,51 @@ def low(data, **kwargs): salt '*' state.low '{"state": "pkg", "fun": "installed", "name": "vi"}' ''' st_kwargs = __salt__.kwargs - __opts__['grains'] = __grains__ + __opts__[u'grains'] = __grains__ chunks = [data] st_ = salt.client.ssh.state.SSHHighState( __opts__, __pillar__, __salt__, - __context__['fileclient']) + __context__[u'fileclient']) for chunk in chunks: - chunk['__id__'] = chunk['name'] if not chunk.get('__id__') else chunk['__id__'] + chunk[u'__id__'] = chunk[u'name'] if not chunk.get(u'__id__') else chunk[u'__id__'] err = st_.state.verify_data(data) if err: return err file_refs = salt.client.ssh.state.lowstate_file_refs( chunks, _merge_extra_filerefs( - kwargs.get('extra_filerefs', ''), - __opts__.get('extra_filerefs', '') + kwargs.get(u'extra_filerefs', u''), + __opts__.get(u'extra_filerefs', u'') ) ) - roster = salt.roster.Roster(__opts__, __opts__.get('roster', 'flat')) - roster_grains = roster.opts['grains'] + roster = salt.roster.Roster(__opts__, __opts__.get(u'roster', u'flat')) + roster_grains = roster.opts[u'grains'] # Create the tar containing the state pkg and relevant files. trans_tar = salt.client.ssh.state.prep_trans_tar( __opts__, - __context__['fileclient'], + __context__[u'fileclient'], chunks, file_refs, __pillar__, - st_kwargs['id_'], + st_kwargs[u'id_'], roster_grains) - trans_tar_sum = salt.utils.get_hash(trans_tar, __opts__['hash_type']) - cmd = 'state.pkg {0}/salt_state.tgz pkg_sum={1} hash_type={2}'.format( - __opts__['thin_dir'], + trans_tar_sum = salt.utils.get_hash(trans_tar, __opts__[u'hash_type']) + cmd = u'state.pkg {0}/salt_state.tgz pkg_sum={1} hash_type={2}'.format( + __opts__[u'thin_dir'], trans_tar_sum, - __opts__['hash_type']) + __opts__[u'hash_type']) single = salt.client.ssh.Single( __opts__, cmd, - fsclient=__context__['fileclient'], + fsclient=__context__[u'fileclient'], minion_opts=__salt__.minion_opts, **st_kwargs) single.shell.send( trans_tar, - '{0}/salt_state.tgz'.format(__opts__['thin_dir'])) + u'{0}/salt_state.tgz'.format(__opts__[u'thin_dir'])) stdout, stderr, _ = single.cmd_block() # Clean up our tar @@ -201,7 +203,7 @@ def low(data, **kwargs): try: return json.loads(stdout, object_hook=salt.utils.decode_dict) except Exception as e: - log.error("JSON Render failed for: {0}\n{1}".format(stdout, stderr)) + log.error(u"JSON Render failed for: %s\n%s", stdout, stderr) log.error(str(e)) # If for some reason the json load fails, return the stdout @@ -219,49 +221,49 @@ def high(data, **kwargs): salt '*' state.high '{"vim": {"pkg": ["installed"]}}' ''' - __pillar__.update(kwargs.get('pillar', {})) + __pillar__.update(kwargs.get(u'pillar', {})) st_kwargs = __salt__.kwargs - __opts__['grains'] = __grains__ + __opts__[u'grains'] = __grains__ st_ = salt.client.ssh.state.SSHHighState( __opts__, __pillar__, __salt__, - __context__['fileclient']) + __context__[u'fileclient']) chunks = st_.state.compile_high_data(data) file_refs = salt.client.ssh.state.lowstate_file_refs( chunks, _merge_extra_filerefs( - kwargs.get('extra_filerefs', ''), - __opts__.get('extra_filerefs', '') + kwargs.get(u'extra_filerefs', u''), + __opts__.get(u'extra_filerefs', u'') ) ) - roster = salt.roster.Roster(__opts__, __opts__.get('roster', 'flat')) - roster_grains = roster.opts['grains'] + roster = salt.roster.Roster(__opts__, __opts__.get(u'roster', u'flat')) + roster_grains = roster.opts[u'grains'] # Create the tar containing the state pkg and relevant files. trans_tar = salt.client.ssh.state.prep_trans_tar( __opts__, - __context__['fileclient'], + __context__[u'fileclient'], chunks, file_refs, __pillar__, - st_kwargs['id_'], + st_kwargs[u'id_'], roster_grains) - trans_tar_sum = salt.utils.get_hash(trans_tar, __opts__['hash_type']) - cmd = 'state.pkg {0}/salt_state.tgz pkg_sum={1} hash_type={2}'.format( - __opts__['thin_dir'], + trans_tar_sum = salt.utils.get_hash(trans_tar, __opts__[u'hash_type']) + cmd = u'state.pkg {0}/salt_state.tgz pkg_sum={1} hash_type={2}'.format( + __opts__[u'thin_dir'], trans_tar_sum, - __opts__['hash_type']) + __opts__[u'hash_type']) single = salt.client.ssh.Single( __opts__, cmd, - fsclient=__context__['fileclient'], + fsclient=__context__[u'fileclient'], minion_opts=__salt__.minion_opts, **st_kwargs) single.shell.send( trans_tar, - '{0}/salt_state.tgz'.format(__opts__['thin_dir'])) + u'{0}/salt_state.tgz'.format(__opts__[u'thin_dir'])) stdout, stderr, _ = single.cmd_block() # Clean up our tar @@ -274,7 +276,7 @@ def high(data, **kwargs): try: return json.loads(stdout, object_hook=salt.utils.decode_dict) except Exception as e: - log.error("JSON Render failed for: {0}\n{1}".format(stdout, stderr)) + log.error(u"JSON Render failed for: %s\n%s", stdout, stderr) log.error(str(e)) # If for some reason the json load fails, return the stdout @@ -316,56 +318,56 @@ def highstate(test=None, **kwargs): salt '*' state.highstate exclude=sls_to_exclude salt '*' state.highstate exclude="[{'id': 'id_to_exclude'}, {'sls': 'sls_to_exclude'}]" ''' - __pillar__.update(kwargs.get('pillar', {})) + __pillar__.update(kwargs.get(u'pillar', {})) st_kwargs = __salt__.kwargs - __opts__['grains'] = __grains__ + __opts__[u'grains'] = __grains__ st_ = salt.client.ssh.state.SSHHighState( __opts__, __pillar__, __salt__, - __context__['fileclient']) + __context__[u'fileclient']) chunks = st_.compile_low_chunks() file_refs = salt.client.ssh.state.lowstate_file_refs( chunks, _merge_extra_filerefs( - kwargs.get('extra_filerefs', ''), - __opts__.get('extra_filerefs', '') + kwargs.get(u'extra_filerefs', u''), + __opts__.get(u'extra_filerefs', u'') ) ) # Check for errors for chunk in chunks: if not isinstance(chunk, dict): - __context__['retcode'] = 1 + __context__[u'retcode'] = 1 return chunks - roster = salt.roster.Roster(__opts__, __opts__.get('roster', 'flat')) - roster_grains = roster.opts['grains'] + roster = salt.roster.Roster(__opts__, __opts__.get(u'roster', u'flat')) + roster_grains = roster.opts[u'grains'] # Create the tar containing the state pkg and relevant files. trans_tar = salt.client.ssh.state.prep_trans_tar( __opts__, - __context__['fileclient'], + __context__[u'fileclient'], chunks, file_refs, __pillar__, - st_kwargs['id_'], + st_kwargs[u'id_'], roster_grains) - trans_tar_sum = salt.utils.get_hash(trans_tar, __opts__['hash_type']) - cmd = 'state.pkg {0}/salt_state.tgz test={1} pkg_sum={2} hash_type={3}'.format( - __opts__['thin_dir'], + trans_tar_sum = salt.utils.get_hash(trans_tar, __opts__[u'hash_type']) + cmd = u'state.pkg {0}/salt_state.tgz test={1} pkg_sum={2} hash_type={3}'.format( + __opts__[u'thin_dir'], test, trans_tar_sum, - __opts__['hash_type']) + __opts__[u'hash_type']) single = salt.client.ssh.Single( __opts__, cmd, - fsclient=__context__['fileclient'], + fsclient=__context__[u'fileclient'], minion_opts=__salt__.minion_opts, **st_kwargs) single.shell.send( trans_tar, - '{0}/salt_state.tgz'.format(__opts__['thin_dir'])) + u'{0}/salt_state.tgz'.format(__opts__[u'thin_dir'])) stdout, stderr, _ = single.cmd_block() # Clean up our tar @@ -378,7 +380,7 @@ def highstate(test=None, **kwargs): try: return json.loads(stdout, object_hook=salt.utils.decode_dict) except Exception as e: - log.error("JSON Render failed for: {0}\n{1}".format(stdout, stderr)) + log.error(u"JSON Render failed for: %s\n%s", stdout, stderr) log.error(str(e)) # If for some reason the json load fails, return the stdout @@ -397,55 +399,55 @@ def top(topfn, test=None, **kwargs): salt '*' state.top reverse_top.sls exclude=sls_to_exclude salt '*' state.top reverse_top.sls exclude="[{'id': 'id_to_exclude'}, {'sls': 'sls_to_exclude'}]" ''' - __pillar__.update(kwargs.get('pillar', {})) + __pillar__.update(kwargs.get(u'pillar', {})) st_kwargs = __salt__.kwargs - __opts__['grains'] = __grains__ + __opts__[u'grains'] = __grains__ if salt.utils.test_mode(test=test, **kwargs): - __opts__['test'] = True + __opts__[u'test'] = True else: - __opts__['test'] = __opts__.get('test', None) + __opts__[u'test'] = __opts__.get(u'test', None) st_ = salt.client.ssh.state.SSHHighState( __opts__, __pillar__, __salt__, - __context__['fileclient']) - st_.opts['state_top'] = os.path.join('salt://', topfn) + __context__[u'fileclient']) + st_.opts[u'state_top'] = os.path.join(u'salt://', topfn) chunks = st_.compile_low_chunks() file_refs = salt.client.ssh.state.lowstate_file_refs( chunks, _merge_extra_filerefs( - kwargs.get('extra_filerefs', ''), - __opts__.get('extra_filerefs', '') + kwargs.get(u'extra_filerefs', u''), + __opts__.get(u'extra_filerefs', u'') ) ) - roster = salt.roster.Roster(__opts__, __opts__.get('roster', 'flat')) - roster_grains = roster.opts['grains'] + roster = salt.roster.Roster(__opts__, __opts__.get(u'roster', u'flat')) + roster_grains = roster.opts[u'grains'] # Create the tar containing the state pkg and relevant files. trans_tar = salt.client.ssh.state.prep_trans_tar( __opts__, - __context__['fileclient'], + __context__[u'fileclient'], chunks, file_refs, __pillar__, - st_kwargs['id_'], + st_kwargs[u'id_'], roster_grains) - trans_tar_sum = salt.utils.get_hash(trans_tar, __opts__['hash_type']) - cmd = 'state.pkg {0}/salt_state.tgz test={1} pkg_sum={2} hash_type={3}'.format( - __opts__['thin_dir'], + trans_tar_sum = salt.utils.get_hash(trans_tar, __opts__[u'hash_type']) + cmd = u'state.pkg {0}/salt_state.tgz test={1} pkg_sum={2} hash_type={3}'.format( + __opts__[u'thin_dir'], test, trans_tar_sum, - __opts__['hash_type']) + __opts__[u'hash_type']) single = salt.client.ssh.Single( __opts__, cmd, - fsclient=__context__['fileclient'], + fsclient=__context__[u'fileclient'], minion_opts=__salt__.minion_opts, **st_kwargs) single.shell.send( trans_tar, - '{0}/salt_state.tgz'.format(__opts__['thin_dir'])) + u'{0}/salt_state.tgz'.format(__opts__[u'thin_dir'])) stdout, stderr, _ = single.cmd_block() # Clean up our tar @@ -458,7 +460,7 @@ def top(topfn, test=None, **kwargs): try: return json.loads(stdout, object_hook=salt.utils.decode_dict) except Exception as e: - log.error("JSON Render failed for: {0}\n{1}".format(stdout, stderr)) + log.error(u"JSON Render failed for: %s\n%s", stdout, stderr) log.error(str(e)) # If for some reason the json load fails, return the stdout @@ -475,12 +477,12 @@ def show_highstate(): salt '*' state.show_highstate ''' - __opts__['grains'] = __grains__ + __opts__[u'grains'] = __grains__ st_ = salt.client.ssh.state.SSHHighState( __opts__, __pillar__, __salt__, - __context__['fileclient']) + __context__[u'fileclient']) return st_.compile_highstate() @@ -494,16 +496,16 @@ def show_lowstate(): salt '*' state.show_lowstate ''' - __opts__['grains'] = __grains__ + __opts__[u'grains'] = __grains__ st_ = salt.client.ssh.state.SSHHighState( __opts__, __pillar__, __salt__, - __context__['fileclient']) + __context__[u'fileclient']) return st_.compile_low_chunks() -def show_sls(mods, saltenv='base', test=None, **kwargs): +def show_sls(mods, saltenv=u'base', test=None, **kwargs): ''' Display the state data from a specific sls or list of sls files on the master @@ -514,20 +516,20 @@ def show_sls(mods, saltenv='base', test=None, **kwargs): salt '*' state.show_sls core,edit.vim dev ''' - __pillar__.update(kwargs.get('pillar', {})) - __opts__['grains'] = __grains__ + __pillar__.update(kwargs.get(u'pillar', {})) + __opts__[u'grains'] = __grains__ opts = copy.copy(__opts__) if salt.utils.test_mode(test=test, **kwargs): - opts['test'] = True + opts[u'test'] = True else: - opts['test'] = __opts__.get('test', None) + opts[u'test'] = __opts__.get(u'test', None) st_ = salt.client.ssh.state.SSHHighState( __opts__, __pillar__, __salt__, - __context__['fileclient']) - if isinstance(mods, string_types): - mods = mods.split(',') + __context__[u'fileclient']) + if isinstance(mods, six.string_types): + mods = mods.split(u',') high_data, errors = st_.render_highstate({saltenv: mods}) high_data, ext_errors = st_.state.reconcile_extend(high_data) errors += ext_errors @@ -543,7 +545,7 @@ def show_sls(mods, saltenv='base', test=None, **kwargs): return high_data -def show_low_sls(mods, saltenv='base', test=None, **kwargs): +def show_low_sls(mods, saltenv=u'base', test=None, **kwargs): ''' Display the low state data from a specific sls or list of sls files on the master. @@ -556,21 +558,21 @@ def show_low_sls(mods, saltenv='base', test=None, **kwargs): salt '*' state.show_sls core,edit.vim dev ''' - __pillar__.update(kwargs.get('pillar', {})) - __opts__['grains'] = __grains__ + __pillar__.update(kwargs.get(u'pillar', {})) + __opts__[u'grains'] = __grains__ opts = copy.copy(__opts__) if salt.utils.test_mode(test=test, **kwargs): - opts['test'] = True + opts[u'test'] = True else: - opts['test'] = __opts__.get('test', None) + opts[u'test'] = __opts__.get(u'test', None) st_ = salt.client.ssh.state.SSHHighState( __opts__, __pillar__, __salt__, - __context__['fileclient']) - if isinstance(mods, string_types): - mods = mods.split(',') + __context__[u'fileclient']) + if isinstance(mods, six.string_types): + mods = mods.split(u',') high_data, errors = st_.render_highstate({saltenv: mods}) high_data, ext_errors = st_.state.reconcile_extend(high_data) errors += ext_errors @@ -597,12 +599,12 @@ def show_top(): salt '*' state.show_top ''' - __opts__['grains'] = __grains__ + __opts__[u'grains'] = __grains__ st_ = salt.client.ssh.state.SSHHighState( __opts__, __pillar__, __salt__, - __context__['fileclient']) + __context__[u'fileclient']) top_data = st_.get_top() errors = [] errors += st_.verify_tops(top_data) @@ -632,30 +634,30 @@ def single(fun, name, test=None, **kwargs): ''' st_kwargs = __salt__.kwargs - __opts__['grains'] = __grains__ + __opts__[u'grains'] = __grains__ # state.fun -> [state, fun] - comps = fun.split('.') + comps = fun.split(u'.') if len(comps) < 2: - __context__['retcode'] = 1 - return 'Invalid function passed' + __context__[u'retcode'] = 1 + return u'Invalid function passed' # Create the low chunk, using kwargs as a base - kwargs.update({'state': comps[0], - 'fun': comps[1], - '__id__': name, - 'name': name}) + kwargs.update({u'state': comps[0], + u'fun': comps[1], + u'__id__': name, + u'name': name}) opts = copy.deepcopy(__opts__) # Set test mode if salt.utils.test_mode(test=test, **kwargs): - opts['test'] = True + opts[u'test'] = True else: - opts['test'] = __opts__.get('test', None) + opts[u'test'] = __opts__.get(u'test', None) # Get the override pillar data - __pillar__.update(kwargs.get('pillar', {})) + __pillar__.update(kwargs.get(u'pillar', {})) # Create the State environment st_ = salt.client.ssh.state.SSHState(__opts__, __pillar__) @@ -663,7 +665,7 @@ def single(fun, name, test=None, **kwargs): # Verify the low chunk err = st_.verify_data(kwargs) if err: - __context__['retcode'] = 1 + __context__[u'retcode'] = 1 return err # Must be a list of low-chunks @@ -674,46 +676,46 @@ def single(fun, name, test=None, **kwargs): file_refs = salt.client.ssh.state.lowstate_file_refs( chunks, _merge_extra_filerefs( - kwargs.get('extra_filerefs', ''), - __opts__.get('extra_filerefs', '') + kwargs.get(u'extra_filerefs', u''), + __opts__.get(u'extra_filerefs', u'') ) ) - roster = salt.roster.Roster(__opts__, __opts__.get('roster', 'flat')) - roster_grains = roster.opts['grains'] + roster = salt.roster.Roster(__opts__, __opts__.get(u'roster', u'flat')) + roster_grains = roster.opts[u'grains'] # Create the tar containing the state pkg and relevant files. trans_tar = salt.client.ssh.state.prep_trans_tar( __opts__, - __context__['fileclient'], + __context__[u'fileclient'], chunks, file_refs, __pillar__, - st_kwargs['id_'], + st_kwargs[u'id_'], roster_grains) # Create a hash so we can verify the tar on the target system - trans_tar_sum = salt.utils.get_hash(trans_tar, __opts__['hash_type']) + trans_tar_sum = salt.utils.get_hash(trans_tar, __opts__[u'hash_type']) # We use state.pkg to execute the "state package" - cmd = 'state.pkg {0}/salt_state.tgz test={1} pkg_sum={2} hash_type={3}'.format( - __opts__['thin_dir'], + cmd = u'state.pkg {0}/salt_state.tgz test={1} pkg_sum={2} hash_type={3}'.format( + __opts__[u'thin_dir'], test, trans_tar_sum, - __opts__['hash_type']) + __opts__[u'hash_type']) # Create a salt-ssh Single object to actually do the ssh work single = salt.client.ssh.Single( __opts__, cmd, - fsclient=__context__['fileclient'], + fsclient=__context__[u'fileclient'], minion_opts=__salt__.minion_opts, **st_kwargs) # Copy the tar down single.shell.send( trans_tar, - '{0}/salt_state.tgz'.format(__opts__['thin_dir'])) + u'{0}/salt_state.tgz'.format(__opts__[u'thin_dir'])) # Run the state.pkg command on the target stdout, stderr, _ = single.cmd_block() @@ -728,7 +730,7 @@ def single(fun, name, test=None, **kwargs): try: return json.loads(stdout, object_hook=salt.utils.decode_dict) except Exception as e: - log.error("JSON Render failed for: {0}\n{1}".format(stdout, stderr)) + log.error(u"JSON Render failed for: %s\n%s", stdout, stderr) log.error(str(e)) # If for some reason the json load fails, return the stdout diff --git a/salt/cloud/__init__.py b/salt/cloud/__init__.py index 7cbb91e462d..0372f91f63f 100644 --- a/salt/cloud/__init__.py +++ b/salt/cloud/__init__.py @@ -5,7 +5,7 @@ correct cloud modules ''' # Import python libs -from __future__ import absolute_import, print_function, generators +from __future__ import absolute_import, print_function, generators, unicode_literals import os import copy import glob @@ -30,13 +30,12 @@ import salt.config import salt.client import salt.loader import salt.utils +import salt.utils.args import salt.utils.cloud +import salt.utils.context import salt.utils.dictupdate import salt.utils.files import salt.syspaths -from salt.utils import reinit_crypto -from salt.utils import context -from salt.ext.six import string_types from salt.template import compile_template # Import third party libs @@ -48,7 +47,7 @@ except ImportError: except ImportError: pass # pycrypto < 2.1 import yaml -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import input # pylint: disable=import-error,redefined-builtin # Get logging started @@ -342,7 +341,7 @@ class CloudClient(object): vm_overrides = {} kwargs['profile'] = profile mapper = salt.cloud.Map(self._opts_defaults(**kwargs)) - if isinstance(names, str): + if isinstance(names, six.string_types): names = names.split(',') return salt.utils.simple_types_filter( mapper.run_profile(profile, names, vm_overrides=vm_overrides) @@ -367,7 +366,7 @@ class CloudClient(object): Destroy the named VMs ''' mapper = salt.cloud.Map(self._opts_defaults(destroy=True)) - if isinstance(names, str): + if isinstance(names, six.string_types): names = names.split(',') return salt.utils.simple_types_filter( mapper.destroy(names) @@ -381,10 +380,9 @@ class CloudClient(object): .. code-block:: python - client.create(names=['myinstance'], provider='my-ec2-config', - kwargs={'image': 'ami-1624987f', 'size': 't1.micro', - 'ssh_username': 'ec2-user', 'securitygroup': 'default', - 'delvol_on_destroy': True}) + client.create(provider='my-ec2-config', names=['myinstance'], + image='ami-1624987f', size='t1.micro', ssh_username='ec2-user', + securitygroup='default', delvol_on_destroy=True) ''' mapper = salt.cloud.Map(self._opts_defaults()) providers = self.opts['providers'] @@ -392,14 +390,23 @@ class CloudClient(object): provider += ':{0}'.format(next(six.iterkeys(providers[provider]))) else: return False - if isinstance(names, str): + if isinstance(names, six.string_types): names = names.split(',') ret = {} for name in names: vm_ = kwargs.copy() vm_['name'] = name vm_['driver'] = provider + + # This function doesn't require a profile, but many cloud drivers + # check for profile information (which includes the provider key) to + # help with config file debugging and setting up instances. Setting + # the profile and provider defaults here avoids errors in other + # cloud functions relying on these keys. See SaltStack Issue #41971 + # and PR #38166 for more information. vm_['profile'] = None + vm_['provider'] = provider + ret[name] = salt.utils.simple_types_filter( mapper.create(vm_)) return ret @@ -425,7 +432,7 @@ class CloudClient(object): provider += ':{0}'.format(next(six.iterkeys(providers[provider]))) else: return False - if isinstance(names, str): + if isinstance(names, six.string_types): names = names.split(',') ret = {} @@ -617,7 +624,7 @@ class Cloud(object): pmap[alias] = {} try: - with context.func_globals_inject( + with salt.utils.context.func_globals_inject( self.clouds[fun], __active_provider_name__=':'.join([alias, driver]) ): @@ -700,7 +707,7 @@ class Cloud(object): def get_running_by_names(self, names, query='list_nodes', cached=False, profile=None): - if isinstance(names, string_types): + if isinstance(names, six.string_types): names = [names] matches = {} @@ -722,18 +729,9 @@ class Cloud(object): continue for vm_name, details in six.iteritems(vms): - # If VM was created with use_fqdn with either of the softlayer drivers, - # we need to strip the VM name and only search for the short hostname. - if driver == 'softlayer' or driver == 'softlayer_hw': - ret = [] - for name in names: - name = name.split('.')[0] - ret.append(name) - if vm_name not in ret: - continue # XXX: The logic below can be removed once the aws driver # is removed - elif vm_name not in names: + if vm_name not in names: continue elif driver == 'ec2' and 'aws' in handled_drivers and \ @@ -819,7 +817,7 @@ class Cloud(object): try: - with context.func_globals_inject( + with salt.utils.context.func_globals_inject( self.clouds[fun], __active_provider_name__=':'.join([alias, driver]) ): @@ -862,7 +860,7 @@ class Cloud(object): data[alias] = {} try: - with context.func_globals_inject( + with salt.utils.context.func_globals_inject( self.clouds[fun], __active_provider_name__=':'.join([alias, driver]) ): @@ -905,7 +903,7 @@ class Cloud(object): data[alias] = {} try: - with context.func_globals_inject( + with salt.utils.context.func_globals_inject( self.clouds[fun], __active_provider_name__=':'.join([alias, driver]) ): @@ -1027,7 +1025,7 @@ class Cloud(object): log.info('Destroying in non-parallel mode.') for alias, driver, name in vms_to_destroy: fun = '{0}.destroy'.format(driver) - with context.func_globals_inject( + with salt.utils.context.func_globals_inject( self.clouds[fun], __active_provider_name__=':'.join([alias, driver]) ): @@ -1272,7 +1270,7 @@ class Cloud(object): try: alias, driver = vm_['provider'].split(':') func = '{0}.create'.format(driver) - with context.func_globals_inject( + with salt.utils.context.func_globals_inject( self.clouds[fun], __active_provider_name__=':'.join([alias, driver]) ): @@ -1358,7 +1356,7 @@ class Cloud(object): return try: - with context.func_globals_inject( + with salt.utils.context.func_globals_inject( self.clouds[fun], __active_provider_name__=extra_['provider'] ): @@ -1388,7 +1386,7 @@ class Cloud(object): vm_overrides = {} try: - with salt.utils.fopen(self.opts['conf_file'], 'r') as mcc: + with salt.utils.files.fopen(self.opts['conf_file'], 'r') as mcc: main_cloud_config = yaml.safe_load(mcc) if not main_cloud_config: main_cloud_config = {} @@ -1506,7 +1504,7 @@ class Cloud(object): invalid_functions[fun].append(vm_name) continue - with context.func_globals_inject( + with salt.utils.context.func_globals_inject( self.clouds[fun], __active_provider_name__=':'.join([alias, driver]) ): @@ -1518,7 +1516,7 @@ class Cloud(object): # Clean kwargs of "__pub_*" data before running the cloud action call. # Prevents calling positional "kwarg" arg before "call" when no kwarg # argument is present in the cloud driver function's arg spec. - kwargs = salt.utils.clean_kwargs(**kwargs) + kwargs = salt.utils.args.clean_kwargs(**kwargs) if kwargs: ret[alias][driver][vm_name] = self.clouds[fun]( @@ -1590,7 +1588,7 @@ class Cloud(object): ) ) - with context.func_globals_inject( + with salt.utils.context.func_globals_inject( self.clouds[fun], __active_provider_name__=':'.join([alias, driver]) ): @@ -1638,7 +1636,7 @@ class Cloud(object): self.opts['providers'].pop(alias) continue - with context.func_globals_inject( + with salt.utils.context.func_globals_inject( self.clouds[fun], __active_provider_name__=':'.join([alias, driver]) ): @@ -1807,7 +1805,7 @@ class Map(Cloud): if isinstance(mapped, (list, tuple)): entries = {} for mapping in mapped: - if isinstance(mapping, string_types): + if isinstance(mapping, six.string_types): # Foo: # - bar1 # - bar2 @@ -1847,7 +1845,7 @@ class Map(Cloud): map_[profile] = entries continue - if isinstance(mapped, string_types): + if isinstance(mapped, six.string_types): # If it's a single string entry, let's make iterable because of # the next step mapped = [mapped] @@ -2098,7 +2096,7 @@ class Map(Cloud): # Generate the fingerprint of the master pubkey in order to # mitigate man-in-the-middle attacks master_temp_pub = salt.utils.files.mkstemp() - with salt.utils.fopen(master_temp_pub, 'w') as mtp: + with salt.utils.files.fopen(master_temp_pub, 'w') as mtp: mtp.write(pub) master_finger = salt.utils.pem_finger(master_temp_pub, sum_type=self.opts['hash_type']) os.unlink(master_temp_pub) @@ -2302,7 +2300,7 @@ def create_multiprocessing(parallel_data, queue=None): This function will be called from another process when running a map in parallel mode. The result from the create is always a json object. ''' - reinit_crypto() + salt.utils.reinit_crypto() parallel_data['opts']['output'] = 'json' cloud = Cloud(parallel_data['opts']) @@ -2334,14 +2332,14 @@ def destroy_multiprocessing(parallel_data, queue=None): This function will be called from another process when running a map in parallel mode. The result from the destroy is always a json object. ''' - reinit_crypto() + salt.utils.reinit_crypto() parallel_data['opts']['output'] = 'json' clouds = salt.loader.clouds(parallel_data['opts']) try: fun = clouds['{0}.destroy'.format(parallel_data['driver'])] - with context.func_globals_inject( + with salt.utils.context.func_globals_inject( fun, __active_provider_name__=':'.join([ parallel_data['alias'], @@ -2370,11 +2368,11 @@ def run_parallel_map_providers_query(data, queue=None): This function will be called from another process when building the providers map. ''' - reinit_crypto() + salt.utils.reinit_crypto() cloud = Cloud(data['opts']) try: - with context.func_globals_inject( + with salt.utils.context.func_globals_inject( cloud.clouds[data['fun']], __active_provider_name__=':'.join([ data['alias'], diff --git a/salt/cloud/cli.py b/salt/cloud/cli.py index fb8cdd25e15..7c85edce412 100644 --- a/salt/cloud/cli.py +++ b/salt/cloud/cli.py @@ -20,23 +20,24 @@ import logging from salt.ext.six.moves import input # Import salt libs +import salt.cloud +import salt.utils.cloud import salt.config import salt.defaults.exitcodes import salt.output +import salt.syspaths as syspaths import salt.utils -from salt.utils import parsers +import salt.utils.parsers +from salt.exceptions import SaltCloudException, SaltCloudSystemExit from salt.utils.verify import check_user, verify_env, verify_files, verify_log -# Import salt.cloud libs -import salt.cloud -import salt.utils.cloud -from salt.exceptions import SaltCloudException, SaltCloudSystemExit -import salt.ext.six as six -import salt.syspaths as syspaths +# Import 3rd-party libs +from salt.ext import six + log = logging.getLogger(__name__) -class SaltCloud(parsers.SaltCloudParser): +class SaltCloud(salt.utils.parsers.SaltCloudParser): def run(self): ''' diff --git a/salt/cloud/clouds/aliyun.py b/salt/cloud/clouds/aliyun.py index acdcde6124f..1980e8f1758 100644 --- a/salt/cloud/clouds/aliyun.py +++ b/salt/cloud/clouds/aliyun.py @@ -50,7 +50,8 @@ from salt.exceptions import ( SaltCloudExecutionTimeout ) -# Import Third Party Libs +# Import 3rd-party libs +from salt.ext import six try: import requests HAS_REQUESTS = True @@ -745,7 +746,7 @@ def _compute_signature(parameters, access_key_secret): ''' def percent_encode(line): - if not isinstance(line, str): + if not isinstance(line, six.string_types): return line s = line diff --git a/salt/cloud/clouds/azurearm.py b/salt/cloud/clouds/azurearm.py index 6d6d29b11ae..0397aaed449 100644 --- a/salt/cloud/clouds/azurearm.py +++ b/salt/cloud/clouds/azurearm.py @@ -59,11 +59,14 @@ import logging import pprint import base64 import yaml +import collections import salt.cache import salt.config as config import salt.utils import salt.utils.cloud -import salt.ext.six as six +import salt.utils.files +from salt.utils.versions import LooseVersion +from salt.ext import six import salt.version from salt.exceptions import ( SaltCloudSystemExit, @@ -77,7 +80,6 @@ HAS_LIBS = False try: import salt.utils.msazure from salt.utils.msazure import object_to_dict - import azure.storage from azure.common.credentials import ( UserPassCredentials, ServicePrincipalCredentials, @@ -103,6 +105,7 @@ try: from azure.mgmt.network.models import ( IPAllocationMethod, NetworkInterface, + NetworkInterfaceDnsSettings, NetworkInterfaceIPConfiguration, NetworkSecurityGroup, PublicIPAddress, @@ -112,7 +115,9 @@ try: from azure.mgmt.storage import StorageManagementClient from azure.mgmt.web import WebSiteManagementClient from msrestazure.azure_exceptions import CloudError - HAS_LIBS = True + from azure.multiapi.storage.v2016_05_31 import CloudStorageAccount + from azure.cli import core + HAS_LIBS = LooseVersion(core.__version__) >= LooseVersion("2.0.12") except ImportError: pass # pylint: enable=wrong-import-position,wrong-import-order @@ -249,7 +254,9 @@ def avail_locations(conn=None, call=None): # pylint: disable=unused-argument ret = {} regions = webconn.global_model.get_subscription_geo_regions() - for location in regions.value: # pylint: disable=no-member + if hasattr(regions, 'value'): + regions = regions.value + for location in regions: # pylint: disable=no-member lowername = str(location.name).lower().replace(' ', '') ret[lowername] = object_to_dict(location) return ret @@ -419,13 +426,14 @@ def list_nodes_full(conn=None, call=None): # pylint: disable=unused-argument for group in list_resource_groups(): nodes = compconn.virtual_machines.list(group) for node in nodes: + private_ips, public_ips = __get_ips_from_node(group, node) ret[node.name] = object_to_dict(node) ret[node.name]['id'] = node.id ret[node.name]['name'] = node.name ret[node.name]['size'] = node.hardware_profile.vm_size ret[node.name]['state'] = node.provisioning_state - ret[node.name]['private_ips'] = node.network_profile.network_interfaces - ret[node.name]['public_ips'] = node.network_profile.network_interfaces + ret[node.name]['private_ips'] = private_ips + ret[node.name]['public_ips'] = public_ips ret[node.name]['storage_profile']['data_disks'] = [] ret[node.name]['resource_group'] = group for disk in node.storage_profile.data_disks: @@ -445,6 +453,30 @@ def list_nodes_full(conn=None, call=None): # pylint: disable=unused-argument return ret +def __get_ips_from_node(resource_group, node): + ''' + List private and public IPs from a VM interface + ''' + global netconn # pylint: disable=global-statement,invalid-name + if not netconn: + netconn = get_conn(NetworkManagementClient) + + private_ips = [] + public_ips = [] + for node_iface in node.network_profile.network_interfaces: + node_iface_name = node_iface.id.split('/')[-1] + network_interface = netconn.network_interfaces.get(resource_group, node_iface_name) + for ip_configuration in network_interface.ip_configurations: + if ip_configuration.private_ip_address: + private_ips.append(ip_configuration.private_ip_address) + if ip_configuration.public_ip_address and ip_configuration.public_ip_address.id: + public_iface_name = ip_configuration.public_ip_address.id.split('/')[-1] + public_iface = netconn.public_ip_addresses.get(resource_group, public_iface_name) + public_ips.append(public_iface.ip_address) + + return private_ips, public_ips + + def list_resource_groups(conn=None, call=None): # pylint: disable=unused-argument ''' List resource groups associated with the account @@ -911,6 +943,17 @@ def create_interface(call=None, kwargs=None): # pylint: disable=unused-argument ) ] + dns_settings = None + if kwargs.get('dns_servers') is not None: + if isinstance(kwargs['dns_servers'], list): + dns_settings = NetworkInterfaceDnsSettings( + dns_servers=kwargs['dns_servers'], + applied_dns_servers=kwargs['dns_servers'], + internal_dns_name_label=None, + internal_fqdn=None, + internal_domain_name_suffix=None, + ) + network_security_group = None if kwargs.get('security_group') is not None: network_security_group = netconn.network_security_groups.get( @@ -922,6 +965,7 @@ def create_interface(call=None, kwargs=None): # pylint: disable=unused-argument location=kwargs['location'], network_security_group=network_security_group, ip_configurations=ip_configurations, + dns_settings=dns_settings, ) poller = netconn.network_interfaces.create_or_update( @@ -986,7 +1030,7 @@ def request_instance(call=None, kwargs=None): # pylint: disable=unused-argument ) else: if os.path.exists(userdata_file): - with salt.utils.fopen(userdata_file, 'r') as fh_: + with salt.utils.files.fopen(userdata_file, 'r') as fh_: userdata = fh_.read() userdata = salt.utils.cloud.userdata_template(__opts__, vm_, userdata) @@ -1289,6 +1333,15 @@ def destroy(name, conn=None, call=None, kwargs=None): # pylint: disable=unused- '-a or --action.' ) + __utils__['cloud.fire_event']( + 'event', + 'destroying instance', + 'salt/cloud/{0}/destroying'.format(name), + args={'name': name}, + sock_dir=__opts__['sock_dir'], + transport=__opts__['transport'] + ) + global compconn # pylint: disable=global-statement,invalid-name if not compconn: compconn = get_conn() @@ -1382,6 +1435,15 @@ def destroy(name, conn=None, call=None, kwargs=None): # pylint: disable=unused- ) ) + __utils__['cloud.fire_event']( + 'event', + 'destroyed instance', + 'salt/cloud/{0}/destroyed'.format(name), + args={'name': name}, + sock_dir=__opts__['sock_dir'], + transport=__opts__['transport'] + ) + return ret @@ -1630,10 +1692,15 @@ def pages_to_list(items): while True: try: page = items.next() # pylint: disable=incompatible-py3-code - for item in page: - objs.append(item) + if isinstance(page, collections.Iterable): + for item in page: + objs.append(item) + else: + objs.append(page) except GeneratorExit: break + except StopIteration: + break return objs @@ -1663,7 +1730,7 @@ def list_containers(call=None, kwargs=None): # pylint: disable=unused-argument if not storconn: storconn = get_conn(StorageManagementClient) - storageaccount = azure.storage.CloudStorageAccount( + storageaccount = CloudStorageAccount( config.get_cloud_config_value( 'storage_account', get_configured_provider(), __opts__, search_global=False @@ -1704,7 +1771,7 @@ def list_blobs(call=None, kwargs=None): # pylint: disable=unused-argument 'A container must be specified' ) - storageaccount = azure.storage.CloudStorageAccount( + storageaccount = CloudStorageAccount( config.get_cloud_config_value( 'storage_account', get_configured_provider(), __opts__, search_global=False @@ -1744,7 +1811,7 @@ def delete_blob(call=None, kwargs=None): # pylint: disable=unused-argument 'A blob must be specified' ) - storageaccount = azure.storage.CloudStorageAccount( + storageaccount = CloudStorageAccount( config.get_cloud_config_value( 'storage_account', get_configured_provider(), __opts__, search_global=False diff --git a/salt/cloud/clouds/cloudstack.py b/salt/cloud/clouds/cloudstack.py index 2f3af258690..8732f05f478 100644 --- a/salt/cloud/clouds/cloudstack.py +++ b/salt/cloud/clouds/cloudstack.py @@ -30,6 +30,8 @@ import logging # Import salt cloud libs import salt.config as config +import salt.utils.cloud +import salt.utils.event from salt.cloud.libcloudfuncs import * # pylint: disable=redefined-builtin,wildcard-import,unused-wildcard-import from salt.utils import namespaced_function from salt.exceptions import SaltCloudSystemExit @@ -168,8 +170,15 @@ def get_security_groups(conn, vm_): ''' Return a list of security groups to use, defaulting to ['default'] ''' - return config.get_cloud_config_value('securitygroup', vm_, __opts__, - default=['default']) + securitygroup_enabled = config.get_cloud_config_value( + 'securitygroup_enabled', vm_, __opts__, default=True + ) + if securitygroup_enabled: + return config.get_cloud_config_value( + 'securitygroup', vm_, __opts__, default=['default'] + ) + else: + return False def get_password(vm_): @@ -270,6 +279,7 @@ def create(vm_): 'event', 'starting create', 'salt/cloud/{0}/creating'.format(vm_['name']), + sock_dir=__opts__['sock_dir'], args=__utils__['cloud.filter_event']('creating', vm_, ['name', 'profile', 'provider', 'driver']), transport=__opts__['transport'] ) @@ -281,9 +291,12 @@ def create(vm_): 'image': get_image(conn, vm_), 'size': get_size(conn, vm_), 'location': get_location(conn, vm_), - 'ex_security_groups': get_security_groups(conn, vm_) } + sg = get_security_groups(conn, vm_) + if sg is not False: + kwargs['ex_security_groups'] = sg + if get_keypair(vm_) is not False: kwargs['ex_keyname'] = get_keypair(vm_) @@ -306,6 +319,7 @@ def create(vm_): 'event', 'requesting instance', 'salt/cloud/{0}/requesting'.format(vm_['name']), + sock_dir=__opts__['sock_dir'], args={ 'kwargs': __utils__['cloud.filter_event']( 'requesting', @@ -329,12 +343,13 @@ def create(vm_): if 'VirtualName' not in ex_blockdevicemapping: ex_blockdevicemapping['VirtualName'] = '{0}-{1}'.format(vm_['name'], len(volumes)) __utils__['cloud.fire_event']( - 'event', - 'requesting volume', - 'salt/cloud/{0}/requesting'.format(ex_blockdevicemapping['VirtualName']), - {'kwargs': {'name': ex_blockdevicemapping['VirtualName'], - 'device': ex_blockdevicemapping['DeviceName'], - 'size': ex_blockdevicemapping['VolumeSize']}}, + 'event', + 'requesting volume', + 'salt/cloud/{0}/requesting'.format(ex_blockdevicemapping['VirtualName']), + sock_dir=__opts__['sock_dir'], + args={'kwargs': {'name': ex_blockdevicemapping['VirtualName'], + 'device': ex_blockdevicemapping['DeviceName'], + 'size': ex_blockdevicemapping['VolumeSize']}}, ) try: volumes[ex_blockdevicemapping['DeviceName']] = conn.create_volume( @@ -408,6 +423,7 @@ def create(vm_): 'event', 'created instance', 'salt/cloud/{0}/created'.format(vm_['name']), + sock_dir=__opts__['sock_dir'], args=__utils__['cloud.filter_event']('created', vm_, ['name', 'profile', 'provider', 'driver']), transport=__opts__['transport'] ) @@ -429,7 +445,8 @@ def destroy(name, conn=None, call=None): 'event', 'destroying instance', 'salt/cloud/{0}/destroying'.format(name), - {'name': name}, + sock_dir=__opts__['sock_dir'], + args={'name': name}, ) if not conn: @@ -453,7 +470,8 @@ def destroy(name, conn=None, call=None): 'event', 'detaching volume', 'salt/cloud/{0}/detaching'.format(volume.name), - {'name': volume.name}, + sock_dir=__opts__['sock_dir'], + args={'name': volume.name}, ) if not conn.detach_volume(volume): log.error('Failed to Detach volume: {0}'.format(volume.name)) @@ -463,7 +481,8 @@ def destroy(name, conn=None, call=None): 'event', 'detached volume', 'salt/cloud/{0}/detached'.format(volume.name), - {'name': volume.name}, + sock_dir=__opts__['sock_dir'], + args={'name': volume.name}, ) log.info('Destroying volume: {0}'.format(volume.name)) @@ -471,7 +490,8 @@ def destroy(name, conn=None, call=None): 'event', 'destroying volume', 'salt/cloud/{0}/destroying'.format(volume.name), - {'name': volume.name}, + sock_dir=__opts__['sock_dir'], + args={'name': volume.name}, ) if not conn.destroy_volume(volume): log.error('Failed to Destroy volume: {0}'.format(volume.name)) @@ -481,7 +501,8 @@ def destroy(name, conn=None, call=None): 'event', 'destroyed volume', 'salt/cloud/{0}/destroyed'.format(volume.name), - {'name': volume.name}, + sock_dir=__opts__['sock_dir'], + args={'name': volume.name}, ) log.info('Destroying VM: {0}'.format(name)) ret = conn.destroy_node(node) @@ -495,7 +516,8 @@ def destroy(name, conn=None, call=None): 'event', 'destroyed instance', 'salt/cloud/{0}/destroyed'.format(name), - {'name': name}, + sock_dir=__opts__['sock_dir'], + args={'name': name}, ) if __opts__['delete_sshkeys'] is True: salt.utils.cloud.remove_sshkey(node.public_ips[0]) diff --git a/salt/cloud/clouds/digital_ocean.py b/salt/cloud/clouds/digitalocean.py similarity index 97% rename from salt/cloud/clouds/digital_ocean.py rename to salt/cloud/clouds/digitalocean.py index daabcbddfe3..d5bcb4fb6f3 100644 --- a/salt/cloud/clouds/digital_ocean.py +++ b/salt/cloud/clouds/digitalocean.py @@ -20,7 +20,7 @@ under the "SSH Keys" section. personal_access_token: xxx ssh_key_file: /path/to/ssh/key/file ssh_key_names: my-key-name,my-key-name-2 - driver: digital_ocean + driver: digitalocean :depends: requests ''' @@ -36,6 +36,7 @@ import decimal # Import Salt Libs import salt.utils.cloud +import salt.utils.files import salt.config as config from salt.exceptions import ( SaltCloudConfigError, @@ -45,9 +46,8 @@ from salt.exceptions import ( SaltCloudExecutionFailure, SaltCloudExecutionTimeout ) -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import zip -from salt.ext.six import string_types # Import Third Party Libs try: @@ -59,10 +59,11 @@ except ImportError: # Get logging started log = logging.getLogger(__name__) -__virtualname__ = 'digital_ocean' +__virtualname__ = 'digitalocean' +__virtual_aliases__ = ('digital_ocean', 'do') -# Only load in this module if the DIGITAL_OCEAN configurations are in place +# Only load in this module if the DIGITALOCEAN configurations are in place def __virtual__(): ''' Check for DigitalOcean configurations @@ -208,7 +209,7 @@ def get_image(vm_): vm_image = config.get_cloud_config_value( 'image', vm_, __opts__, search_global=False ) - if not isinstance(vm_image, string_types): + if not isinstance(vm_image, six.string_types): vm_image = str(vm_image) for image in images: @@ -274,7 +275,7 @@ def create(vm_): try: # Check for required profile parameters before sending any API calls. if vm_['profile'] and config.is_profile_configured(__opts__, - __active_provider_name__ or 'digital_ocean', + __active_provider_name__ or 'digitalocean', vm_['profile'], vm_=vm_) is False: return False @@ -383,7 +384,7 @@ def create(vm_): ) if userdata_file is not None: try: - with salt.utils.fopen(userdata_file, 'r') as fp_: + with salt.utils.files.fopen(userdata_file, 'r') as fp_: kwargs['user_data'] = salt.utils.cloud.userdata_template( __opts__, vm_, fp_.read() ) @@ -441,7 +442,7 @@ def create(vm_): ret = create_node(kwargs) except Exception as exc: log.error( - 'Error creating {0} on DIGITAL_OCEAN\n\n' + 'Error creating {0} on DIGITALOCEAN\n\n' 'The following exception was thrown when trying to ' 'run the initial deployment: {1}'.format( vm_['name'], @@ -713,15 +714,15 @@ def import_keypair(kwargs=None, call=None): file(mandatory): public key file-name keyname(mandatory): public key name in the provider ''' - with salt.utils.fopen(kwargs['file'], 'r') as public_key_filename: + with salt.utils.files.fopen(kwargs['file'], 'r') as public_key_filename: public_key_content = public_key_filename.read() - digital_ocean_kwargs = { + digitalocean_kwargs = { 'name': kwargs['keyname'], 'public_key': public_key_content } - created_result = create_key(digital_ocean_kwargs, call=call) + created_result = create_key(digitalocean_kwargs, call=call) return created_result @@ -938,11 +939,11 @@ def show_pricing(kwargs=None, call=None): if not profile: return {'Error': 'The requested profile was not found'} - # Make sure the profile belongs to Digital Ocean + # Make sure the profile belongs to DigitalOcean provider = profile.get('provider', '0:0') comps = provider.split(':') - if len(comps) < 2 or comps[1] != 'digital_ocean': - return {'Error': 'The requested profile does not belong to Digital Ocean'} + if len(comps) < 2 or comps[1] != 'digitalocean': + return {'Error': 'The requested profile does not belong to DigitalOcean'} raw = {} ret = {} @@ -968,7 +969,7 @@ def list_floating_ips(call=None): CLI Examples: - ... code-block:: bash + .. code-block:: bash salt-cloud -f list_floating_ips my-digitalocean-config ''' @@ -1008,7 +1009,7 @@ def show_floating_ip(kwargs=None, call=None): CLI Examples: - ... code-block:: bash + .. code-block:: bash salt-cloud -f show_floating_ip my-digitalocean-config floating_ip='45.55.96.47' ''' @@ -1041,7 +1042,7 @@ def create_floating_ip(kwargs=None, call=None): CLI Examples: - ... code-block:: bash + .. code-block:: bash salt-cloud -f create_floating_ip my-digitalocean-config region='NYC2' @@ -1083,7 +1084,7 @@ def delete_floating_ip(kwargs=None, call=None): CLI Examples: - ... code-block:: bash + .. code-block:: bash salt-cloud -f delete_floating_ip my-digitalocean-config floating_ip='45.55.96.47' ''' @@ -1118,7 +1119,7 @@ def assign_floating_ip(kwargs=None, call=None): CLI Examples: - ... code-block:: bash + .. code-block:: bash salt-cloud -f assign_floating_ip my-digitalocean-config droplet_id=1234567 floating_ip='45.55.96.47' ''' @@ -1151,7 +1152,7 @@ def unassign_floating_ip(kwargs=None, call=None): CLI Examples: - ... code-block:: bash + .. code-block:: bash salt-cloud -f unassign_floating_ip my-digitalocean-config floating_ip='45.55.96.47' ''' diff --git a/salt/cloud/clouds/ec2.py b/salt/cloud/clouds/ec2.py index afca0f904db..3dc463c4b12 100644 --- a/salt/cloud/clouds/ec2.py +++ b/salt/cloud/clouds/ec2.py @@ -89,14 +89,12 @@ import re import decimal # Import Salt Libs -import salt.utils +import salt.utils.cloud +import salt.utils.files import salt.utils.hashutils from salt._compat import ElementTree as ET import salt.utils.http as http import salt.utils.aws as aws - -# Import salt.cloud libs -import salt.utils.cloud import salt.config as config from salt.exceptions import ( SaltCloudException, @@ -107,7 +105,7 @@ from salt.exceptions import ( ) # pylint: disable=import-error,no-name-in-module,redefined-builtin -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import map, range, zip from salt.ext.six.moves.urllib.parse import urlparse as _urlparse, urlencode as _urlencode @@ -1030,10 +1028,18 @@ def ssh_interface(vm_): Return the ssh_interface type to connect to. Either 'public_ips' (default) or 'private_ips'. ''' - return config.get_cloud_config_value( + ret = config.get_cloud_config_value( 'ssh_interface', vm_, __opts__, default='public_ips', search_global=False ) + if ret not in ('public_ips', 'private_ips'): + log.warning(( + 'Invalid ssh_interface: {0}. ' + 'Allowed options are ("public_ips", "private_ips"). ' + 'Defaulting to "public_ips".' + ).format(ret)) + ret = 'public_ips' + return ret def get_ssh_gateway_config(vm_): @@ -1046,7 +1052,7 @@ def get_ssh_gateway_config(vm_): ) # Check to see if a SSH Gateway will be used. - if not isinstance(ssh_gateway, str): + if not isinstance(ssh_gateway, six.string_types): return None # Create dictionary of configuration items @@ -1433,7 +1439,7 @@ def _create_eni_if_necessary(interface, vm_): ) associate_public_ip = interface.get('AssociatePublicIpAddress', False) - if isinstance(associate_public_ip, str): + if isinstance(associate_public_ip, six.string_types): # Assume id of EIP as value _associate_eip_with_interface(eni_id, associate_public_ip, vm_=vm_) @@ -1787,7 +1793,7 @@ def request_instance(vm_=None, call=None): else: log.trace('userdata_file: {0}'.format(userdata_file)) if os.path.exists(userdata_file): - with salt.utils.fopen(userdata_file, 'r') as fh_: + with salt.utils.files.fopen(userdata_file, 'r') as fh_: userdata = fh_.read() userdata = salt.utils.cloud.userdata_template(__opts__, vm_, userdata) @@ -2328,6 +2334,9 @@ def wait_for_instance( use_winrm = config.get_cloud_config_value( 'use_winrm', vm_, __opts__, default=False ) + winrm_verify_ssl = config.get_cloud_config_value( + 'winrm_verify_ssl', vm_, __opts__, default=True + ) if win_passwd and win_passwd == 'auto': log.debug('Waiting for auto-generated Windows EC2 password') @@ -2399,7 +2408,8 @@ def wait_for_instance( winrm_port, username, win_passwd, - timeout=ssh_connect_timeout): + timeout=ssh_connect_timeout, + verify=winrm_verify_ssl): raise SaltCloudSystemExit( 'Failed to authenticate against remote windows host' ) @@ -2440,7 +2450,7 @@ def wait_for_instance( continue keys += '\n{0} {1}'.format(ip_address, line) - with salt.utils.fopen(known_hosts_file, 'a') as fp_: + with salt.utils.files.fopen(known_hosts_file, 'a') as fp_: fp_.write(keys) fp_.close() @@ -2609,7 +2619,7 @@ def create(vm_=None, call=None): data, vm_ = request_instance(vm_, location) # If data is a str, it's an error - if isinstance(data, str): + if isinstance(data, six.string_types): log.error('Error requesting instance: {0}'.format(data)) return {} @@ -2642,7 +2652,7 @@ def create(vm_=None, call=None): ) for value in six.itervalues(tags): - if not isinstance(value, str): + if not isinstance(value, six.string_types): raise SaltCloudConfigError( '\'tag\' values must be strings. Try quoting the values. ' 'e.g. "2013-09-19T20:09:46Z".' @@ -2829,7 +2839,7 @@ def create_attach_volumes(name, kwargs, call=None, wait_to_finish=True): if 'instance_id' not in kwargs: kwargs['instance_id'] = _get_node(name)['instanceId'] - if isinstance(kwargs['volumes'], str): + if isinstance(kwargs['volumes'], six.string_types): volumes = yaml.safe_load(kwargs['volumes']) else: volumes = kwargs['volumes'] @@ -3422,34 +3432,7 @@ def list_nodes_full(location=None, call=None): 'or --function.' ) - if not location: - ret = {} - locations = set( - get_location(vm_) for vm_ in six.itervalues(__opts__['profiles']) - if _vm_provider_driver(vm_) - ) - - # If there aren't any profiles defined for EC2, check - # the provider config file, or use the default location. - if not locations: - locations = [get_location()] - - for loc in locations: - ret.update(_list_nodes_full(loc)) - return ret - - return _list_nodes_full(location) - - -def _vm_provider_driver(vm_): - alias, driver = vm_['driver'].split(':') - if alias not in __opts__['providers']: - return None - - if driver not in __opts__['providers'][alias]: - return None - - return driver == 'ec2' + return _list_nodes_full(location or get_location()) def _extract_name_tag(item): @@ -4385,7 +4368,7 @@ def import_keypair(kwargs=None, call=None): public_key_file = kwargs['file'] if os.path.exists(public_key_file): - with salt.utils.fopen(public_key_file, 'r') as fh_: + with salt.utils.files.fopen(public_key_file, 'r') as fh_: public_key = fh_.read() if public_key is not None: @@ -4774,7 +4757,7 @@ def get_password_data( if 'key' not in kwargs: if 'key_file' in kwargs: - with salt.utils.fopen(kwargs['key_file'], 'r') as kf_: + with salt.utils.files.fopen(kwargs['key_file'], 'r') as kf_: kwargs['key'] = kf_.read() if 'key' in kwargs: @@ -4874,7 +4857,7 @@ def _parse_pricing(url, name): outfile = os.path.join( __opts__['cachedir'], 'ec2-pricing-{0}.p'.format(name) ) - with salt.utils.fopen(outfile, 'w') as fho: + with salt.utils.files.fopen(outfile, 'w') as fho: msgpack.dump(regions, fho) return True @@ -4942,7 +4925,7 @@ def show_pricing(kwargs=None, call=None): if not os.path.isfile(pricefile): update_pricing({'type': name}, 'function') - with salt.utils.fopen(pricefile, 'r') as fhi: + with salt.utils.files.fopen(pricefile, 'r') as fhi: ec2_price = msgpack.load(fhi) region = get_location(profile) diff --git a/salt/cloud/clouds/gce.py b/salt/cloud/clouds/gce.py index 950fa533e60..6e26cf8b958 100644 --- a/salt/cloud/clouds/gce.py +++ b/salt/cloud/clouds/gce.py @@ -83,10 +83,11 @@ except ImportError: # Import salt libs from salt.utils import namespaced_function -import salt.ext.six as six +from salt.ext import six import salt.utils.cloud +import salt.utils.files +import salt.utils.http import salt.config as config -from salt.utils import http from salt.cloud.libcloudfuncs import * # pylint: disable=redefined-builtin,wildcard-import,unused-wildcard-import from salt.exceptions import ( SaltCloudSystemExit, @@ -2614,12 +2615,12 @@ def update_pricing(kwargs=None, call=None): .. versionadded:: 2015.8.0 ''' url = 'https://cloudpricingcalculator.appspot.com/static/data/pricelist.json' - price_json = http.query(url, decode=True, decode_type='json') + price_json = salt.utils.http.query(url, decode=True, decode_type='json') outfile = os.path.join( __opts__['cachedir'], 'gce-pricing.p' ) - with salt.utils.fopen(outfile, 'w') as fho: + with salt.utils.files.fopen(outfile, 'w') as fho: msgpack.dump(price_json['dict'], fho) return True @@ -2642,7 +2643,7 @@ def show_pricing(kwargs=None, call=None): if not profile: return {'Error': 'The requested profile was not found'} - # Make sure the profile belongs to Digital Ocean + # Make sure the profile belongs to DigitalOcean provider = profile.get('provider', '0:0') comps = provider.split(':') if len(comps) < 2 or comps[1] != 'gce': @@ -2658,7 +2659,7 @@ def show_pricing(kwargs=None, call=None): if not os.path.exists(pricefile): update_pricing() - with salt.utils.fopen(pricefile, 'r') as fho: + with salt.utils.files.fopen(pricefile, 'r') as fho: sizes = msgpack.load(fho) per_hour = float(sizes['gcp_price_list'][size][region]) diff --git a/salt/cloud/clouds/joyent.py b/salt/cloud/clouds/joyent.py index 3a2f119f9f0..8f40c28657b 100644 --- a/salt/cloud/clouds/joyent.py +++ b/salt/cloud/clouds/joyent.py @@ -65,12 +65,12 @@ from Crypto.PublicKey import RSA from Crypto.Signature import PKCS1_v1_5 # Import salt libs -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import http_client # pylint: disable=import-error,no-name-in-module -import salt.utils.http import salt.utils.cloud +import salt.utils.files +import salt.utils.http import salt.config as config -from salt.utils.cloud import is_public_ip from salt.cloud.libcloudfuncs import node_state from salt.exceptions import ( SaltCloudSystemExit, @@ -693,7 +693,7 @@ def reformat_node(item=None, full=False): item['public_ips'] = [] if 'ips' in item: for ip in item['ips']: - if is_public_ip(ip): + if salt.utils.cloud.is_public_ip(ip): item['public_ips'].append(ip) else: item['private_ips'].append(ip) @@ -951,7 +951,7 @@ def import_key(kwargs=None, call=None): )) return False - with salt.utils.fopen(kwargs['keyfile'], 'r') as fp_: + with salt.utils.files.fopen(kwargs['keyfile'], 'r') as fp_: kwargs['key'] = fp_.read() send_data = {'name': kwargs['keyname'], 'key': kwargs['key']} @@ -1070,11 +1070,11 @@ def query(action=None, timenow = datetime.datetime.utcnow() timestamp = timenow.strftime('%a, %d %b %Y %H:%M:%S %Z').strip() - with salt.utils.fopen(ssh_keyfile, 'r') as kh_: - rsa_key = RSA.importKey(kh_) + with salt.utils.files.fopen(ssh_keyfile, 'r') as kh_: + rsa_key = RSA.importKey(kh_.read()) rsa_ = PKCS1_v1_5.new(rsa_key) hash_ = SHA256.new() - hash_.update(timestamp) + hash_.update(timestamp.encode(__salt_system_encoding__)) signed = base64.b64encode(rsa_.sign(hash_)) keyid = '/{0}/keys/{1}'.format(user.split('/')[0], ssh_keyname) @@ -1085,7 +1085,7 @@ def query(action=None, 'Date': timestamp, 'Authorization': 'Signature keyId="{0}",algorithm="rsa-sha256" {1}'.format( keyid, - signed + signed.decode(__salt_system_encoding__) ), } diff --git a/salt/cloud/clouds/libvirt.py b/salt/cloud/clouds/libvirt.py index 2bc0b36f529..c3a1b56c0e6 100644 --- a/salt/cloud/clouds/libvirt.py +++ b/salt/cloud/clouds/libvirt.py @@ -41,6 +41,7 @@ Example profile: master_port: 5506 Tested on: +- Fedora 26 (libvirt 3.2.1, qemu 2.9.1) - Fedora 25 (libvirt 1.3.3.2, qemu 2.6.1) - Fedora 23 (libvirt 1.2.18, qemu 2.4.1) - Centos 7 (libvirt 1.2.17, qemu 1.5.3) @@ -63,7 +64,7 @@ import os from xml.etree import ElementTree # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six try: import libvirt # pylint: disable=import-error from libvirt import libvirtError @@ -82,9 +83,6 @@ from salt.exceptions import ( SaltCloudSystemExit ) -# Get logging started -log = logging.getLogger(__name__) - VIRT_STATE_NAME_MAP = {0: 'running', 1: 'running', 2: 'running', @@ -99,6 +97,20 @@ IP_LEARNING_XML = """ __virtualname__ = 'libvirt' +# Set up logging +log = logging.getLogger(__name__) + + +def libvirt_error_handler(ctx, error): + ''' + Redirect stderr prints from libvirt to salt logging. + ''' + log.debug("libvirt error {0}".format(error)) + + +if HAS_LIBVIRT: + libvirt.registerErrorHandler(f=libvirt_error_handler, ctx=None) + def __virtual__(): ''' @@ -280,7 +292,7 @@ def create(vm_): validate_xml = vm_.get('validate_xml') if vm_.get('validate_xml') is not None else True - log.info("Cloning machine '{0}' with strategy '{1}' validate_xml='{2}'".format(vm_['name'], clone_strategy, validate_xml)) + log.info("Cloning '{0}' with strategy '{1}' validate_xml='{2}'".format(vm_['name'], clone_strategy, validate_xml)) try: # Check for required profile parameters before sending any API calls. @@ -516,7 +528,7 @@ def destroy(name, call=None): 'event', 'destroying instance', 'salt/cloud/{0}/destroying'.format(name), - {'name': name}, + args={'name': name}, sock_dir=__opts__['sock_dir'], transport=__opts__['transport'] ) @@ -527,7 +539,7 @@ def destroy(name, call=None): 'event', 'destroyed instance', 'salt/cloud/{0}/destroyed'.format(name), - {'name': name}, + args={'name': name}, sock_dir=__opts__['sock_dir'], transport=__opts__['transport'] ) diff --git a/salt/cloud/clouds/linode.py b/salt/cloud/clouds/linode.py index 990767ece4d..b64b69685cf 100644 --- a/salt/cloud/clouds/linode.py +++ b/salt/cloud/clouds/linode.py @@ -35,7 +35,7 @@ import datetime # Import Salt Libs import salt.config as config -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import range from salt.exceptions import ( SaltCloudConfigError, @@ -44,9 +44,6 @@ from salt.exceptions import ( SaltCloudSystemExit ) -# Import Salt-Cloud Libs -import salt.utils.cloud - # Get logging started log = logging.getLogger(__name__) @@ -524,6 +521,7 @@ def create(vm_): 'event', 'waiting for ssh', 'salt/cloud/{0}/waiting_for_ssh'.format(name), + sock_dir=__opts__['sock_dir'], args={'ip_address': vm_['ssh_host']}, transport=__opts__['transport'] ) @@ -1192,7 +1190,7 @@ def list_nodes_select(call=None): ''' Return a list of the VMs that are on the provider, with select fields. ''' - return salt.utils.cloud.list_nodes_select( + return __utils__['cloud.list_nodes_select']( list_nodes_full(), __opts__['query.selection'], call, ) @@ -1502,7 +1500,7 @@ def _query(action=None, if LASTCALL >= now: time.sleep(ratelimit_sleep) - result = salt.utils.http.query( + result = __utils__['http.query']( url, method, params=args, diff --git a/salt/cloud/clouds/lxc.py b/salt/cloud/clouds/lxc.py index 48aa6cc1fd9..3043874e3fe 100644 --- a/salt/cloud/clouds/lxc.py +++ b/salt/cloud/clouds/lxc.py @@ -30,7 +30,7 @@ import salt.runner # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six # Get logging started log = logging.getLogger(__name__) diff --git a/salt/cloud/clouds/msazure.py b/salt/cloud/clouds/msazure.py index 4a41f4a4408..31f7186177a 100644 --- a/salt/cloud/clouds/msazure.py +++ b/salt/cloud/clouds/msazure.py @@ -52,6 +52,7 @@ from salt.exceptions import SaltCloudSystemExit import salt.utils.cloud # Import 3rd-party libs +from salt.ext import six HAS_LIBS = False try: import azure @@ -709,7 +710,7 @@ def create_attach_volumes(name, kwargs, call=None, wait_to_finish=True): if kwargs is None: kwargs = {} - if isinstance(kwargs['volumes'], str): + if isinstance(kwargs['volumes'], six.string_types): volumes = yaml.safe_load(kwargs['volumes']) else: volumes = kwargs['volumes'] @@ -803,7 +804,7 @@ def create_attach_volumes(name, kwargs, call=None, wait_to_finish=True): if kwargs is None: kwargs = {} - if isinstance(kwargs['volumes'], str): + if isinstance(kwargs['volumes'], six.string_types): volumes = yaml.safe_load(kwargs['volumes']) else: volumes = kwargs['volumes'] diff --git a/salt/cloud/clouds/nova.py b/salt/cloud/clouds/nova.py index 6cf43eb0af2..d59600f04af 100644 --- a/salt/cloud/clouds/nova.py +++ b/salt/cloud/clouds/nova.py @@ -209,8 +209,10 @@ import pprint import yaml # Import Salt Libs -import salt.ext.six as six -import salt.utils +from salt.ext import six +import salt.utils.cloud +import salt.utils.files +import salt.utils.pycrypto import salt.client from salt.utils.openstack import nova try: @@ -220,8 +222,6 @@ except ImportError as exc: # Import Salt Cloud Libs from salt.cloud.libcloudfuncs import * # pylint: disable=W0614,W0401 -import salt.utils.cloud -import salt.utils.pycrypto as sup import salt.config as config from salt.utils import namespaced_function from salt.exceptions import ( @@ -651,7 +651,7 @@ def request_instance(vm_=None, call=None): kwargs['files'] = {} for src_path in files: if os.path.exists(files[src_path]): - with salt.utils.fopen(files[src_path], 'r') as fp_: + with salt.utils.files.fopen(files[src_path], 'r') as fp_: kwargs['files'][src_path] = fp_.read() else: kwargs['files'][src_path] = files[src_path] @@ -661,7 +661,7 @@ def request_instance(vm_=None, call=None): ) if userdata_file is not None: try: - with salt.utils.fopen(userdata_file, 'r') as fp_: + with salt.utils.files.fopen(userdata_file, 'r') as fp_: kwargs['userdata'] = salt.utils.cloud.userdata_template( __opts__, vm_, fp_.read() ) @@ -728,12 +728,18 @@ def request_instance(vm_=None, call=None): else: pool = floating_ip_conf.get('pool', 'public') - for fl_ip, opts in six.iteritems(conn.floating_ip_list()): - if opts['fixed_ip'] is None and opts['pool'] == pool: - floating_ip = fl_ip - break - if floating_ip is None: + try: floating_ip = conn.floating_ip_create(pool)['ip'] + except Exception: + log.info('A new IP address was unable to be allocated. ' + 'An IP address will be pulled from the already allocated list, ' + 'This will cause a race condition when building in parallel.') + for fl_ip, opts in six.iteritems(conn.floating_ip_list()): + if opts['fixed_ip'] is None and opts['pool'] == pool: + floating_ip = fl_ip + break + if floating_ip is None: + log.error('No IP addresses available to allocate for this server: {0}'.format(vm_['name'])) def __query_node_data(vm_): try: @@ -981,7 +987,7 @@ def create(vm_): ) data = conn.server_show_libcloud(vm_['instance_id']) if vm_['key_filename'] is None and 'change_password' in __opts__ and __opts__['change_password'] is True: - vm_['password'] = sup.secure_password() + vm_['password'] = salt.utils.pycrypto.secure_password() conn.root_password(vm_['instance_id'], vm_['password']) else: # Put together all of the information required to request the instance, diff --git a/salt/cloud/clouds/oneandone.py b/salt/cloud/clouds/oneandone.py new file mode 100644 index 00000000000..73ed0c853a1 --- /dev/null +++ b/salt/cloud/clouds/oneandone.py @@ -0,0 +1,849 @@ +# -*- coding: utf-8 -*- +''' +1&1 Cloud Server Module +======================= + +======= +The 1&1 SaltStack cloud module allows a 1&1 server to +be automatically deployed and bootstrapped with Salt. + +:depends: 1and1 >= 1.2.0 + +The module requires the 1&1 api_token to be provided. +The server should also be assigned a public LAN, a private LAN, +or both along with SSH key pairs. +... + +Set up the cloud configuration at ``/etc/salt/cloud.providers`` or +``/etc/salt/cloud.providers.d/oneandone.conf``: + +.. code-block:: yaml + + my-oneandone-config: + driver: oneandone + # The 1&1 api token + api_token: + # SSH private key filename + ssh_private_key: /path/to/private_key + # SSH public key filename + ssh_public_key: /path/to/public_key + +.. code-block:: yaml + + my-oneandone-profile: + provider: my-oneandone-config + # Either provide fixed_instance_size_id or vcore, cores_per_processor, ram, and hdds. + # Size of the ID desired for the server + fixed_instance_size: S + # Total amount of processors + vcore: 2 + # Number of cores per processor + cores_per_processor: 2 + # RAM memory size in GB + ram: 4 + # Hard disks + hdds: + - + is_main: true + size: 20 + - + is_main: false + size: 20 + # ID of the appliance image that will be installed on server + appliance_id: + # ID of the datacenter where the server will be created + datacenter_id: + # Description of the server + description: My server description + # Password of the server. Password must contain more than 8 characters + # using uppercase letters, numbers and other special symbols. + password: P4$$w0rD + # Power on server after creation - default True + power_on: true + # Firewall policy ID. If it is not provided, the server will assign + # the best firewall policy, creating a new one if necessary. + # If the parameter is sent with a 0 value, the server will be created with all ports blocked. + firewall_policy_id: + # IP address ID + ip_id: + # Load balancer ID + load_balancer_id: + # Monitoring policy ID + monitoring_policy_id: + +Set ``deploy`` to False if Salt should not be installed on the node. + +.. code-block:: yaml + + my-oneandone-profile: + deploy: False +''' + +# Import python libs +from __future__ import absolute_import +import logging +import os +import pprint +import time + +# Import salt libs +import salt.config as config +from salt.exceptions import ( + SaltCloudConfigError, + SaltCloudNotFound, + SaltCloudExecutionFailure, + SaltCloudExecutionTimeout, + SaltCloudSystemExit +) +import salt.utils.files + +# Import salt.cloud libs +import salt.utils.cloud +from salt.ext import six + +try: + from oneandone.client import ( + OneAndOneService, Server, Hdd + ) + HAS_ONEANDONE = True +except ImportError: + HAS_ONEANDONE = False + +# Get logging started +log = logging.getLogger(__name__) + +__virtualname__ = 'oneandone' + + +# Only load in this module if the 1&1 configurations are in place +def __virtual__(): + ''' + Check for 1&1 configurations. + ''' + if get_configured_provider() is False: + return False + + if get_dependencies() is False: + return False + + return __virtualname__ + + +def get_configured_provider(): + ''' + Return the first configured instance. + ''' + return config.is_provider_configured( + __opts__, + __active_provider_name__ or __virtualname__, + ('api_token',) + ) + + +def get_dependencies(): + ''' + Warn if dependencies are not met. + ''' + return config.check_driver_dependencies( + __virtualname__, + {'oneandone': HAS_ONEANDONE} + ) + + +def get_conn(): + ''' + Return a conn object for the passed VM data + ''' + return OneAndOneService( + api_token=config.get_cloud_config_value( + 'api_token', + get_configured_provider(), + __opts__, + search_global=False + ) + ) + + +def get_size(vm_): + ''' + Return the VM's size object + ''' + vm_size = config.get_cloud_config_value( + 'fixed_instance_size', vm_, __opts__, default=None, + search_global=False + ) + sizes = avail_sizes() + + if not vm_size: + size = next((item for item in sizes if item['name'] == 'S'), None) + return size + + size = next((item for item in sizes if item['name'] == vm_size or item['id'] == vm_size), None) + if size: + return size + + raise SaltCloudNotFound( + 'The specified size, \'{0}\', could not be found.'.format(vm_size) + ) + + +def get_image(vm_): + ''' + Return the image object to use + ''' + vm_image = config.get_cloud_config_value('image', vm_, __opts__).encode( + 'ascii', 'salt-cloud-force-ascii' + ) + + images = avail_images() + for key, value in six.iteritems(images): + if vm_image and vm_image in (images[key]['id'], images[key]['name']): + return images[key] + + raise SaltCloudNotFound( + 'The specified image, \'{0}\', could not be found.'.format(vm_image) + ) + + +def avail_locations(conn=None, call=None): + ''' + List available locations/datacenters for 1&1 + ''' + if call == 'action': + raise SaltCloudSystemExit( + 'The avail_locations function must be called with ' + '-f or --function, or with the --list-locations option' + ) + + datacenters = [] + + if not conn: + conn = get_conn() + + for datacenter in conn.list_datacenters(): + datacenters.append({datacenter['country_code']: datacenter}) + + return {'Locations': datacenters} + + +def avail_images(conn=None, call=None): + ''' + Return a list of the server appliances that are on the provider + ''' + if call == 'action': + raise SaltCloudSystemExit( + 'The avail_images function must be called with ' + '-f or --function, or with the --list-images option' + ) + + if not conn: + conn = get_conn() + + ret = {} + + for appliance in conn.list_appliances(): + ret[appliance['name']] = appliance + + return ret + + +def avail_sizes(call=None): + ''' + Return a dict of all available VM sizes on the cloud provider with + relevant data. + ''' + if call == 'action': + raise SaltCloudSystemExit( + 'The avail_sizes function must be called with ' + '-f or --function, or with the --list-sizes option' + ) + + conn = get_conn() + + sizes = conn.fixed_server_flavors() + + return sizes + + +def script(vm_): + ''' + Return the script deployment object + ''' + return salt.utils.cloud.os_script( + config.get_cloud_config_value('script', vm_, __opts__), + vm_, + __opts__, + salt.utils.cloud.salt_config_to_yaml( + salt.utils.cloud.minion_config(__opts__, vm_) + ) + ) + + +def list_nodes(conn=None, call=None): + ''' + Return a list of VMs that are on the provider + ''' + if call == 'action': + raise SaltCloudSystemExit( + 'The list_nodes function must be called with -f or --function.' + ) + + if not conn: + conn = get_conn() + + ret = {} + nodes = conn.list_servers() + + for node in nodes: + public_ips = [] + private_ips = [] + ret = {} + + size = node.get('hardware').get('fixed_instance_size_id', 'Custom size') + + if node.get('private_networks') and len(node['private_networks']) > 0: + for private_ip in node['private_networks']: + private_ips.append(private_ip) + + if node.get('ips') and len(node['ips']) > 0: + for public_ip in node['ips']: + public_ips.append(public_ip['ip']) + + server = { + 'id': node['id'], + 'image': node['image']['id'], + 'size': size, + 'state': node['status']['state'], + 'private_ips': private_ips, + 'public_ips': public_ips + } + ret[node['name']] = server + + return ret + + +def list_nodes_full(conn=None, call=None): + ''' + Return a list of the VMs that are on the provider, with all fields + ''' + if call == 'action': + raise SaltCloudSystemExit( + 'The list_nodes_full function must be called with -f or ' + '--function.' + ) + + if not conn: + conn = get_conn() + + ret = {} + nodes = conn.list_servers() + + for node in nodes: + ret[node['name']] = node + + return ret + + +def list_nodes_select(conn=None, call=None): + ''' + Return a list of the VMs that are on the provider, with select fields + ''' + if not conn: + conn = get_conn() + + return salt.utils.cloud.list_nodes_select( + list_nodes_full(conn, 'function'), + __opts__['query.selection'], + call, + ) + + +def show_instance(name, call=None): + ''' + Show the details from the provider concerning an instance + ''' + if call != 'action': + raise SaltCloudSystemExit( + 'The show_instance action must be called with -a or --action.' + ) + + nodes = list_nodes_full() + __utils__['cloud.cache_node']( + nodes[name], + __active_provider_name__, + __opts__ + ) + return nodes[name] + + +def _get_server(vm_): + ''' + Construct server instance from cloud profile config + ''' + description = config.get_cloud_config_value( + 'description', vm_, __opts__, default=None, + search_global=False + ) + + ssh_key = load_public_key(vm_) + + vcore = None + cores_per_processor = None + ram = None + fixed_instance_size_id = None + + if 'fixed_instance_size' in vm_: + fixed_instance_size = get_size(vm_) + fixed_instance_size_id = fixed_instance_size['id'] + elif (vm_['vcore'] and vm_['cores_per_processor'] and + vm_['ram'] and vm_['hdds']): + vcore = config.get_cloud_config_value( + 'vcore', vm_, __opts__, default=None, + search_global=False + ) + cores_per_processor = config.get_cloud_config_value( + 'cores_per_processor', vm_, __opts__, default=None, + search_global=False + ) + ram = config.get_cloud_config_value( + 'ram', vm_, __opts__, default=None, + search_global=False + ) + else: + raise SaltCloudConfigError("'fixed_instance_size' or 'vcore'," + "'cores_per_processor', 'ram', and 'hdds'" + "must be provided.") + + appliance_id = config.get_cloud_config_value( + 'appliance_id', vm_, __opts__, default=None, + search_global=False + ) + + password = config.get_cloud_config_value( + 'password', vm_, __opts__, default=None, + search_global=False + ) + + firewall_policy_id = config.get_cloud_config_value( + 'firewall_policy_id', vm_, __opts__, default=None, + search_global=False + ) + + ip_id = config.get_cloud_config_value( + 'ip_id', vm_, __opts__, default=None, + search_global=False + ) + + load_balancer_id = config.get_cloud_config_value( + 'load_balancer_id', vm_, __opts__, default=None, + search_global=False + ) + + monitoring_policy_id = config.get_cloud_config_value( + 'monitoring_policy_id', vm_, __opts__, default=None, + search_global=False + ) + + datacenter_id = config.get_cloud_config_value( + 'datacenter_id', vm_, __opts__, default=None, + search_global=False + ) + + private_network_id = config.get_cloud_config_value( + 'private_network_id', vm_, __opts__, default=None, + search_global=False + ) + + power_on = config.get_cloud_config_value( + 'power_on', vm_, __opts__, default=True, + search_global=False + ) + + # Contruct server object + return Server( + name=vm_['name'], + description=description, + fixed_instance_size_id=fixed_instance_size_id, + vcore=vcore, + cores_per_processor=cores_per_processor, + ram=ram, + appliance_id=appliance_id, + password=password, + power_on=power_on, + firewall_policy_id=firewall_policy_id, + ip_id=ip_id, + load_balancer_id=load_balancer_id, + monitoring_policy_id=monitoring_policy_id, + datacenter_id=datacenter_id, + rsa_key=ssh_key, + private_network_id=private_network_id + ) + + +def _get_hdds(vm_): + ''' + Construct VM hdds from cloud profile config + ''' + _hdds = config.get_cloud_config_value( + 'hdds', vm_, __opts__, default=None, + search_global=False + ) + + hdds = [] + + for hdd in _hdds: + hdds.append( + Hdd( + size=hdd['size'], + is_main=hdd['is_main'] + ) + ) + + return hdds + + +def create(vm_): + ''' + Create a single VM from a data dict + ''' + try: + # Check for required profile parameters before sending any API calls. + if (vm_['profile'] and + config.is_profile_configured(__opts__, + (__active_provider_name__ or + 'oneandone'), + vm_['profile']) is False): + return False + except AttributeError: + pass + + data = None + conn = get_conn() + hdds = [] + + # Assemble the composite server object. + server = _get_server(vm_) + + if not bool(server.specs['hardware']['fixed_instance_size_id']): + # Assemble the hdds object. + hdds = _get_hdds(vm_) + + __utils__['cloud.fire_event']( + 'event', + 'requesting instance', + 'salt/cloud/{0}/requesting'.format(vm_['name']), + args={'name': vm_['name']}, + sock_dir=__opts__['sock_dir'], + transport=__opts__['transport'] + ) + + try: + data = conn.create_server(server=server, hdds=hdds) + + _wait_for_completion(conn, + get_wait_timeout(vm_), + data['id']) + except Exception as exc: # pylint: disable=W0703 + log.error( + 'Error creating {0} on 1and1\n\n' + 'The following exception was thrown by the 1and1 library ' + 'when trying to run the initial deployment: \n{1}'.format( + vm_['name'], exc + ), + exc_info_on_loglevel=logging.DEBUG + ) + return False + + vm_['server_id'] = data['id'] + password = data['first_password'] + + def __query_node_data(vm_, data): + ''' + Query node data until node becomes available. + ''' + running = False + try: + data = show_instance(vm_['name'], 'action') + if not data: + return False + log.debug( + 'Loaded node data for {0}:\nname: {1}\nstate: {2}'.format( + vm_['name'], + pprint.pformat(data['name']), + data['status']['state'] + ) + ) + except Exception as err: + log.error( + 'Failed to get nodes list: {0}'.format( + err + ), + # Show the trackback if the debug logging level is enabled + exc_info_on_loglevel=logging.DEBUG + ) + # Trigger a failure in the wait for IP function + return False + + running = data['status']['state'].lower() == 'powered_on' + if not running: + # Still not running, trigger another iteration + return + + vm_['ssh_host'] = data['ips'][0]['ip'] + + return data + + try: + data = salt.utils.cloud.wait_for_ip( + __query_node_data, + update_args=(vm_, data), + timeout=config.get_cloud_config_value( + 'wait_for_ip_timeout', vm_, __opts__, default=10 * 60), + interval=config.get_cloud_config_value( + 'wait_for_ip_interval', vm_, __opts__, default=10), + ) + except (SaltCloudExecutionTimeout, SaltCloudExecutionFailure) as exc: + try: + # It might be already up, let's destroy it! + destroy(vm_['name']) + except SaltCloudSystemExit: + pass + finally: + raise SaltCloudSystemExit(str(exc.message)) + + log.debug('VM is now running') + log.info('Created Cloud VM {0}'.format(vm_)) + log.debug( + '{0} VM creation details:\n{1}'.format( + vm_, pprint.pformat(data) + ) + ) + + __utils__['cloud.fire_event']( + 'event', + 'created instance', + 'salt/cloud/{0}/created'.format(vm_['name']), + args={ + 'name': vm_['name'], + 'profile': vm_['profile'], + 'provider': vm_['driver'], + }, + sock_dir=__opts__['sock_dir'], + transport=__opts__['transport'] + ) + + if 'ssh_host' in vm_: + vm_['password'] = password + vm_['key_filename'] = get_key_filename(vm_) + ret = __utils__['cloud.bootstrap'](vm_, __opts__) + ret.update(data) + return ret + else: + raise SaltCloudSystemExit('A valid IP address was not found.') + + +def destroy(name, call=None): + ''' + destroy a server by name + + :param name: name given to the server + :param call: call value in this case is 'action' + :return: array of booleans , true if successfully stopped and true if + successfully removed + + CLI Example: + + .. code-block:: bash + + salt-cloud -d vm_name + + ''' + if call == 'function': + raise SaltCloudSystemExit( + 'The destroy action must be called with -d, --destroy, ' + '-a or --action.' + ) + + __utils__['cloud.fire_event']( + 'event', + 'destroying instance', + 'salt/cloud/{0}/destroying'.format(name), + args={'name': name}, + sock_dir=__opts__['sock_dir'], + transport=__opts__['transport'] + ) + + conn = get_conn() + node = get_node(conn, name) + + conn.delete_server(server_id=node['id']) + + __utils__['cloud.fire_event']( + 'event', + 'destroyed instance', + 'salt/cloud/{0}/destroyed'.format(name), + args={'name': name}, + sock_dir=__opts__['sock_dir'], + transport=__opts__['transport'] + ) + + if __opts__.get('update_cachedir', False) is True: + __utils__['cloud.delete_minion_cachedir']( + name, + __active_provider_name__.split(':')[0], + __opts__ + ) + + return True + + +def reboot(name, call=None): + ''' + reboot a server by name + :param name: name given to the machine + :param call: call value in this case is 'action' + :return: true if successful + + CLI Example: + + .. code-block:: bash + + salt-cloud -a reboot vm_name + ''' + conn = get_conn() + node = get_node(conn, name) + + conn.modify_server_status(server_id=node['id'], action='REBOOT') + + return True + + +def stop(name, call=None): + ''' + stop a server by name + :param name: name given to the machine + :param call: call value in this case is 'action' + :return: true if successful + + CLI Example: + + .. code-block:: bash + + salt-cloud -a stop vm_name + ''' + conn = get_conn() + node = get_node(conn, name) + + conn.stop_server(server_id=node['id']) + + return True + + +def start(name, call=None): + ''' + start a server by name + :param name: name given to the machine + :param call: call value in this case is 'action' + :return: true if successful + + + CLI Example: + + .. code-block:: bash + + salt-cloud -a start vm_name + ''' + conn = get_conn() + node = get_node(conn, name) + + conn.start_server(server_id=node['id']) + + return True + + +def get_node(conn, name): + ''' + Return a node for the named VM + ''' + for node in conn.list_servers(per_page=1000): + if node['name'] == name: + return node + + +def get_key_filename(vm_): + ''' + Check SSH private key file and return absolute path if exists. + ''' + key_filename = config.get_cloud_config_value( + 'ssh_private_key', vm_, __opts__, search_global=False, default=None + ) + if key_filename is not None: + key_filename = os.path.expanduser(key_filename) + if not os.path.isfile(key_filename): + raise SaltCloudConfigError( + 'The defined ssh_private_key \'{0}\' does not exist'.format( + key_filename + ) + ) + + return key_filename + + +def load_public_key(vm_): + ''' + Load the public key file if exists. + ''' + public_key_filename = config.get_cloud_config_value( + 'ssh_public_key', vm_, __opts__, search_global=False, default=None + ) + if public_key_filename is not None: + public_key_filename = os.path.expanduser(public_key_filename) + if not os.path.isfile(public_key_filename): + raise SaltCloudConfigError( + 'The defined ssh_public_key \'{0}\' does not exist'.format( + public_key_filename + ) + ) + + with salt.utils.files.fopen(public_key_filename, 'r') as public_key: + key = public_key.read().replace('\n', '') + + return key + + +def get_wait_timeout(vm_): + ''' + Return the wait_for_timeout for resource provisioning. + ''' + return config.get_cloud_config_value( + 'wait_for_timeout', vm_, __opts__, default=15 * 60, + search_global=False + ) + + +def _wait_for_completion(conn, wait_timeout, server_id): + ''' + Poll request status until resource is provisioned. + ''' + wait_timeout = time.time() + wait_timeout + while wait_timeout > time.time(): + time.sleep(5) + + server = conn.get_server(server_id) + server_state = server['status']['state'].lower() + + if server_state == "powered_on": + return + elif server_state == 'failed': + raise Exception('Server creation failed for {0}'.format(server_id)) + elif server_state in ('active', + 'enabled', + 'deploying', + 'configuring'): + continue + else: + raise Exception( + 'Unknown server state {0}'.format(server_state)) + raise Exception( + 'Timed out waiting for server create completion for {0}'.format(server_id) + ) diff --git a/salt/cloud/clouds/opennebula.py b/salt/cloud/clouds/opennebula.py index 1e660c40225..55aa531d0de 100644 --- a/salt/cloud/clouds/opennebula.py +++ b/salt/cloud/clouds/opennebula.py @@ -77,6 +77,7 @@ from salt.exceptions import ( SaltCloudSystemExit ) import salt.utils +import salt.utils.files # Import Third Party Libs try: @@ -1310,7 +1311,7 @@ def image_allocate(call=None, kwargs=None): '\'data\' will take precedence.' ) elif path: - with salt.utils.fopen(path, mode='r') as rfh: + with salt.utils.files.fopen(path, mode='r') as rfh: data = rfh.read() else: raise SaltCloudSystemExit( @@ -1876,7 +1877,7 @@ def image_update(call=None, kwargs=None): '\'data\' will take precedence.' ) elif path: - with salt.utils.fopen(path, mode='r') as rfh: + with salt.utils.files.fopen(path, mode='r') as rfh: data = rfh.read() else: raise SaltCloudSystemExit( @@ -1969,7 +1970,7 @@ def secgroup_allocate(call=None, kwargs=None): '\'data\' will take precedence.' ) elif path: - with salt.utils.fopen(path, mode='r') as rfh: + with salt.utils.files.fopen(path, mode='r') as rfh: data = rfh.read() else: raise SaltCloudSystemExit( @@ -2269,7 +2270,7 @@ def secgroup_update(call=None, kwargs=None): '\'data\' will take precedence.' ) elif path: - with salt.utils.fopen(path, mode='r') as rfh: + with salt.utils.files.fopen(path, mode='r') as rfh: data = rfh.read() else: raise SaltCloudSystemExit( @@ -2335,7 +2336,7 @@ def template_allocate(call=None, kwargs=None): '\'data\' will take precedence.' ) elif path: - with salt.utils.fopen(path, mode='r') as rfh: + with salt.utils.files.fopen(path, mode='r') as rfh: data = rfh.read() else: raise SaltCloudSystemExit( @@ -2650,7 +2651,7 @@ def template_update(call=None, kwargs=None): '\'data\' will take precedence.' ) elif path: - with salt.utils.fopen(path, mode='r') as rfh: + with salt.utils.files.fopen(path, mode='r') as rfh: data = rfh.read() else: raise SaltCloudSystemExit( @@ -2783,7 +2784,7 @@ def vm_allocate(call=None, kwargs=None): '\'data\' will take precedence.' ) elif path: - with salt.utils.fopen(path, mode='r') as rfh: + with salt.utils.files.fopen(path, mode='r') as rfh: data = rfh.read() else: raise SaltCloudSystemExit( @@ -2849,7 +2850,7 @@ def vm_attach(name, kwargs=None, call=None): '\'data\' will take precedence.' ) elif path: - with salt.utils.fopen(path, mode='r') as rfh: + with salt.utils.files.fopen(path, mode='r') as rfh: data = rfh.read() else: raise SaltCloudSystemExit( @@ -2916,7 +2917,7 @@ def vm_attach_nic(name, kwargs=None, call=None): '\'data\' will take precedence.' ) elif path: - with salt.utils.fopen(path, mode='r') as rfh: + with salt.utils.files.fopen(path, mode='r') as rfh: data = rfh.read() else: raise SaltCloudSystemExit( @@ -3603,7 +3604,7 @@ def vm_resize(name, kwargs=None, call=None): '\'data\' will take precedence.' ) elif path: - with salt.utils.fopen(path, mode='r') as rfh: + with salt.utils.files.fopen(path, mode='r') as rfh: data = rfh.read() else: raise SaltCloudSystemExit( @@ -3831,7 +3832,7 @@ def vm_update(name, kwargs=None, call=None): '\'data\' will take precedence.' ) elif path: - with salt.utils.fopen(path, mode='r') as rfh: + with salt.utils.files.fopen(path, mode='r') as rfh: data = rfh.read() else: raise SaltCloudSystemExit( @@ -3919,7 +3920,7 @@ def vn_add_ar(call=None, kwargs=None): '\'data\' will take precedence.' ) elif path: - with salt.utils.fopen(path, mode='r') as rfh: + with salt.utils.files.fopen(path, mode='r') as rfh: data = rfh.read() else: raise SaltCloudSystemExit( @@ -3992,7 +3993,7 @@ def vn_allocate(call=None, kwargs=None): '\'data\' will take precedence.' ) elif path: - with salt.utils.fopen(path, mode='r') as rfh: + with salt.utils.files.fopen(path, mode='r') as rfh: data = rfh.read() else: raise SaltCloudSystemExit( @@ -4217,7 +4218,7 @@ def vn_hold(call=None, kwargs=None): '\'data\' will take precedence.' ) elif path: - with salt.utils.fopen(path, mode='r') as rfh: + with salt.utils.files.fopen(path, mode='r') as rfh: data = rfh.read() else: raise SaltCloudSystemExit( @@ -4362,7 +4363,7 @@ def vn_release(call=None, kwargs=None): '\'data\' will take precedence.' ) elif path: - with salt.utils.fopen(path, mode='r') as rfh: + with salt.utils.files.fopen(path, mode='r') as rfh: data = rfh.read() else: raise SaltCloudSystemExit( @@ -4449,7 +4450,7 @@ def vn_reserve(call=None, kwargs=None): '\'data\' will take precedence.' ) elif path: - with salt.utils.fopen(path, mode='r') as rfh: + with salt.utils.files.fopen(path, mode='r') as rfh: data = rfh.read() else: raise SaltCloudSystemExit( diff --git a/salt/cloud/clouds/openstack.py b/salt/cloud/clouds/openstack.py index 96c1902cfbf..b26a36a47c4 100644 --- a/salt/cloud/clouds/openstack.py +++ b/salt/cloud/clouds/openstack.py @@ -135,6 +135,14 @@ Alternatively, one could use the private IP to connect by specifying: ssh_interface: private_ips +.. note:: + + When using floating ips from networks, if the OpenStack driver is unable to + allocate a new ip address for the server, it will check that for + unassociated ip addresses in the floating ip pool. If SaltCloud is running + in parallel mode, it is possible that more than one server will attempt to + use the same ip address. + ''' # Import python libs @@ -143,7 +151,9 @@ import os import logging import socket import pprint -from salt.utils.versions import LooseVersion as _LooseVersion + +# This import needs to be here so the version check can be done below +import salt.utils.versions # Import libcloud try: @@ -162,7 +172,8 @@ try: # However, older versions of libcloud must still be supported with this work-around. # This work-around can be removed when the required minimum version of libcloud is # 2.0.0 (See PR #40837 - which is implemented in Salt Oxygen). - if _LooseVersion(libcloud.__version__) < _LooseVersion('1.4.0'): + if salt.utils.versions.LooseVersion(libcloud.__version__) < \ + salt.utils.versions.LooseVersion('1.4.0'): # See https://github.com/saltstack/salt/issues/32743 import libcloud.security libcloud.security.CA_CERTS_PATH.append('/etc/ssl/certs/YaST-CA.pem') @@ -174,13 +185,11 @@ except Exception: from salt.cloud.libcloudfuncs import * # pylint: disable=W0614,W0401 # Import salt libs -import salt.utils - -# Import salt.cloud libs +import salt.utils # Can be removed once namespaced_function has been moved import salt.utils.cloud -import salt.utils.pycrypto as sup +import salt.utils.files +import salt.utils.pycrypto import salt.config as config -from salt.utils import namespaced_function from salt.exceptions import ( SaltCloudConfigError, SaltCloudNotFound, @@ -205,19 +214,19 @@ __virtualname__ = 'openstack' # Some of the libcloud functions need to be in the same namespace as the # functions defined in the module, so we create new function objects inside # this module namespace -get_size = namespaced_function(get_size, globals()) -get_image = namespaced_function(get_image, globals()) -avail_locations = namespaced_function(avail_locations, globals()) -avail_images = namespaced_function(avail_images, globals()) -avail_sizes = namespaced_function(avail_sizes, globals()) -script = namespaced_function(script, globals()) -destroy = namespaced_function(destroy, globals()) -reboot = namespaced_function(reboot, globals()) -list_nodes = namespaced_function(list_nodes, globals()) -list_nodes_full = namespaced_function(list_nodes_full, globals()) -list_nodes_select = namespaced_function(list_nodes_select, globals()) -show_instance = namespaced_function(show_instance, globals()) -get_node = namespaced_function(get_node, globals()) +get_size = salt.utils.namespaced_function(get_size, globals()) +get_image = salt.utils.namespaced_function(get_image, globals()) +avail_locations = salt.utils.namespaced_function(avail_locations, globals()) +avail_images = salt.utils.namespaced_function(avail_images, globals()) +avail_sizes = salt.utils.namespaced_function(avail_sizes, globals()) +script = salt.utils.namespaced_function(script, globals()) +destroy = salt.utils.namespaced_function(destroy, globals()) +reboot = salt.utils.namespaced_function(reboot, globals()) +list_nodes = salt.utils.namespaced_function(list_nodes, globals()) +list_nodes_full = salt.utils.namespaced_function(list_nodes_full, globals()) +list_nodes_select = salt.utils.namespaced_function(list_nodes_select, globals()) +show_instance = salt.utils.namespaced_function(show_instance, globals()) +get_node = salt.utils.namespaced_function(get_node, globals()) # Only load in this module is the OPENSTACK configurations are in place @@ -231,7 +240,7 @@ def __virtual__(): if get_dependencies() is False: return False - salt.utils.warn_until( + salt.utils.versions.warn_until( 'Oxygen', 'This driver has been deprecated and will be removed in the ' '{version} release of Salt. Please use the nova driver instead.' @@ -529,7 +538,7 @@ def request_instance(vm_=None, call=None): if files: kwargs['ex_files'] = {} for src_path in files: - with salt.utils.fopen(files[src_path], 'r') as fp_: + with salt.utils.files.fopen(files[src_path], 'r') as fp_: kwargs['ex_files'][src_path] = fp_.read() userdata_file = config.get_cloud_config_value( @@ -537,7 +546,7 @@ def request_instance(vm_=None, call=None): ) if userdata_file is not None: try: - with salt.utils.fopen(userdata_file, 'r') as fp_: + with salt.utils.files.fopen(userdata_file, 'r') as fp_: kwargs['ex_userdata'] = salt.utils.cloud.userdata_template( __opts__, vm_, fp_.read() ) @@ -761,7 +770,7 @@ def create(vm_): ) data = conn.ex_get_node_details(vm_['instance_id']) if vm_['key_filename'] is None and 'change_password' in __opts__ and __opts__['change_password'] is True: - vm_['password'] = sup.secure_password() + vm_['password'] = salt.utils.pycrypto.secure_password() conn.ex_set_password(data, vm_['password']) networks(vm_) else: @@ -855,40 +864,43 @@ def _assign_floating_ips(vm_, conn, kwargs): pool = OpenStack_1_1_FloatingIpPool( net['floating'], conn.connection ) - for idx in pool.list_floating_ips(): - if idx.node_id is None: - floating.append(idx) + try: + floating.append(pool.create_floating_ip()) + except Exception as e: + log.debug('Cannot allocate IP from floating pool \'%s\'. Checking for unassociated ips.', + net['floating']) + for idx in pool.list_floating_ips(): + if idx.node_id is None: + floating.append(idx) + break if not floating: - try: - floating.append(pool.create_floating_ip()) - except Exception as e: - raise SaltCloudSystemExit( - 'Floating pool \'{0}\' does not have any more ' - 'please create some more or use a different ' - 'pool.'.format(net['floating']) - ) + raise SaltCloudSystemExit( + 'There are no more floating IP addresses ' + 'available, please create some more' + ) # otherwise, attempt to obtain list without specifying pool # this is the same as 'nova floating-ip-list' elif ssh_interface(vm_) != 'private_ips': try: # This try/except is here because it appears some - # *cough* Rackspace *cough* # OpenStack providers return a 404 Not Found for the # floating ip pool URL if there are no pools setup pool = OpenStack_1_1_FloatingIpPool( '', conn.connection ) - for idx in pool.list_floating_ips(): - if idx.node_id is None: - floating.append(idx) + try: + floating.append(pool.create_floating_ip()) + except Exception as e: + log.debug('Cannot allocate IP from the default floating pool. Checking for unassociated ips.') + for idx in pool.list_floating_ips(): + if idx.node_id is None: + floating.append(idx) + break if not floating: - try: - floating.append(pool.create_floating_ip()) - except Exception as e: - raise SaltCloudSystemExit( - 'There are no more floating IP addresses ' - 'available, please create some more' - ) + log.warning( + 'There are no more floating IP addresses ' + 'available, please create some more if necessary' + ) except Exception as e: if str(e).startswith('404'): pass diff --git a/salt/cloud/clouds/parallels.py b/salt/cloud/clouds/parallels.py index a789351d35f..a8db20eaaea 100644 --- a/salt/cloud/clouds/parallels.py +++ b/salt/cloud/clouds/parallels.py @@ -51,6 +51,9 @@ from salt.exceptions import ( SaltCloudExecutionTimeout ) +# Import 3rd-party libs +from salt.ext import six + # Get logging started log = logging.getLogger(__name__) @@ -397,7 +400,7 @@ def query(action=None, command=None, args=None, method='GET', data=None): args = {} kwargs = {'data': data} - if isinstance(data, str) and '= 3.0.0 +:depends: profitbrick >= 3.1.0 The module requires ProfitBricks credentials to be supplied along with an existing virtual datacenter UUID where the server resources will @@ -98,7 +98,8 @@ import pprint import time # Import salt libs -import salt.utils +import salt.utils.cloud +import salt.utils.files import salt.config as config from salt.exceptions import ( SaltCloudConfigError, @@ -108,16 +109,14 @@ from salt.exceptions import ( SaltCloudSystemExit ) -# Import salt.cloud libs -import salt.utils.cloud - # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six try: from profitbricks.client import ( ProfitBricksService, Server, NIC, Volume, FirewallRule, - Datacenter, LoadBalancer, LAN + Datacenter, LoadBalancer, LAN, + PBNotFoundError ) HAS_PROFITBRICKS = True except ImportError: @@ -482,7 +481,13 @@ def list_nodes(conn=None, call=None): ret = {} datacenter_id = get_datacenter_id() - nodes = conn.list_servers(datacenter_id=datacenter_id) + + try: + nodes = conn.list_servers(datacenter_id=datacenter_id) + except PBNotFoundError: + log.error('Failed to get nodes list from datacenter: {0}'.format( + datacenter_id)) + raise for item in nodes['items']: node = {'id': item['id']} @@ -642,7 +647,7 @@ def get_public_keys(vm_): ) ) ssh_keys = [] - with salt.utils.fopen(key_filename) as rfh: + with salt.utils.files.fopen(key_filename) as rfh: for key in rfh.readlines(): ssh_keys.append(key) diff --git a/salt/cloud/clouds/proxmox.py b/salt/cloud/clouds/proxmox.py index 014add1f878..f208e88d1ad 100644 --- a/salt/cloud/clouds/proxmox.py +++ b/salt/cloud/clouds/proxmox.py @@ -35,7 +35,7 @@ import re import json # Import salt libs -import salt.ext.six as six +from salt.ext import six import salt.utils # Import salt cloud libs diff --git a/salt/cloud/clouds/saltify.py b/salt/cloud/clouds/saltify.py index 1d8a1e97dc3..3c0980dfea3 100644 --- a/salt/cloud/clouds/saltify.py +++ b/salt/cloud/clouds/saltify.py @@ -17,8 +17,16 @@ from __future__ import absolute_import import logging # Import salt libs +import salt.utils import salt.config as config -from salt.exceptions import SaltCloudException +import salt.netapi +import salt.ext.six as six +if six.PY3: + import ipaddress +else: + import salt.ext.ipaddress as ipaddress + +from salt.exceptions import SaltCloudException, SaltCloudSystemExit # Get logging started log = logging.getLogger(__name__) @@ -47,28 +55,188 @@ def __virtual__(): return True -def list_nodes(): +def _get_connection_info(): ''' - Because this module is not specific to any cloud providers, there will be - no nodes to list. + Return connection information for the passed VM data + ''' + vm_ = get_configured_provider() + + try: + ret = {'username': vm_['username'], + 'password': vm_['password'], + 'eauth': vm_['eauth'], + 'vm': vm_, + } + except KeyError: + raise SaltCloudException( + 'Configuration must define salt-api "username", "password" and "eauth"') + return ret + + +def avail_locations(call=None): + ''' + This function returns a list of locations available. + + .. code-block:: bash + + salt-cloud --list-locations my-cloud-provider + + [ saltify will always returns an empty dictionary ] + ''' + + return {} + + +def avail_images(call=None): + ''' + This function returns a list of images available for this cloud provider. + + .. code-block:: bash + + salt-cloud --list-images saltify + + returns a list of available profiles. + + ..versionadded:: Oxygen + + ''' + vm_ = get_configured_provider() + return {'Profiles': [profile for profile in vm_['profiles']]} + + +def avail_sizes(call=None): + ''' + This function returns a list of sizes available for this cloud provider. + + .. code-block:: bash + + salt-cloud --list-sizes saltify + + [ saltify always returns an empty dictionary ] ''' return {} -def list_nodes_full(): +def list_nodes(call=None): ''' - Because this module is not specific to any cloud providers, there will be - no nodes to list. + List the nodes which have salt-cloud:driver:saltify grains. + + .. code-block:: bash + + salt-cloud -Q + + returns a list of dictionaries of defined standard fields. + + salt-api setup required for operation. + + ..versionadded:: Oxygen + ''' - return {} + nodes = _list_nodes_full(call) + return _build_required_items(nodes) -def list_nodes_select(): +def _build_required_items(nodes): + ret = {} + for name, grains in nodes.items(): + if grains: + private_ips = [] + public_ips = [] + ips = grains['ipv4'] + grains['ipv6'] + for adrs in ips: + ip_ = ipaddress.ip_address(adrs) + if not ip_.is_loopback: + if ip_.is_private: + private_ips.append(adrs) + else: + public_ips.append(adrs) + + ret[name] = { + 'id': grains['id'], + 'image': grains['salt-cloud']['profile'], + 'private_ips': private_ips, + 'public_ips': public_ips, + 'size': '', + 'state': 'running' + } + + return ret + + +def list_nodes_full(call=None): ''' - Because this module is not specific to any cloud providers, there will be - no nodes to list. + Lists complete information for all nodes. + + .. code-block:: bash + + salt-cloud -F + + returns a list of dictionaries. + for 'saltify' minions, returns dict of grains (enhanced). + salt-api setup required for operation. + + ..versionadded:: Oxygen ''' - return {} + + ret = _list_nodes_full(call) + + for key, grains in ret.items(): # clean up some hyperverbose grains -- everything is too much + try: + del grains['cpu_flags'], grains['disks'], grains['pythonpath'], grains['dns'], grains['gpus'] + except KeyError: + pass # ignore absence of things we are eliminating + except TypeError: + del ret[key] # eliminate all reference to unexpected (None) values. + + reqs = _build_required_items(ret) + + for name in ret: + ret[name].update(reqs[name]) + + return ret + + +def _list_nodes_full(call=None): + ''' + List the nodes, ask all 'saltify' minions, return dict of grains. + ''' + local = salt.netapi.NetapiClient(__opts__) + cmd = {'client': 'local', + 'tgt': 'salt-cloud:driver:saltify', + 'fun': 'grains.items', + 'arg': '', + 'tgt_type': 'grain', + } + cmd.update(_get_connection_info()) + + return local.run(cmd) + + +def list_nodes_select(call=None): + ''' + Return a list of the minions that have salt-cloud grains, with + select fields. + ''' + return salt.utils.cloud.list_nodes_select( + list_nodes_full('function'), __opts__['query.selection'], call, + ) + + +def show_instance(name, call=None): + ''' + List the a single node, return dict of grains. + ''' + local = salt.netapi.NetapiClient(__opts__) + cmd = {'client': 'local', + 'tgt': 'name', + 'fun': 'grains.items', + 'arg': '', + 'tgt_type': 'glob', + } + cmd.update(_get_connection_info()) + ret = local.run(cmd) + ret.update(_build_required_items(ret)) + return ret def create(vm_): @@ -190,3 +358,130 @@ def _verify(vm_): except SaltCloudException as exc: log.error('Exception: %s', exc) return False + + +def destroy(name, call=None): + ''' Destroy a node. + + .. versionadded:: Oxygen + + CLI Example: + + .. code-block:: bash + + salt-cloud --destroy mymachine + + salt-api setup required for operation. + + ''' + if call == 'function': + raise SaltCloudSystemExit( + 'The destroy action must be called with -d, --destroy, ' + '-a, or --action.' + ) + + opts = __opts__ + + __utils__['cloud.fire_event']( + 'event', + 'destroying instance', + 'salt/cloud/{0}/destroying'.format(name), + args={'name': name}, + sock_dir=opts['sock_dir'], + transport=opts['transport'] + ) + + local = salt.netapi.NetapiClient(opts) + cmd = {'client': 'local', + 'tgt': name, + 'fun': 'grains.get', + 'arg': ['salt-cloud'], + } + cmd.update(_get_connection_info()) + vm_ = cmd['vm'] + my_info = local.run(cmd) + try: + vm_.update(my_info[name]) # get profile name to get config value + except (IndexError, TypeError): + pass + if config.get_cloud_config_value( + 'remove_config_on_destroy', vm_, opts, default=True + ): + cmd.update({'fun': 'service.disable', 'arg': ['salt-minion']}) + ret = local.run(cmd) # prevent generating new keys on restart + if ret and ret[name]: + log.info('disabled salt-minion service on %s', name) + cmd.update({'fun': 'config.get', 'arg': ['conf_file']}) + ret = local.run(cmd) + if ret and ret[name]: + confile = ret[name] + cmd.update({'fun': 'file.remove', 'arg': [confile]}) + ret = local.run(cmd) + if ret and ret[name]: + log.info('removed minion %s configuration file %s', + name, confile) + cmd.update({'fun': 'config.get', 'arg': ['pki_dir']}) + ret = local.run(cmd) + if ret and ret[name]: + pki_dir = ret[name] + cmd.update({'fun': 'file.remove', 'arg': [pki_dir]}) + ret = local.run(cmd) + if ret and ret[name]: + log.info( + 'removed minion %s key files in %s', + name, + pki_dir) + + if config.get_cloud_config_value( + 'shutdown_on_destroy', vm_, opts, default=False + ): + cmd.update({'fun': 'system.shutdown', 'arg': ''}) + ret = local.run(cmd) + if ret and ret[name]: + log.info('system.shutdown for minion %s successful', name) + + __utils__['cloud.fire_event']( + 'event', + 'destroyed instance', + 'salt/cloud/{0}/destroyed'.format(name), + args={'name': name}, + sock_dir=opts['sock_dir'], + transport=opts['transport'] + ) + + return {'Destroyed': '{0} was destroyed.'.format(name)} + + +def reboot(name, call=None): + ''' + Reboot a saltify minion. + + salt-api setup required for operation. + + ..versionadded:: Oxygen + + name + The name of the VM to reboot. + + CLI Example: + + .. code-block:: bash + + salt-cloud -a reboot vm_name + ''' + + if call != 'action': + raise SaltCloudException( + 'The reboot action must be called with -a or --action.' + ) + + local = salt.netapi.NetapiClient(__opts__) + cmd = {'client': 'local', + 'tgt': name, + 'fun': 'system.reboot', + 'arg': '', + } + cmd.update(_get_connection_info()) + ret = local.run(cmd) + + return ret diff --git a/salt/cloud/clouds/softlayer.py b/salt/cloud/clouds/softlayer.py index 457447b4318..d24bcab660b 100644 --- a/salt/cloud/clouds/softlayer.py +++ b/salt/cloud/clouds/softlayer.py @@ -35,6 +35,9 @@ import salt.utils.cloud import salt.config as config from salt.exceptions import SaltCloudSystemExit +# Import 3rd-party libs +from salt.ext import six + # Attempt to import softlayer lib try: import SoftLayer @@ -297,7 +300,7 @@ def create(vm_): if isinstance(disks, int): disks = [str(disks)] - elif isinstance(disks, str): + elif isinstance(disks, six.string_types): disks = [size.strip() for size in disks.split(',')] count = 0 @@ -508,7 +511,7 @@ def list_nodes_full(mask='mask[id]', call=None): conn = get_conn(service='SoftLayer_Account') response = conn.getVirtualGuests() for node_id in response: - hostname = node_id['hostname'].split('.')[0] + hostname = node_id['hostname'] ret[hostname] = node_id __utils__['cloud.cache_node_list'](ret, __active_provider_name__.split(':')[0], __opts__) return ret @@ -594,9 +597,6 @@ def destroy(name, call=None): transport=__opts__['transport'] ) - # If the VM was created with use_fqdn, the short hostname will be used instead. - name = name.split('.')[0] - node = show_instance(name, call='action') conn = get_conn() response = conn.deleteObject(id=node['id']) diff --git a/salt/cloud/clouds/softlayer_hw.py b/salt/cloud/clouds/softlayer_hw.py index 34dae95ccad..030391fe6d3 100644 --- a/salt/cloud/clouds/softlayer_hw.py +++ b/salt/cloud/clouds/softlayer_hw.py @@ -526,9 +526,6 @@ def destroy(name, call=None): transport=__opts__['transport'] ) - # If the VM was created with use_fqdn, the short hostname will be used instead. - name = name.split('.')[0] - node = show_instance(name, call='action') conn = get_conn(service='SoftLayer_Ticket') response = conn.createCancelServerTicket( diff --git a/salt/cloud/clouds/virtualbox.py b/salt/cloud/clouds/virtualbox.py index 903722fd39b..4266ce5f0e0 100644 --- a/salt/cloud/clouds/virtualbox.py +++ b/salt/cloud/clouds/virtualbox.py @@ -24,7 +24,6 @@ import logging # Import salt libs from salt.exceptions import SaltCloudSystemExit import salt.config as config -import salt.utils.cloud as cloud # Import Third Party Libs try: @@ -136,7 +135,7 @@ def create(vm_info): ) log.debug("Going to fire event: starting create") - cloud.fire_event( + __utils__['cloud.fire_event']( 'event', 'starting create', 'salt/cloud/{0}/creating'.format(vm_info['name']), @@ -151,7 +150,7 @@ def create(vm_info): 'clone_from': vm_info['clonefrom'] } - cloud.fire_event( + __utils__['cloud.fire_event']( 'event', 'requesting instance', 'salt/cloud/{0}/requesting'.format(vm_info['name']), @@ -174,10 +173,10 @@ def create(vm_info): vm_info['key_filename'] = key_filename vm_info['ssh_host'] = ip - res = cloud.bootstrap(vm_info, __opts__) + res = __utils__['cloud.bootstrap'](vm_info) vm_result.update(res) - cloud.fire_event( + __utils__['cloud.fire_event']( 'event', 'created machine', 'salt/cloud/{0}/created'.format(vm_info['name']), @@ -269,7 +268,7 @@ def list_nodes(kwargs=None, call=None): "private_ips", "public_ips", ] - return cloud.list_nodes_select( + return __utils__['cloud.list_nodes_select']( list_nodes_full('function'), attributes, call, ) @@ -278,7 +277,7 @@ def list_nodes_select(call=None): """ Return a list of the VMs that are on the provider, with select fields """ - return cloud.list_nodes_select( + return __utils__['cloud.list_nodes_select']( list_nodes_full('function'), __opts__['query.selection'], call, ) @@ -306,7 +305,7 @@ def destroy(name, call=None): if not vb_machine_exists(name): return "{0} doesn't exist and can't be deleted".format(name) - cloud.fire_event( + __utils__['cloud.fire_event']( 'event', 'destroying instance', 'salt/cloud/{0}/destroying'.format(name), @@ -317,7 +316,7 @@ def destroy(name, call=None): vb_destroy_machine(name) - cloud.fire_event( + __utils__['cloud.fire_event']( 'event', 'destroyed instance', 'salt/cloud/{0}/destroyed'.format(name), diff --git a/salt/cloud/clouds/vmware.py b/salt/cloud/clouds/vmware.py index 5b49a74af45..98ab762230c 100644 --- a/salt/cloud/clouds/vmware.py +++ b/salt/cloud/clouds/vmware.py @@ -127,6 +127,7 @@ import subprocess import salt.utils import salt.utils.cloud import salt.utils.network +import salt.utils.stringutils import salt.utils.xmlutil import salt.utils.vmware from salt.exceptions import SaltCloudSystemExit @@ -135,7 +136,7 @@ from salt.exceptions import SaltCloudSystemExit import salt.config as config # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six try: # Attempt to import pyVmomi libs from pyVmomi import vim @@ -3800,7 +3801,7 @@ def add_host(kwargs=None, call=None): p1 = subprocess.Popen(('echo', '-n'), stdout=subprocess.PIPE, stderr=subprocess.PIPE) p2 = subprocess.Popen(('openssl', 's_client', '-connect', '{0}:443'.format(host_name)), stdin=p1.stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE) p3 = subprocess.Popen(('openssl', 'x509', '-noout', '-fingerprint', '-sha1'), stdin=p2.stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out = salt.utils.to_str(p3.stdout.read()) + out = salt.utils.stringutils.to_str(p3.stdout.read()) ssl_thumbprint = out.split('=')[-1].strip() log.debug('SSL thumbprint received from the host system: {0}'.format(ssl_thumbprint)) spec.sslThumbprint = ssl_thumbprint diff --git a/salt/cloud/clouds/vultrpy.py b/salt/cloud/clouds/vultrpy.py index e1ca1c5af98..09a8b2edfa2 100644 --- a/salt/cloud/clouds/vultrpy.py +++ b/salt/cloud/clouds/vultrpy.py @@ -41,7 +41,7 @@ import time # Import salt libs import salt.config as config -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves.urllib.parse import urlencode as _urlencode # pylint: disable=E0611 from salt.exceptions import ( SaltCloudConfigError, diff --git a/salt/cloud/clouds/xen.py b/salt/cloud/clouds/xen.py index 168397f81e8..d1caaab282b 100644 --- a/salt/cloud/clouds/xen.py +++ b/salt/cloud/clouds/xen.py @@ -150,7 +150,14 @@ def _get_session(): __opts__, search_global=False ) - session = XenAPI.Session(url) + ignore_ssl = config.get_cloud_config_value( + 'ignore_ssl', + get_configured_provider(), + __opts__, + default=False, + search_global=False + ) + session = XenAPI.Session(url, ignore_ssl=ignore_ssl) log.debug('url: {} user: {} password: {}, originator: {}'.format( url, user, diff --git a/salt/cloud/deploy/bootstrap-salt.sh b/salt/cloud/deploy/bootstrap-salt.sh index ededd585088..ddda8e63989 100755 --- a/salt/cloud/deploy/bootstrap-salt.sh +++ b/salt/cloud/deploy/bootstrap-salt.sh @@ -18,7 +18,7 @@ #====================================================================================================================== set -o nounset # Treat unset variables as an error -__ScriptVersion="2017.05.24" +__ScriptVersion="2017.08.17" __ScriptName="bootstrap-salt.sh" __ScriptFullName="$0" @@ -121,8 +121,7 @@ __check_command_exists() { #--- FUNCTION ------------------------------------------------------------------------------------------------------- # NAME: __check_pip_allowed -# DESCRIPTION: Simple function to let the users know that -P needs to be -# used. +# DESCRIPTION: Simple function to let the users know that -P needs to be used. #---------------------------------------------------------------------------------------------------------------------- __check_pip_allowed() { if [ $# -eq 1 ]; then @@ -155,10 +154,16 @@ __check_config_dir() { __fetch_url "/tmp/${CC_DIR_BASE}" "${CC_DIR_NAME}" CC_DIR_NAME="/tmp/${CC_DIR_BASE}" ;; + *://*) + echoerror "Unsupported URI scheme for $CC_DIR_NAME" + echo "null" + return + ;; *) if [ ! -e "${CC_DIR_NAME}" ]; then + echoerror "The configuration directory or archive $CC_DIR_NAME does not exist." echo "null" - return 0 + return fi ;; esac @@ -187,6 +192,28 @@ __check_config_dir() { echo "${CC_DIR_NAME}" } +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: __check_unparsed_options +# DESCRIPTION: Checks the placed after the install arguments +#---------------------------------------------------------------------------------------------------------------------- +__check_unparsed_options() { + shellopts="$1" + # grep alternative for SunOS + if [ -f /usr/xpg4/bin/grep ]; then + grep='/usr/xpg4/bin/grep' + else + grep='grep' + fi + unparsed_options=$( echo "$shellopts" | ${grep} -E '(^|[[:space:]])[-]+[[:alnum:]]' ) + if [ "$unparsed_options" != "" ]; then + __usage + echo + echoerror "options are only allowed before install arguments" + echo + exit 1 + fi +} + #---------------------------------------------------------------------------------------------------------------------- # Handle command line arguments @@ -243,6 +270,10 @@ _REPO_URL="repo.saltstack.com" _PY_EXE="" _INSTALL_PY="$BS_FALSE" +# Defaults for install arguments +ITYPE="stable" + + #--- FUNCTION ------------------------------------------------------------------------------------------------------- # NAME: __usage # DESCRIPTION: Display usage information. @@ -380,19 +411,7 @@ do v ) echo "$0 -- Version $__ScriptVersion"; exit 0 ;; n ) _COLORS=0; __detect_color_support ;; D ) _ECHO_DEBUG=$BS_TRUE ;; - - c ) _TEMP_CONFIG_DIR=$(__check_config_dir "$OPTARG") - # If the configuration directory does not exist, error out - if [ "$_TEMP_CONFIG_DIR" = "null" ]; then - echoerror "Unsupported URI scheme for $OPTARG" - exit 1 - fi - if [ ! -d "$_TEMP_CONFIG_DIR" ]; then - echoerror "The configuration directory ${_TEMP_CONFIG_DIR} does not exist." - exit 1 - fi - ;; - + c ) _TEMP_CONFIG_DIR="$OPTARG" ;; g ) _SALT_REPO_URL=$OPTARG ;; G ) echowarn "The '-G' option is DEPRECATED and will be removed in the future stable release!" @@ -401,14 +420,7 @@ do ;; w ) _DOWNSTREAM_PKG_REPO=$BS_TRUE ;; - - k ) _TEMP_KEYS_DIR="$OPTARG" - # If the configuration directory does not exist, error out - if [ ! -d "$_TEMP_KEYS_DIR" ]; then - echoerror "The pre-seed keys directory ${_TEMP_KEYS_DIR} does not exist." - exit 1 - fi - ;; + k ) _TEMP_KEYS_DIR="$OPTARG" ;; s ) _SLEEP=$OPTARG ;; M ) _INSTALL_MASTER=$BS_TRUE ;; S ) _INSTALL_SYNDIC=$BS_TRUE ;; @@ -451,205 +463,28 @@ done shift $((OPTIND-1)) -__check_unparsed_options() { - shellopts="$1" - # grep alternative for SunOS - if [ -f /usr/xpg4/bin/grep ]; then - grep='/usr/xpg4/bin/grep' - else - grep='grep' - fi - unparsed_options=$( echo "$shellopts" | ${grep} -E '(^|[[:space:]])[-]+[[:alnum:]]' ) - if [ "$unparsed_options" != "" ]; then - __usage - echo - echoerror "options are only allowed before install arguments" - echo - exit 1 - fi -} +# Define our logging file and pipe paths +LOGFILE="/tmp/$( echo "$__ScriptName" | sed s/.sh/.log/g )" +LOGPIPE="/tmp/$( echo "$__ScriptName" | sed s/.sh/.logpipe/g )" - -# Check that we're actually installing one of minion/master/syndic -if [ "$_INSTALL_MINION" -eq $BS_FALSE ] && [ "$_INSTALL_MASTER" -eq $BS_FALSE ] && [ "$_INSTALL_SYNDIC" -eq $BS_FALSE ] && [ "$_CONFIG_ONLY" -eq $BS_FALSE ]; then - echowarn "Nothing to install or configure" - exit 0 -fi - -# Check that we're installing a minion if we're being passed a master address -if [ "$_INSTALL_MINION" -eq $BS_FALSE ] && [ "$_SALT_MASTER_ADDRESS" != "null" ]; then - echoerror "Don't pass a master address (-A) if no minion is going to be bootstrapped." +# Create our logging pipe +# On FreeBSD we have to use mkfifo instead of mknod +mknod "$LOGPIPE" p >/dev/null 2>&1 || mkfifo "$LOGPIPE" >/dev/null 2>&1 +if [ $? -ne 0 ]; then + echoerror "Failed to create the named pipe required to log" exit 1 fi -# Check that we're installing a minion if we're being passed a minion id -if [ "$_INSTALL_MINION" -eq $BS_FALSE ] && [ "$_SALT_MINION_ID" != "null" ]; then - echoerror "Don't pass a minion id (-i) if no minion is going to be bootstrapped." - exit 1 -fi +# What ever is written to the logpipe gets written to the logfile +tee < "$LOGPIPE" "$LOGFILE" & -# Check that we're installing or configuring a master if we're being passed a master config json dict -if [ "$_CUSTOM_MASTER_CONFIG" != "null" ]; then - if [ "$_INSTALL_MASTER" -eq $BS_FALSE ] && [ "$_CONFIG_ONLY" -eq $BS_FALSE ]; then - echoerror "Don't pass a master config JSON dict (-J) if no master is going to be bootstrapped or configured." - exit 1 - fi -fi +# Close STDOUT, reopen it directing it to the logpipe +exec 1>&- +exec 1>"$LOGPIPE" +# Close STDERR, reopen it directing it to the logpipe +exec 2>&- +exec 2>"$LOGPIPE" -# Check that we're installing or configuring a minion if we're being passed a minion config json dict -if [ "$_CUSTOM_MINION_CONFIG" != "null" ]; then - if [ "$_INSTALL_MINION" -eq $BS_FALSE ] && [ "$_CONFIG_ONLY" -eq $BS_FALSE ]; then - echoerror "Don't pass a minion config JSON dict (-j) if no minion is going to be bootstrapped or configured." - exit 1 - fi -fi - -# Define installation type -if [ "$#" -eq 0 ];then - ITYPE="stable" -else - __check_unparsed_options "$*" - ITYPE=$1 - shift -fi - -# Check installation type -if [ "$(echo "$ITYPE" | egrep '(stable|testing|daily|git)')" = "" ]; then - echoerror "Installation type \"$ITYPE\" is not known..." - exit 1 -fi - -# If doing a git install, check what branch/tag/sha will be checked out -if [ "$ITYPE" = "git" ]; then - if [ "$#" -eq 0 ];then - GIT_REV="develop" - else - __check_unparsed_options "$*" - GIT_REV="$1" - shift - fi - - # Disable shell warning about unbound variable during git install - STABLE_REV="latest" - -# If doing stable install, check if version specified -elif [ "$ITYPE" = "stable" ]; then - if [ "$#" -eq 0 ];then - STABLE_REV="latest" - else - __check_unparsed_options "$*" - - if [ "$(echo "$1" | egrep '^(latest|1\.6|1\.7|2014\.1|2014\.7|2015\.5|2015\.8|2016\.3|2016\.11)$')" != "" ]; then - STABLE_REV="$1" - shift - elif [ "$(echo "$1" | egrep '^([0-9]*\.[0-9]*\.[0-9]*)$')" != "" ]; then - STABLE_REV="archive/$1" - shift - else - echo "Unknown stable version: $1 (valid: 1.6, 1.7, 2014.1, 2014.7, 2015.5, 2015.8, 2016.3, 2016.11, latest, \$MAJOR.\$MINOR.\$PATCH)" - exit 1 - fi - fi -fi - -# -a and -V only work from git -if [ "$ITYPE" != "git" ]; then - if [ $_PIP_ALL -eq $BS_TRUE ]; then - echoerror "Pip installing all python packages with -a is only possible when installing Salt via git" - exit 1 - fi - if [ "$_VIRTUALENV_DIR" != "null" ]; then - echoerror "Virtualenv installs via -V is only possible when installing Salt via git" - exit 1 - fi -fi - -# Set the _REPO_URL value based on if -R was passed or not. Defaults to repo.saltstack.com. -if [ "$_CUSTOM_REPO_URL" != "null" ]; then - _REPO_URL="$_CUSTOM_REPO_URL" - - # Check for -r since -R is being passed. Set -r with a warning. - if [ "$_DISABLE_REPOS" -eq $BS_FALSE ]; then - echowarn "Detected -R option. No other repositories will be configured when -R is used. Setting -r option to True." - _DISABLE_REPOS=$BS_TRUE - fi -fi - -# Check for any unparsed arguments. Should be an error. -if [ "$#" -gt 0 ]; then - __check_unparsed_options "$*" - __usage - echo - echoerror "Too many arguments." - exit 1 -fi - -# Check the _DISABLE_SSL value and set HTTP or HTTPS. -if [ "$_DISABLE_SSL" -eq $BS_TRUE ]; then - HTTP_VAL="http" -else - HTTP_VAL="https" -fi - -# Check the _QUIET_GIT_INSTALLATION value and set SETUP_PY_INSTALL_ARGS. -if [ "$_QUIET_GIT_INSTALLATION" -eq $BS_TRUE ]; then - SETUP_PY_INSTALL_ARGS="-q" -else - SETUP_PY_INSTALL_ARGS="" -fi - -# whoami alternative for SunOS -if [ -f /usr/xpg4/bin/id ]; then - whoami='/usr/xpg4/bin/id -un' -else - whoami='whoami' -fi - -# Root permissions are required to run this script -if [ "$($whoami)" != "root" ]; then - echoerror "Salt requires root privileges to install. Please re-run this script as root." - exit 1 -fi - -# Export the http_proxy configuration to our current environment -if [ "${_HTTP_PROXY}" != "" ]; then - export http_proxy="$_HTTP_PROXY" - export https_proxy="$_HTTP_PROXY" -fi - -# Let's discover how we're being called -# shellcheck disable=SC2009 -CALLER=$(ps -a -o pid,args | grep $$ | grep -v grep | tr -s ' ' | cut -d ' ' -f 3) - -if [ "${CALLER}x" = "${0}x" ]; then - CALLER="shell pipe" -fi - -# Work around for 'Docker + salt-bootstrap failure' https://github.com/saltstack/salt-bootstrap/issues/394 -if [ "${_DISABLE_SALT_CHECKS}" -eq $BS_FALSE ] && [ -f /tmp/disable_salt_checks ]; then - # shellcheck disable=SC2016 - echowarn 'Found file: /tmp/disable_salt_checks, setting _DISABLE_SALT_CHECKS=$BS_TRUE' - _DISABLE_SALT_CHECKS=$BS_TRUE -fi - -# Because -a can only be installed into virtualenv -if [ "${_PIP_ALL}" -eq $BS_TRUE ] && [ "${_VIRTUALENV_DIR}" = "null" ]; then - usage - # Could possibly set up a default virtualenv location when -a flag is passed - echoerror "Using -a requires -V because pip pkgs should be siloed from python system pkgs" - exit 1 -fi - -# Make sure virtualenv directory does not already exist -if [ -d "${_VIRTUALENV_DIR}" ]; then - echoerror "The directory ${_VIRTUALENV_DIR} for virtualenv already exists" - exit 1 -fi - -echoinfo "Running version: ${__ScriptVersion}" -echoinfo "Executed by: ${CALLER}" -echoinfo "Command line: '${__ScriptFullName} ${__ScriptArgs}'" -#echowarn "Running the unstable version of ${__ScriptName}" #--- FUNCTION ------------------------------------------------------------------------------------------------------- # NAME: __exit_cleanup @@ -684,8 +519,10 @@ __exit_cleanup() { fi # Remove the logging pipe when the script exits - echodebug "Removing the logging pipe $LOGPIPE" - rm -f "$LOGPIPE" + if [ -p "$LOGPIPE" ]; then + echodebug "Removing the logging pipe $LOGPIPE" + rm -f "$LOGPIPE" + fi # Kill tee when exiting, CentOS, at least requires this # shellcheck disable=SC2009 @@ -713,28 +550,192 @@ __exit_cleanup() { trap "__exit_cleanup" EXIT INT -# Define our logging file and pipe paths -LOGFILE="/tmp/$( echo "$__ScriptName" | sed s/.sh/.log/g )" -LOGPIPE="/tmp/$( echo "$__ScriptName" | sed s/.sh/.logpipe/g )" +# Let's discover how we're being called +# shellcheck disable=SC2009 +CALLER=$(ps -a -o pid,args | grep $$ | grep -v grep | tr -s ' ' | cut -d ' ' -f 3) -# Create our logging pipe -# On FreeBSD we have to use mkfifo instead of mknod -mknod "$LOGPIPE" p >/dev/null 2>&1 || mkfifo "$LOGPIPE" >/dev/null 2>&1 -if [ $? -ne 0 ]; then - echoerror "Failed to create the named pipe required to log" +if [ "${CALLER}x" = "${0}x" ]; then + CALLER="shell pipe" +fi + +echoinfo "Running version: ${__ScriptVersion}" +echoinfo "Executed by: ${CALLER}" +echoinfo "Command line: '${__ScriptFullName} ${__ScriptArgs}'" +#echowarn "Running the unstable version of ${__ScriptName}" + +# Define installation type +if [ "$#" -gt 0 ];then + __check_unparsed_options "$*" + ITYPE=$1 + shift +fi + +# Check installation type +if [ "$(echo "$ITYPE" | egrep '(stable|testing|daily|git)')" = "" ]; then + echoerror "Installation type \"$ITYPE\" is not known..." exit 1 fi -# What ever is written to the logpipe gets written to the logfile -tee < "$LOGPIPE" "$LOGFILE" & +# If doing a git install, check what branch/tag/sha will be checked out +if [ "$ITYPE" = "git" ]; then + if [ "$#" -eq 0 ];then + GIT_REV="develop" + else + GIT_REV="$1" + shift + fi -# Close STDOUT, reopen it directing it to the logpipe -exec 1>&- -exec 1>"$LOGPIPE" -# Close STDERR, reopen it directing it to the logpipe -exec 2>&- -exec 2>"$LOGPIPE" + # Disable shell warning about unbound variable during git install + STABLE_REV="latest" +# If doing stable install, check if version specified +elif [ "$ITYPE" = "stable" ]; then + if [ "$#" -eq 0 ];then + STABLE_REV="latest" + else + if [ "$(echo "$1" | egrep '^(latest|1\.6|1\.7|2014\.1|2014\.7|2015\.5|2015\.8|2016\.3|2016\.11|2017\.7)$')" != "" ]; then + STABLE_REV="$1" + shift + elif [ "$(echo "$1" | egrep '^([0-9]*\.[0-9]*\.[0-9]*)$')" != "" ]; then + STABLE_REV="archive/$1" + shift + else + echo "Unknown stable version: $1 (valid: 1.6, 1.7, 2014.1, 2014.7, 2015.5, 2015.8, 2016.3, 2016.11, 2017.7, latest, \$MAJOR.\$MINOR.\$PATCH)" + exit 1 + fi + fi +fi + +# Check for any unparsed arguments. Should be an error. +if [ "$#" -gt 0 ]; then + __usage + echo + echoerror "Too many arguments." + exit 1 +fi + +# whoami alternative for SunOS +if [ -f /usr/xpg4/bin/id ]; then + whoami='/usr/xpg4/bin/id -un' +else + whoami='whoami' +fi + +# Root permissions are required to run this script +if [ "$($whoami)" != "root" ]; then + echoerror "Salt requires root privileges to install. Please re-run this script as root." + exit 1 +fi + +# Check that we're actually installing one of minion/master/syndic +if [ "$_INSTALL_MINION" -eq $BS_FALSE ] && [ "$_INSTALL_MASTER" -eq $BS_FALSE ] && [ "$_INSTALL_SYNDIC" -eq $BS_FALSE ] && [ "$_CONFIG_ONLY" -eq $BS_FALSE ]; then + echowarn "Nothing to install or configure" + exit 1 +fi + +# Check that we're installing a minion if we're being passed a master address +if [ "$_INSTALL_MINION" -eq $BS_FALSE ] && [ "$_SALT_MASTER_ADDRESS" != "null" ]; then + echoerror "Don't pass a master address (-A) if no minion is going to be bootstrapped." + exit 1 +fi + +# Check that we're installing a minion if we're being passed a minion id +if [ "$_INSTALL_MINION" -eq $BS_FALSE ] && [ "$_SALT_MINION_ID" != "null" ]; then + echoerror "Don't pass a minion id (-i) if no minion is going to be bootstrapped." + exit 1 +fi + +# Check that we're installing or configuring a master if we're being passed a master config json dict +if [ "$_CUSTOM_MASTER_CONFIG" != "null" ]; then + if [ "$_INSTALL_MASTER" -eq $BS_FALSE ] && [ "$_CONFIG_ONLY" -eq $BS_FALSE ]; then + echoerror "Don't pass a master config JSON dict (-J) if no master is going to be bootstrapped or configured." + exit 1 + fi +fi + +# Check that we're installing or configuring a minion if we're being passed a minion config json dict +if [ "$_CUSTOM_MINION_CONFIG" != "null" ]; then + if [ "$_INSTALL_MINION" -eq $BS_FALSE ] && [ "$_CONFIG_ONLY" -eq $BS_FALSE ]; then + echoerror "Don't pass a minion config JSON dict (-j) if no minion is going to be bootstrapped or configured." + exit 1 + fi +fi + +# If the configuration directory or archive does not exist, error out +if [ "$_TEMP_CONFIG_DIR" != "null" ]; then + _TEMP_CONFIG_DIR="$(__check_config_dir "$_TEMP_CONFIG_DIR")" + [ "$_TEMP_CONFIG_DIR" = "null" ] && exit 1 +fi + +# If the pre-seed keys directory does not exist, error out +if [ "$_TEMP_KEYS_DIR" != "null" ] && [ ! -d "$_TEMP_KEYS_DIR" ]; then + echoerror "The pre-seed keys directory ${_TEMP_KEYS_DIR} does not exist." + exit 1 +fi + +# -a and -V only work from git +if [ "$ITYPE" != "git" ]; then + if [ $_PIP_ALL -eq $BS_TRUE ]; then + echoerror "Pip installing all python packages with -a is only possible when installing Salt via git" + exit 1 + fi + if [ "$_VIRTUALENV_DIR" != "null" ]; then + echoerror "Virtualenv installs via -V is only possible when installing Salt via git" + exit 1 + fi +fi + +# Set the _REPO_URL value based on if -R was passed or not. Defaults to repo.saltstack.com. +if [ "$_CUSTOM_REPO_URL" != "null" ]; then + _REPO_URL="$_CUSTOM_REPO_URL" + + # Check for -r since -R is being passed. Set -r with a warning. + if [ "$_DISABLE_REPOS" -eq $BS_FALSE ]; then + echowarn "Detected -R option. No other repositories will be configured when -R is used. Setting -r option to True." + _DISABLE_REPOS=$BS_TRUE + fi +fi + +# Check the _DISABLE_SSL value and set HTTP or HTTPS. +if [ "$_DISABLE_SSL" -eq $BS_TRUE ]; then + HTTP_VAL="http" +else + HTTP_VAL="https" +fi + +# Check the _QUIET_GIT_INSTALLATION value and set SETUP_PY_INSTALL_ARGS. +if [ "$_QUIET_GIT_INSTALLATION" -eq $BS_TRUE ]; then + SETUP_PY_INSTALL_ARGS="-q" +else + SETUP_PY_INSTALL_ARGS="" +fi + +# Export the http_proxy configuration to our current environment +if [ "${_HTTP_PROXY}" != "" ]; then + export http_proxy="$_HTTP_PROXY" + export https_proxy="$_HTTP_PROXY" +fi + +# Work around for 'Docker + salt-bootstrap failure' https://github.com/saltstack/salt-bootstrap/issues/394 +if [ "${_DISABLE_SALT_CHECKS}" -eq $BS_FALSE ] && [ -f /tmp/disable_salt_checks ]; then + # shellcheck disable=SC2016 + echowarn 'Found file: /tmp/disable_salt_checks, setting _DISABLE_SALT_CHECKS=$BS_TRUE' + _DISABLE_SALT_CHECKS=$BS_TRUE +fi + +# Because -a can only be installed into virtualenv +if [ "${_PIP_ALL}" -eq $BS_TRUE ] && [ "${_VIRTUALENV_DIR}" = "null" ]; then + usage + # Could possibly set up a default virtualenv location when -a flag is passed + echoerror "Using -a requires -V because pip pkgs should be siloed from python system pkgs" + exit 1 +fi + +# Make sure virtualenv directory does not already exist +if [ -d "${_VIRTUALENV_DIR}" ]; then + echoerror "The directory ${_VIRTUALENV_DIR} for virtualenv already exists" + exit 1 +fi # Handle the insecure flags if [ "$_INSECURE_DL" -eq $BS_TRUE ]; then @@ -856,8 +857,10 @@ __derive_debian_numeric_version() { elif [ "$INPUT_VERSION" = "jessie/sid" ]; then NUMERIC_VERSION=$(__parse_version_string "8.0") elif [ "$INPUT_VERSION" = "stretch/sid" ]; then - # Let's start detecting the upcoming Debian 9 (Stretch) NUMERIC_VERSION=$(__parse_version_string "9.0") + elif [ "$INPUT_VERSION" = "buster/sid" ]; then + # Let's start detecting the upcoming Debian 10 (Buster) release + NUMERIC_VERSION=$(__parse_version_string "10.0") else echowarn "Unable to parse the Debian Version (codename: '$INPUT_VERSION')" fi @@ -1089,11 +1092,11 @@ __gather_linux_system_info() { #--- FUNCTION ------------------------------------------------------------------------------------------------------- -# NAME: __install_python_and_deps() -# DESCRIPTION: Install a different version of python and its dependencies on a host. Currently this has only been -# tested on Centos 6 and is considered experimental. +# NAME: __install_python() +# DESCRIPTION: Install a different version of python on a host. Currently this has only been tested on CentOS 6 and +# is considered experimental. #---------------------------------------------------------------------------------------------------------------------- -__install_python_and_deps() { +__install_python() { if [ "$_PY_EXE" = "" ]; then echoerror "Must specify -x with -y to install a specific python version" exit 1 @@ -1103,7 +1106,7 @@ __install_python_and_deps() { __PACKAGES="${PY_PKG_V}" - if [ $_DISABLE_REPOS -eq $BS_FALSE ]; then + if [ ${_DISABLE_REPOS} -eq ${BS_FALSE} ]; then echoinfo "Attempting to install a repo to help provide a separate python package" echoinfo "$DISTRO_NAME_L" case "$DISTRO_NAME_L" in @@ -1112,7 +1115,7 @@ __install_python_and_deps() { ;; *) echoerror "Installing a repo to provide a python package is only supported on Redhat/CentOS. - If a repo is already available please try running script with -r" + If a repo is already available, please try running script with -r." exit 1 ;; esac @@ -1123,13 +1126,6 @@ __install_python_and_deps() { echoinfo "Installing ${__PACKAGES}" __yum_install_noinput "${__PACKAGES}" || return 1 - - _PIP_PACKAGES="tornado PyYAML msgpack-python jinja2 pycrypto zmq" - if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ]; then - _PIP_PACKAGES="${_PIP_PACKAGES} apache-libcloud" - fi - - __install_pip_pkgs "${_PIP_PACKAGES}" "${_PY_EXE}" || return 1 } @@ -1243,22 +1239,6 @@ __gather_system_info() { } -#--- FUNCTION ------------------------------------------------------------------------------------------------------- -# NAME: __get_dpkg_architecture -# DESCRIPTION: Determine primary architecture for packages to install on Debian and derivatives -#---------------------------------------------------------------------------------------------------------------------- -__get_dpkg_architecture() { - if __check_command_exists dpkg; then - DPKG_ARCHITECTURE="$(dpkg --print-architecture)" - else - echoerror "dpkg: command not found." - return 1 - fi - - return 0 -} - - #--- FUNCTION ------------------------------------------------------------------------------------------------------- # NAME: __ubuntu_derivatives_translation # DESCRIPTION: Map Ubuntu derivatives to their Ubuntu base versions. @@ -1305,6 +1285,63 @@ __ubuntu_derivatives_translation() { } +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: __check_dpkg_architecture +# DESCRIPTION: Determine the primary architecture for packages to install on Debian and derivatives +# and issue all necessary error messages. +#---------------------------------------------------------------------------------------------------------------------- +__check_dpkg_architecture() { + if __check_command_exists dpkg; then + DPKG_ARCHITECTURE="$(dpkg --print-architecture)" + else + echoerror "dpkg: command not found." + return 1 + fi + + __REPO_ARCH="$DPKG_ARCHITECTURE" + __return_code=0 + + case $DPKG_ARCHITECTURE in + "i386") + error_msg="$_REPO_URL likely doesn't have all required 32-bit packages for $DISTRO_NAME $DISTRO_MAJOR_VERSION." + # amd64 is just a part of repository URI, 32-bit pkgs are hosted under the same location + __REPO_ARCH="amd64" + ;; + "amd64") + error_msg="" + ;; + "armhf") + if [ "$DISTRO_NAME_L" = "ubuntu" ] || [ "$DISTRO_MAJOR_VERSION" -lt 8 ]; then + error_msg="Support for armhf packages at $_REPO_URL is limited to Debian/Raspbian 8 platforms." + __return_code=1 + else + error_msg="" + fi + ;; + *) + error_msg="$_REPO_URL doesn't have packages for your system architecture: $DPKG_ARCHITECTURE." + __return_code=1 + ;; + esac + + if [ "${error_msg}" != "" ]; then + echoerror "${error_msg}" + if [ "$ITYPE" != "git" ]; then + echoerror "You can try git installation mode, i.e.: sh ${__ScriptName} git v2016.11.5." + echoerror "It may be necessary to use git installation mode with pip and disable the SaltStack apt repository." + echoerror "For example:" + echoerror " sh ${__ScriptName} -r -P git v2016.11.5" + fi + fi + + if [ "${__return_code}" -eq 0 ]; then + return 0 + else + return 1 + fi +} + + #--- FUNCTION ------------------------------------------------------------------------------------------------------- # NAME: __ubuntu_codename_translation # DESCRIPTION: Map Ubuntu major versions to their corresponding codenames @@ -1403,18 +1440,172 @@ __debian_derivatives_translation() { #--- FUNCTION ------------------------------------------------------------------------------------------------------- -# NAME: __check_and_refresh_suse_pkg_repo -# DESCRIPTION: Check if zypper knows about systemsmanagement_saltstack repo yet. -# If it doesn't, add the repo and refresh with the SUSE_PKG_URL. +# NAME: __debian_codename_translation +# DESCRIPTION: Map Debian major versions to their corresponding code names #---------------------------------------------------------------------------------------------------------------------- -__check_and_refresh_suse_pkg_repo() { - # Check to see if systemsmanagement_saltstack exists - __zypper repos | grep systemsmanagement_saltstack >/dev/null 2>&1 +# shellcheck disable=SC2034 +__debian_codename_translation() { - if [ $? -eq 1 ]; then - # zypper does not yet know anything about systemsmanagement_saltstack - __zypper addrepo --refresh "${SUSE_PKG_URL}" || return 1 - fi + case $DISTRO_MAJOR_VERSION in + "7") + DISTRO_CODENAME="wheezy" + ;; + "8") + DISTRO_CODENAME="jessie" + ;; + "9") + DISTRO_CODENAME="stretch" + ;; + "10") + DISTRO_CODENAME="buster" + ;; + *) + DISTRO_CODENAME="jessie" + ;; + esac +} + + +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: __check_end_of_life_versions +# DESCRIPTION: Check for end of life distribution versions +#---------------------------------------------------------------------------------------------------------------------- +__check_end_of_life_versions() { + case "${DISTRO_NAME_L}" in + debian) + # Debian versions below 7 are not supported + if [ "$DISTRO_MAJOR_VERSION" -lt 7 ]; then + echoerror "End of life distributions are not supported." + echoerror "Please consider upgrading to the next stable. See:" + echoerror " https://wiki.debian.org/DebianReleases" + exit 1 + fi + ;; + + ubuntu) + # Ubuntu versions not supported + # + # < 14.04 + # = 14.10 + # = 15.04, 15.10 + if [ "$DISTRO_MAJOR_VERSION" -lt 14 ] || \ + [ "$DISTRO_MAJOR_VERSION" -eq 15 ] || \ + ([ "$DISTRO_MAJOR_VERSION" -lt 16 ] && [ "$DISTRO_MINOR_VERSION" -eq 10 ]); then + echoerror "End of life distributions are not supported." + echoerror "Please consider upgrading to the next stable. See:" + echoerror " https://wiki.ubuntu.com/Releases" + exit 1 + fi + ;; + + opensuse) + # openSUSE versions not supported + # + # <= 12.1 + if ([ "$DISTRO_MAJOR_VERSION" -eq 12 ] && [ "$DISTRO_MINOR_VERSION" -eq 1 ]) || [ "$DISTRO_MAJOR_VERSION" -lt 12 ]; then + echoerror "End of life distributions are not supported." + echoerror "Please consider upgrading to the next stable. See:" + echoerror " http://en.opensuse.org/Lifetime" + exit 1 + fi + ;; + + suse) + # SuSE versions not supported + # + # < 11 SP2 + SUSE_PATCHLEVEL=$(awk '/PATCHLEVEL/ {print $3}' /etc/SuSE-release ) + if [ "${SUSE_PATCHLEVEL}" = "" ]; then + SUSE_PATCHLEVEL="00" + fi + if ([ "$DISTRO_MAJOR_VERSION" -eq 11 ] && [ "$SUSE_PATCHLEVEL" -lt 02 ]) || [ "$DISTRO_MAJOR_VERSION" -lt 11 ]; then + echoerror "Versions lower than SuSE 11 SP2 are not supported." + echoerror "Please consider upgrading to the next stable" + exit 1 + fi + ;; + + fedora) + # Fedora lower than 24 are no longer supported + if [ "$DISTRO_MAJOR_VERSION" -lt 24 ]; then + echoerror "End of life distributions are not supported." + echoerror "Please consider upgrading to the next stable. See:" + echoerror " https://fedoraproject.org/wiki/Releases" + exit 1 + fi + ;; + + centos) + # CentOS versions lower than 6 are no longer supported + if [ "$DISTRO_MAJOR_VERSION" -lt 6 ]; then + echoerror "End of life distributions are not supported." + echoerror "Please consider upgrading to the next stable. See:" + echoerror " http://wiki.centos.org/Download" + exit 1 + fi + ;; + + red_hat*linux) + # Red Hat (Enterprise) Linux versions lower than 6 are no longer supported + if [ "$DISTRO_MAJOR_VERSION" -lt 6 ]; then + echoerror "End of life distributions are not supported." + echoerror "Please consider upgrading to the next stable. See:" + echoerror " https://access.redhat.com/support/policy/updates/errata/" + exit 1 + fi + ;; + + oracle*linux) + # Oracle Linux versions lower than 6 are no longer supported + if [ "$DISTRO_MAJOR_VERSION" -lt 6 ]; then + echoerror "End of life distributions are not supported." + echoerror "Please consider upgrading to the next stable. See:" + echoerror " http://www.oracle.com/us/support/library/elsp-lifetime-069338.pdf" + exit 1 + fi + ;; + + scientific*linux) + # Scientific Linux versions lower than 6 are no longer supported + if [ "$DISTRO_MAJOR_VERSION" -lt 6 ]; then + echoerror "End of life distributions are not supported." + echoerror "Please consider upgrading to the next stable. See:" + echoerror " https://www.scientificlinux.org/downloads/sl-versions/" + exit 1 + fi + ;; + + cloud*linux) + # Cloud Linux versions lower than 6 are no longer supported + if [ "$DISTRO_MAJOR_VERSION" -lt 6 ]; then + echoerror "End of life distributions are not supported." + echoerror "Please consider upgrading to the next stable. See:" + echoerror " https://docs.cloudlinux.com/index.html?cloudlinux_life-cycle.html" + exit 1 + fi + ;; + + amazon*linux*ami) + # Amazon Linux versions lower than 2012.0X no longer supported + if [ "$DISTRO_MAJOR_VERSION" -lt 2012 ]; then + echoerror "End of life distributions are not supported." + echoerror "Please consider upgrading to the next stable. See:" + echoerror " https://aws.amazon.com/amazon-linux-ami/" + exit 1 + fi + ;; + + freebsd) + # FreeBSD versions lower than 9.1 are not supported. + if ([ "$DISTRO_MAJOR_VERSION" -eq 9 ] && [ "$DISTRO_MINOR_VERSION" -lt 01 ]) || [ "$DISTRO_MAJOR_VERSION" -lt 9 ]; then + echoerror "Versions lower than FreeBSD 9.1 are not supported." + exit 1 + fi + ;; + + *) + ;; + esac } @@ -1429,6 +1620,37 @@ echoinfo " OS Version: ${OS_VERSION}" echoinfo " Distribution: ${DISTRO_NAME} ${DISTRO_VERSION}" echo +# Simplify distro name naming on functions +DISTRO_NAME_L=$(echo "$DISTRO_NAME" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-zA-Z0-9_ ]//g' | sed -re 's/([[:space:]])+/_/g') + +# Simplify version naming on functions +if [ "$DISTRO_VERSION" = "" ] || [ ${_SIMPLIFY_VERSION} -eq $BS_FALSE ]; then + DISTRO_MAJOR_VERSION="" + DISTRO_MINOR_VERSION="" + PREFIXED_DISTRO_MAJOR_VERSION="" + PREFIXED_DISTRO_MINOR_VERSION="" +else + DISTRO_MAJOR_VERSION=$(echo "$DISTRO_VERSION" | sed 's/^\([0-9]*\).*/\1/g') + DISTRO_MINOR_VERSION=$(echo "$DISTRO_VERSION" | sed 's/^\([0-9]*\).\([0-9]*\).*/\2/g') + PREFIXED_DISTRO_MAJOR_VERSION="_${DISTRO_MAJOR_VERSION}" + if [ "${PREFIXED_DISTRO_MAJOR_VERSION}" = "_" ]; then + PREFIXED_DISTRO_MAJOR_VERSION="" + fi + PREFIXED_DISTRO_MINOR_VERSION="_${DISTRO_MINOR_VERSION}" + if [ "${PREFIXED_DISTRO_MINOR_VERSION}" = "_" ]; then + PREFIXED_DISTRO_MINOR_VERSION="" + fi +fi + +# For Ubuntu derivatives, pretend to be their Ubuntu base version +__ubuntu_derivatives_translation + +# For Debian derivates, pretend to be their Debian base version +__debian_derivatives_translation + +# Fail soon for end of life versions +__check_end_of_life_versions + echodebug "Binaries will be searched using the following \$PATH: ${PATH}" # Let users know that we'll use a proxy @@ -1469,37 +1691,14 @@ if [ $_START_DAEMONS -eq $BS_FALSE ]; then echoinfo "Daemons will not be started" fi -# Simplify distro name naming on functions -DISTRO_NAME_L=$(echo "$DISTRO_NAME" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-zA-Z0-9_ ]//g' | sed -re 's/([[:space:]])+/_/g') - -# For Ubuntu derivatives, pretend to be their Ubuntu base version -__ubuntu_derivatives_translation - -# For Debian derivates, pretend to be their Debian base version -__debian_derivatives_translation - -# Simplify version naming on functions -if [ "$DISTRO_VERSION" = "" ] || [ ${_SIMPLIFY_VERSION} -eq $BS_FALSE ]; then - DISTRO_MAJOR_VERSION="" - DISTRO_MINOR_VERSION="" - PREFIXED_DISTRO_MAJOR_VERSION="" - PREFIXED_DISTRO_MINOR_VERSION="" -else - DISTRO_MAJOR_VERSION=$(echo "$DISTRO_VERSION" | sed 's/^\([0-9]*\).*/\1/g') - DISTRO_MINOR_VERSION=$(echo "$DISTRO_VERSION" | sed 's/^\([0-9]*\).\([0-9]*\).*/\2/g') - PREFIXED_DISTRO_MAJOR_VERSION="_${DISTRO_MAJOR_VERSION}" - if [ "${PREFIXED_DISTRO_MAJOR_VERSION}" = "_" ]; then - PREFIXED_DISTRO_MAJOR_VERSION="" - fi - PREFIXED_DISTRO_MINOR_VERSION="_${DISTRO_MINOR_VERSION}" - if [ "${PREFIXED_DISTRO_MINOR_VERSION}" = "_" ]; then - PREFIXED_DISTRO_MINOR_VERSION="" - fi +if [ "${DISTRO_NAME_L}" = "ubuntu" ]; then + # For ubuntu versions, obtain the codename from the release version + __ubuntu_codename_translation +elif [ "${DISTRO_NAME_L}" = "debian" ]; then + # For debian versions, obtain the codename from the release version + __debian_codename_translation fi -# For ubuntu versions, obtain the codename from the release version -__ubuntu_codename_translation - # Only Ubuntu has daily packages, let's let users know about that if ([ "${DISTRO_NAME_L}" != "ubuntu" ] && [ "$ITYPE" = "daily" ]); then echoerror "${DISTRO_NAME} does not have daily packages support" @@ -1525,14 +1724,18 @@ if ([ "${DISTRO_NAME_L}" != "ubuntu" ] && [ "$_VIRTUALENV_DIR" != "null" ]); the fi # Only Ubuntu has support for pip installing all packages -if ([ "${DISTRO_NAME_L}" != "ubuntu" ] && [ $_PIP_ALL -eq $BS_TRUE ]);then +if ([ "${DISTRO_NAME_L}" != "ubuntu" ] && [ $_PIP_ALL -eq $BS_TRUE ]); then echoerror "${DISTRO_NAME} does not have -a support" exit 1 fi -# Starting from Ubuntu 16.10, gnupg-curl has been renamed to gnupg1-curl. +# Starting from Debian 9 and Ubuntu 16.10, gnupg-curl has been renamed to gnupg1-curl. GNUPG_CURL="gnupg-curl" -if [ "${DISTRO_NAME_L}" = "ubuntu" ]; then +if [ "$DISTRO_NAME_L" = "debian" ]; then + if [ "$DISTRO_MAJOR_VERSION" -gt 8 ]; then + GNUPG_CURL="gnupg1-curl" + fi +elif [ "$DISTRO_NAME_L" = "ubuntu" ]; then if [ "${DISTRO_VERSION}" = "16.10" ] || [ "$DISTRO_MAJOR_VERSION" -gt 16 ]; then GNUPG_CURL="gnupg1-curl" fi @@ -1624,7 +1827,9 @@ __rpm_import_gpg() { __yum_install_noinput() { ENABLE_EPEL_CMD="" - if [ $_DISABLE_REPOS -eq $BS_FALSE ]; then + # Skip Amazon Linux for the first round, since EPEL is no longer required. + # See issue #724 + if [ $_DISABLE_REPOS -eq $BS_FALSE ] && [ "$DISTRO_NAME_L" != "amazon_linux_ami" ]; then ENABLE_EPEL_CMD="--enablerepo=${_EPEL_REPO}" fi @@ -1768,152 +1973,6 @@ __git_clone_and_checkout() { } -#--- FUNCTION ------------------------------------------------------------------------------------------------------- -# NAME: __check_end_of_life_versions -# DESCRIPTION: Check for end of life distribution versions -#---------------------------------------------------------------------------------------------------------------------- -__check_end_of_life_versions() { - case "${DISTRO_NAME_L}" in - debian) - # Debian versions bellow 6 are not supported - if [ "$DISTRO_MAJOR_VERSION" -lt 7 ]; then - echoerror "End of life distributions are not supported." - echoerror "Please consider upgrading to the next stable. See:" - echoerror " https://wiki.debian.org/DebianReleases" - exit 1 - fi - ;; - - ubuntu) - # Ubuntu versions not supported - # - # < 12.04 - # 13.x, 15.x - # 12.10, 14.10 - if [ "$DISTRO_MAJOR_VERSION" -lt 12 ] || \ - [ "$DISTRO_MAJOR_VERSION" -eq 13 ] || \ - [ "$DISTRO_MAJOR_VERSION" -eq 15 ] || \ - ([ "$DISTRO_MAJOR_VERSION" -lt 16 ] && [ "$DISTRO_MINOR_VERSION" -eq 10 ]); then - echoerror "End of life distributions are not supported." - echoerror "Please consider upgrading to the next stable. See:" - echoerror " https://wiki.ubuntu.com/Releases" - exit 1 - fi - ;; - - opensuse) - # openSUSE versions not supported - # - # <= 12.1 - if ([ "$DISTRO_MAJOR_VERSION" -eq 12 ] && [ "$DISTRO_MINOR_VERSION" -eq 1 ]) || [ "$DISTRO_MAJOR_VERSION" -lt 12 ]; then - echoerror "End of life distributions are not supported." - echoerror "Please consider upgrading to the next stable. See:" - echoerror " http://en.opensuse.org/Lifetime" - exit 1 - fi - ;; - - suse) - # SuSE versions not supported - # - # < 11 SP2 - SUSE_PATCHLEVEL=$(awk '/PATCHLEVEL/ {print $3}' /etc/SuSE-release ) - if [ "${SUSE_PATCHLEVEL}" = "" ]; then - SUSE_PATCHLEVEL="00" - fi - if ([ "$DISTRO_MAJOR_VERSION" -eq 11 ] && [ "$SUSE_PATCHLEVEL" -lt 02 ]) || [ "$DISTRO_MAJOR_VERSION" -lt 11 ]; then - echoerror "Versions lower than SuSE 11 SP2 are not supported." - echoerror "Please consider upgrading to the next stable" - exit 1 - fi - ;; - - fedora) - # Fedora lower than 18 are no longer supported - if [ "$DISTRO_MAJOR_VERSION" -lt 23 ]; then - echoerror "End of life distributions are not supported." - echoerror "Please consider upgrading to the next stable. See:" - echoerror " https://fedoraproject.org/wiki/Releases" - exit 1 - fi - ;; - - centos) - # CentOS versions lower than 5 are no longer supported - if [ "$DISTRO_MAJOR_VERSION" -lt 6 ]; then - echoerror "End of life distributions are not supported." - echoerror "Please consider upgrading to the next stable. See:" - echoerror " http://wiki.centos.org/Download" - exit 1 - fi - ;; - - red_hat*linux) - # Red Hat (Enterprise) Linux versions lower than 5 are no longer supported - if [ "$DISTRO_MAJOR_VERSION" -lt 6 ]; then - echoerror "End of life distributions are not supported." - echoerror "Please consider upgrading to the next stable. See:" - echoerror " https://access.redhat.com/support/policy/updates/errata/" - exit 1 - fi - ;; - - oracle*linux) - # Oracle Linux versions lower than 5 are no longer supported - if [ "$DISTRO_MAJOR_VERSION" -lt 6 ]; then - echoerror "End of life distributions are not supported." - echoerror "Please consider upgrading to the next stable. See:" - echoerror " http://www.oracle.com/us/support/library/elsp-lifetime-069338.pdf" - exit 1 - fi - ;; - - scientific*linux) - # Scientific Linux versions lower than 5 are no longer supported - if [ "$DISTRO_MAJOR_VERSION" -lt 6 ]; then - echoerror "End of life distributions are not supported." - echoerror "Please consider upgrading to the next stable. See:" - echoerror " https://www.scientificlinux.org/downloads/sl-versions/" - exit 1 - fi - ;; - - cloud*linux) - # Cloud Linux versions lower than 5 are no longer supported - if [ "$DISTRO_MAJOR_VERSION" -lt 6 ]; then - echoerror "End of life distributions are not supported." - echoerror "Please consider upgrading to the next stable. See:" - echoerror " https://docs.cloudlinux.com/index.html?cloudlinux_life-cycle.html" - exit 1 - fi - ;; - - amazon*linux*ami) - # Amazon Linux versions lower than 2012.0X no longer supported - if [ "$DISTRO_MAJOR_VERSION" -lt 2012 ]; then - echoerror "End of life distributions are not supported." - echoerror "Please consider upgrading to the next stable. See:" - echoerror " https://aws.amazon.com/amazon-linux-ami/" - exit 1 - fi - ;; - - freebsd) - # FreeBSD versions lower than 9.1 are not supported. - if ([ "$DISTRO_MAJOR_VERSION" -eq 9 ] && [ "$DISTRO_MINOR_VERSION" -lt 01 ]) || [ "$DISTRO_MAJOR_VERSION" -lt 9 ]; then - echoerror "Versions lower than FreeBSD 9.1 are not supported." - exit 1 - fi - ;; - - *) - ;; - esac -} -# Fail soon for end of life versions -__check_end_of_life_versions - - #--- FUNCTION ------------------------------------------------------------------------------------------------------- # NAME: __copyfile # DESCRIPTION: Simple function to copy files. Overrides if asked. @@ -2479,38 +2538,55 @@ __enable_universe_repository() { return 0 } -install_ubuntu_deps() { - if [ "$DISTRO_MAJOR_VERSION" -gt 12 ]; then - # Above Ubuntu 12.04 add-apt-repository is in a different package - __apt_get_install_noinput software-properties-common || return 1 +__install_saltstack_ubuntu_repository() { + + # Workaround for latest non-LTS ubuntu + if [ "$DISTRO_VERSION" = "16.10" ] || [ "$DISTRO_MAJOR_VERSION" -gt 16 ]; then + echowarn "Non-LTS Ubuntu detected, but stable packages requested. Trying packages from latest LTS release. You may experience problems." + UBUNTU_VERSION=16.04 + UBUNTU_CODENAME="xenial" else - __apt_get_install_noinput python-software-properties || return 1 + UBUNTU_VERSION=$DISTRO_VERSION + UBUNTU_CODENAME=$DISTRO_CODENAME fi + # SaltStack's stable Ubuntu repository: + SALTSTACK_UBUNTU_URL="${HTTP_VAL}://${_REPO_URL}/apt/ubuntu/${UBUNTU_VERSION}/${__REPO_ARCH}/${STABLE_REV}" + echo "deb $SALTSTACK_UBUNTU_URL $UBUNTU_CODENAME main" > /etc/apt/sources.list.d/saltstack.list + + # Make sure https transport is available + if [ "$HTTP_VAL" = "https" ] ; then + __apt_get_install_noinput apt-transport-https ca-certificates || return 1 + fi + + __apt_key_fetch "$SALTSTACK_UBUNTU_URL/SALTSTACK-GPG-KEY.pub" || return 1 + + apt-get update +} + +install_ubuntu_deps() { if [ $_DISABLE_REPOS -eq $BS_FALSE ]; then - __enable_universe_repository || return 1 - - # Versions starting with 2015.5.6 and 2015.8.1 are hosted at repo.saltstack.com - if [ "$(echo "$STABLE_REV" | egrep '^(2015\.5|2015\.8|2016\.3|2016\.11|latest|archive\/)')" = "" ]; then - if [ "$DISTRO_MAJOR_VERSION" -lt 14 ]; then - echoinfo "Installing Python Requests/Chardet from Chris Lea's PPA repository" - add-apt-repository -y "ppa:chris-lea/python-requests" || return 1 - add-apt-repository -y "ppa:chris-lea/python-chardet" || return 1 - add-apt-repository -y "ppa:chris-lea/python-urllib3" || return 1 - add-apt-repository -y "ppa:chris-lea/python-crypto" || return 1 - fi - + # Install add-apt-repository + if ! __check_command_exists add-apt-repository; then + __apt_get_install_noinput software-properties-common || return 1 fi + __enable_universe_repository || return 1 + apt-get update fi - # Minimal systems might not have upstart installed, install it - __PACKAGES="upstart" + __PACKAGES='' + + if [ "$DISTRO_MAJOR_VERSION" -lt 16 ]; then + # Minimal systems might not have upstart installed, install it + __PACKAGES="upstart" + fi if [ "$DISTRO_MAJOR_VERSION" -ge 16 ]; then __PACKAGES="${__PACKAGES} python2.7" fi + if [ "$_VIRTUALENV_DIR" != "null" ]; then __PACKAGES="${__PACKAGES} python-virtualenv" fi @@ -2564,59 +2640,10 @@ install_ubuntu_stable_deps() { __apt_get_upgrade_noinput || return 1 fi - if [ ${_DISABLE_REPOS} -eq $BS_FALSE ]; then - __get_dpkg_architecture || return 1 - __REPO_ARCH="$DPKG_ARCHITECTURE" + __check_dpkg_architecture || return 1 - if [ "$DPKG_ARCHITECTURE" = "i386" ]; then - echoerror "$_REPO_URL likely doesn't have all required 32-bit packages for Ubuntu $DISTRO_MAJOR_VERSION (yet?)." - - # amd64 is just a part of repository URI, 32-bit pkgs are hosted under the same location - __REPO_ARCH="amd64" - elif [ "$DPKG_ARCHITECTURE" != "amd64" ]; then - echoerror "$_REPO_URL doesn't have packages for your system architecture: $DPKG_ARCHITECTURE." - if [ "$ITYPE" != "git" ]; then - echoerror "You can try git installation mode, i.e.: sh ${__ScriptName} git v2016.3.1" - exit 1 - fi - fi - - # Versions starting with 2015.5.6, 2015.8.1 and 2016.3.0 are hosted at repo.saltstack.com - if [ "$(echo "$STABLE_REV" | egrep '^(2015\.5|2015\.8|2016\.3|2016\.11|latest|archive\/)')" != "" ]; then - # Workaround for latest non-LTS ubuntu - if [ "$DISTRO_VERSION" = "16.10" ] || [ "$DISTRO_MAJOR_VERSION" -gt 16 ]; then - echowarn "Non-LTS Ubuntu detected, but stable packages requested. Trying packages from latest LTS release. You may experience problems." - UBUNTU_VERSION=16.04 - UBUNTU_CODENAME="xenial" - else - UBUNTU_VERSION=$DISTRO_VERSION - UBUNTU_CODENAME=$DISTRO_CODENAME - fi - - # SaltStack's stable Ubuntu repository: - SALTSTACK_UBUNTU_URL="${HTTP_VAL}://${_REPO_URL}/apt/ubuntu/${UBUNTU_VERSION}/${__REPO_ARCH}/${STABLE_REV}" - echo "deb $SALTSTACK_UBUNTU_URL $UBUNTU_CODENAME main" > /etc/apt/sources.list.d/saltstack.list - - # Make sure https transport is available - if [ "$HTTP_VAL" = "https" ] ; then - __apt_get_install_noinput apt-transport-https ca-certificates || return 1 - fi - - __apt_key_fetch "$SALTSTACK_UBUNTU_URL/SALTSTACK-GPG-KEY.pub" || return 1 - else - # Alternate PPAs: salt16, salt17, salt2014-1, salt2014-7 - if [ ! "$(echo "$STABLE_REV" | egrep '^(1\.6|1\.7)$')" = "" ]; then - STABLE_PPA="saltstack/salt$(echo "$STABLE_REV" | tr -d .)" - elif [ ! "$(echo "$STABLE_REV" | egrep '^(2014\.1|2014\.7)$')" = "" ]; then - STABLE_PPA="saltstack/salt$(echo "$STABLE_REV" | tr . -)" - else - STABLE_PPA="saltstack/salt" - fi - - add-apt-repository -y "ppa:$STABLE_PPA" || return 1 - fi - - apt-get update + if [ "$_DISABLE_REPOS" -eq "$BS_FALSE" ] || [ "$_CUSTOM_REPO_URL" != "null" ]; then + __install_saltstack_ubuntu_repository || return 1 fi install_ubuntu_deps || return 1 @@ -2625,13 +2652,6 @@ install_ubuntu_stable_deps() { install_ubuntu_daily_deps() { install_ubuntu_stable_deps || return 1 - if [ "$DISTRO_MAJOR_VERSION" -gt 12 ]; then - __apt_get_install_noinput software-properties-common || return 1 - else - # Ubuntu 12.04 needs python-software-properties to get add-apt-repository binary - __apt_get_install_noinput python-software-properties || return 1 - fi - if [ $_DISABLE_REPOS -eq $BS_FALSE ]; then __enable_universe_repository || return 1 @@ -2716,11 +2736,13 @@ install_ubuntu_stable() { # shellcheck disable=SC2086 __apt_get_install_noinput ${__PACKAGES} || return 1 + return 0 } install_ubuntu_daily() { install_ubuntu_stable || return 1 + return 0 } @@ -2735,6 +2757,7 @@ install_ubuntu_git() { else python setup.py ${SETUP_PY_INSTALL_ARGS} install --install-layout=deb || return 1 fi + return 0 } @@ -2760,6 +2783,8 @@ install_ubuntu_stable_post() { update-rc.d salt-$fname defaults fi done + + return 0 } install_ubuntu_git_post() { @@ -2772,7 +2797,7 @@ install_ubuntu_git_post() { [ $fname = "syndic" ] && [ "$_INSTALL_SYNDIC" -eq $BS_FALSE ] && continue if [ -f /bin/systemctl ] && [ "$DISTRO_MAJOR_VERSION" -ge 16 ]; then - __copyfile "${_SALT_GIT_CHECKOUT_DIR}/pkg/deb/salt-${fname}.service" "/lib/systemd/system/salt-${fname}.service" + __copyfile "${_SALT_GIT_CHECKOUT_DIR}/pkg/salt-${fname}.service" "/lib/systemd/system/salt-${fname}.service" # Skip salt-api since the service should be opt-in and not necessarily started on boot [ $fname = "api" ] && continue @@ -2796,10 +2821,10 @@ install_ubuntu_git_post() { /sbin/initctl reload-configuration || return 1 fi # No upstart support in Ubuntu!? - elif [ -f "${_SALT_GIT_CHECKOUT_DIR}/debian/salt-${fname}.init" ]; then + elif [ -f "${_SALT_GIT_CHECKOUT_DIR}/pkg/salt-${fname}.init" ]; then echodebug "There's NO upstart support!?" - echodebug "Copying ${_SALT_GIT_CHECKOUT_DIR}/debian/salt-${fname}.init to /etc/init.d/salt-$fname" - __copyfile "${_SALT_GIT_CHECKOUT_DIR}/debian/salt-${fname}.init" "/etc/init.d/salt-$fname" + echodebug "Copying ${_SALT_GIT_CHECKOUT_DIR}/pkg/salt-${fname}.init to /etc/init.d/salt-$fname" + __copyfile "${_SALT_GIT_CHECKOUT_DIR}/pkg/salt-${fname}.init" "/etc/init.d/salt-$fname" chmod +x /etc/init.d/salt-$fname # Skip salt-api since the service should be opt-in and not necessarily started on boot @@ -2810,6 +2835,8 @@ install_ubuntu_git_post() { echoerror "Neither upstart nor init.d was setup for salt-$fname" fi done + + return 0 } install_ubuntu_restart_daemons() { @@ -2862,6 +2889,7 @@ install_ubuntu_restart_daemons() { /etc/init.d/salt-$fname stop > /dev/null 2>&1 /etc/init.d/salt-$fname start done + return 0 } @@ -2895,6 +2923,33 @@ install_ubuntu_check_services() { # # Debian Install Functions # +__install_saltstack_debian_repository() { + if [ "$DISTRO_MAJOR_VERSION" -eq 10 ]; then + # Packages for Debian 10 at repo.saltstack.com are not yet available + # Set up repository for Debian 9 for Debian 10 for now until support + # is available at repo.saltstack.com for Debian 10. + echowarn "Debian 10 distribution detected, but stable packages requested. Trying packages from Debian 9. You may experience problems." + DEBIAN_RELEASE="9" + DEBIAN_CODENAME="stretch" + else + DEBIAN_RELEASE="$DISTRO_MAJOR_VERSION" + DEBIAN_CODENAME="$DISTRO_CODENAME" + fi + + # amd64 is just a part of repository URI, 32-bit pkgs are hosted under the same location + SALTSTACK_DEBIAN_URL="${HTTP_VAL}://${_REPO_URL}/apt/debian/${DEBIAN_RELEASE}/${__REPO_ARCH}/${STABLE_REV}" + echo "deb $SALTSTACK_DEBIAN_URL $DEBIAN_CODENAME main" > "/etc/apt/sources.list.d/saltstack.list" + + if [ "$HTTP_VAL" = "https" ] ; then + __apt_get_install_noinput apt-transport-https ca-certificates || return 1 + fi + + __apt_key_fetch "$SALTSTACK_DEBIAN_URL/SALTSTACK-GPG-KEY.pub" || return 1 + + apt-get update + +} + install_debian_deps() { if [ $_START_DAEMONS -eq $BS_FALSE ]; then echowarn "Not starting daemons on Debian based distributions is not working mostly because starting them is the default behaviour." @@ -2915,95 +2970,7 @@ install_debian_deps() { __apt_get_upgrade_noinput || return 1 fi - # Install procps and pciutils which allows for Docker bootstraps. See #366#issuecomment-39666813 - __PACKAGES="procps pciutils" - __PIP_PACKAGES="" - - # YAML module is used for generating custom master/minion configs - __PACKAGES="${__PACKAGES} python-yaml" - - # shellcheck disable=SC2086 - __apt_get_install_noinput ${__PACKAGES} || return 1 - - if [ "${_EXTRA_PACKAGES}" != "" ]; then - echoinfo "Installing the following extra packages as requested: ${_EXTRA_PACKAGES}" - # shellcheck disable=SC2086 - __apt_get_install_noinput ${_EXTRA_PACKAGES} || return 1 - fi - - if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ]; then - # shellcheck disable=SC2089 - __PIP_PACKAGES="${__PIP_PACKAGES} 'apache-libcloud>=$_LIBCLOUD_MIN_VERSION'" - fi - - if [ "${__PIP_PACKAGES}" != "" ]; then - # shellcheck disable=SC2086,SC2090 - pip install -U ${__PIP_PACKAGES} || return 1 - fi - - return 0 -} - -install_debian_7_deps() { - if [ $_START_DAEMONS -eq $BS_FALSE ]; then - echowarn "Not starting daemons on Debian based distributions is not working mostly because starting them is the default behaviour." - fi - - # No user interaction, libc6 restart services for example - export DEBIAN_FRONTEND=noninteractive - - apt-get update - - if [ "${_UPGRADE_SYS}" -eq $BS_TRUE ]; then - # Try to update GPG keys first if allowed - if [ "${_INSECURE_DL}" -eq $BS_TRUE ]; then - __apt_get_install_noinput --allow-unauthenticated debian-archive-keyring && - apt-key update && apt-get update - fi - - __apt_get_upgrade_noinput || return 1 - fi - - if [ "${_DISABLE_REPOS}" -eq $BS_FALSE ]; then - __get_dpkg_architecture || return 1 - - __REPO_ARCH="$DPKG_ARCHITECTURE" - - if [ "$DPKG_ARCHITECTURE" = "i386" ]; then - echoerror "$_REPO_URL likely doesn't have all required 32-bit packages for Debian $DISTRO_MAJOR_VERSION (yet?)." - - if [ "$ITYPE" != "git" ]; then - echoerror "You can try git installation mode, i.e.: sh ${__ScriptName} git v2016.3.1" - fi - - # amd64 is just a part of repository URI, 32-bit pkgs are hosted under the same location - __REPO_ARCH="amd64" - elif [ "$DPKG_ARCHITECTURE" != "amd64" ]; then - echoerror "$_REPO_URL doesn't have packages for your system architecture: $DPKG_ARCHITECTURE." - exit 1 - fi - - # Versions starting with 2015.8.7 and 2016.3.0 are hosted at repo.saltstack.com - if [ "$(echo "$STABLE_REV" | egrep '^(2015\.8|2016\.3|2016\.11|latest|archive\/201[5-6]\.)')" != "" ]; then - # amd64 is just a part of repository URI, 32-bit pkgs are hosted under the same location - SALTSTACK_DEBIAN_URL="${HTTP_VAL}://${_REPO_URL}/apt/debian/${DISTRO_MAJOR_VERSION}/${__REPO_ARCH}/${STABLE_REV}" - echo "deb $SALTSTACK_DEBIAN_URL wheezy main" > "/etc/apt/sources.list.d/saltstack.list" - - if [ "$HTTP_VAL" = "https" ] ; then - __apt_get_install_noinput apt-transport-https ca-certificates || return 1 - fi - - __apt_key_fetch "$SALTSTACK_DEBIAN_URL/SALTSTACK-GPG-KEY.pub" || return 1 - elif [ -n "$STABLE_REV" ]; then - echoerror "Installation of Salt ${STABLE_REV#*/} packages not supported by ${__ScriptName} ${__ScriptVersion} on Debian $DISTRO_MAJOR_VERSION." - - return 1 - fi - - apt-get update - else - echowarn "Packages from $_REPO_URL are required to install Salt version 2015.8 or higher on Debian $DISTRO_MAJOR_VERSION." - fi + __check_dpkg_architecture || return 1 # Additionally install procps and pciutils which allows for Docker bootstraps. See 366#issuecomment-39666813 __PACKAGES='procps pciutils' @@ -3011,87 +2978,17 @@ install_debian_7_deps() { # YAML module is used for generating custom master/minion configs __PACKAGES="${__PACKAGES} python-yaml" - # shellcheck disable=SC2086 - __apt_get_install_noinput ${__PACKAGES} || return 1 - - if [ "${_EXTRA_PACKAGES}" != "" ]; then - echoinfo "Installing the following extra packages as requested: ${_EXTRA_PACKAGES}" - # shellcheck disable=SC2086 - __apt_get_install_noinput ${_EXTRA_PACKAGES} || return 1 + # Debian 9 needs the dirmgr package in order to import the GPG key later + if [ "$DISTRO_MAJOR_VERSION" -ge 9 ]; then + __PACKAGES="${__PACKAGES} dirmngr" fi - return 0 -} - -install_debian_8_deps() { - if [ $_START_DAEMONS -eq $BS_FALSE ]; then - echowarn "Not starting daemons on Debian based distributions is not working mostly because starting them is the default behaviour." - fi - - # No user interaction, libc6 restart services for example - export DEBIAN_FRONTEND=noninteractive - - apt-get update - - if [ "${_UPGRADE_SYS}" -eq $BS_TRUE ]; then - # Try to update GPG keys first if allowed - if [ "${_INSECURE_DL}" -eq $BS_TRUE ]; then - __apt_get_install_noinput --allow-unauthenticated debian-archive-keyring && - apt-key update && apt-get update - fi - - __apt_get_upgrade_noinput || return 1 - fi - - if [ ${_DISABLE_REPOS} -eq $BS_FALSE ]; then - __get_dpkg_architecture || return 1 - - __REPO_ARCH="$DPKG_ARCHITECTURE" - - if [ "$DPKG_ARCHITECTURE" = "i386" ]; then - echoerror "$_REPO_URL likely doesn't have all required 32-bit packages for Debian $DISTRO_MAJOR_VERSION (yet?)." - - if [ "$ITYPE" != "git" ]; then - echoerror "You can try git installation mode, i.e.: sh ${__ScriptName} git v2016.3.1" - fi - - # amd64 is just a part of repository URI, 32-bit pkgs are hosted under the same location - __REPO_ARCH="amd64" - elif [ "$DPKG_ARCHITECTURE" != "amd64" ] && [ "$DPKG_ARCHITECTURE" != "armhf" ]; then - echoerror "$_REPO_URL doesn't have packages for your system architecture: $DPKG_ARCHITECTURE." - echoerror "Try git installation mode with pip and disable SaltStack apt repository, for example:" - echoerror " sh ${__ScriptName} -r -P git v2016.3.1" - - exit 1 - fi - - # Versions starting with 2015.5.6, 2015.8.1 and 2016.3.0 are hosted at repo.saltstack.com - if [ "$(echo "$STABLE_REV" | egrep '^(2015\.5|2015\.8|2016\.3|2016\.11|latest|archive\/201[5-6]\.)')" != "" ]; then - SALTSTACK_DEBIAN_URL="${HTTP_VAL}://${_REPO_URL}/apt/debian/${DISTRO_MAJOR_VERSION}/${__REPO_ARCH}/${STABLE_REV}" - echo "deb $SALTSTACK_DEBIAN_URL jessie main" > "/etc/apt/sources.list.d/saltstack.list" - - if [ "$HTTP_VAL" = "https" ] ; then - __apt_get_install_noinput apt-transport-https ca-certificates || return 1 - fi - - __apt_key_fetch "$SALTSTACK_DEBIAN_URL/SALTSTACK-GPG-KEY.pub" || return 1 - elif [ -n "$STABLE_REV" ]; then - echoerror "Installation of Salt ${STABLE_REV#*/} packages not supported by ${__ScriptName} ${__ScriptVersion} on Debian $DISTRO_MAJOR_VERSION." - - return 1 - fi - - apt-get update - fi - - # Additionally install procps and pciutils which allows for Docker bootstraps. See 366#issuecomment-39666813 - __PACKAGES='procps pciutils' - # shellcheck disable=SC2086 __apt_get_install_noinput ${__PACKAGES} || return 1 - # YAML module is used for generating custom master/minion configs - __PACKAGES="${__PACKAGES} python-yaml" + if [ "$_DISABLE_REPOS" -eq "$BS_FALSE" ] || [ "$_CUSTOM_REPO_URL" != "null" ]; then + __install_saltstack_debian_repository || return 1 + fi if [ "${_EXTRA_PACKAGES}" != "" ]; then echoinfo "Installing the following extra packages as requested: ${_EXTRA_PACKAGES}" @@ -3135,14 +3032,14 @@ install_debian_git_deps() { } install_debian_7_git_deps() { - install_debian_7_deps || return 1 + install_debian_deps || return 1 install_debian_git_deps || return 1 return 0 } install_debian_8_git_deps() { - install_debian_8_deps || return 1 + install_debian_deps || return 1 if ! __check_command_exists git; then __apt_get_install_noinput git || return 1 @@ -3204,6 +3101,45 @@ install_debian_8_git_deps() { return 0 } +install_debian_9_git_deps() { + install_debian_deps || return 1 + + if ! __check_command_exists git; then + __apt_get_install_noinput git || return 1 + fi + + if [ "$_INSECURE_DL" -eq $BS_FALSE ] && [ "${_SALT_REPO_URL%%://*}" = "https" ]; then + __apt_get_install_noinput ca-certificates + fi + + __git_clone_and_checkout || return 1 + + __PACKAGES="libzmq5 lsb-release python-apt python-backports-abc python-crypto" + __PACKAGES="${__PACKAGES} python-jinja2 python-msgpack python-requests python-systemd" + __PACKAGES="${__PACKAGES} python-tornado python-yaml python-zmq" + + if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ]; then + # Install python-libcloud if asked to + __PACKAGES="${__PACKAGES} python-libcloud" + fi + + # shellcheck disable=SC2086 + __apt_get_install_noinput ${__PACKAGES} || return 1 + + # Let's trigger config_salt() + if [ "$_TEMP_CONFIG_DIR" = "null" ]; then + _TEMP_CONFIG_DIR="${_SALT_GIT_CHECKOUT_DIR}/conf/" + CONFIG_SALT_FUNC="config_salt" + fi + + return 0 +} + +install_debian_10_git_deps() { + install_debian_9_git_deps || return 1 + return 0 +} + install_debian_stable() { __PACKAGES="" @@ -3236,9 +3172,14 @@ install_debian_8_stable() { return 0 } +install_debian_9_stable() { + install_debian_stable || return 1 + return 0 +} + install_debian_git() { if [ -f "${_SALT_GIT_CHECKOUT_DIR}/salt/syspaths.py" ]; then - python setup.py --salt-config-dir="$_SALT_ETC_DIR" --salt-cache-dir="${_SALT_CACHE_DIR}" ${SETUP_PY_INSTALL_ARGS} install --install-layout=deb || return 1 + python setup.py --salt-config-dir="$_SALT_ETC_DIR" --salt-cache-dir="${_SALT_CACHE_DIR}" ${SETUP_PY_INSTALL_ARGS} install --install-layout=deb || return 1 else python setup.py ${SETUP_PY_INSTALL_ARGS} install --install-layout=deb || return 1 fi @@ -3254,6 +3195,11 @@ install_debian_8_git() { return 0 } +install_debian_9_git() { + install_debian_git || return 1 + return 0 +} + install_debian_git_post() { for fname in api master minion syndic; do # Skip if not meant to be installed @@ -3267,9 +3213,9 @@ install_debian_git_post() { if [ -f /bin/systemctl ]; then if [ ! -f /lib/systemd/system/salt-${fname}.service ] || \ ([ -f /lib/systemd/system/salt-${fname}.service ] && [ $_FORCE_OVERWRITE -eq $BS_TRUE ]); then - if [ -f "${_SALT_GIT_CHECKOUT_DIR}/pkg/deb/salt-${fname}.service" ]; then - __copyfile "${_SALT_GIT_CHECKOUT_DIR}/pkg/deb/salt-${fname}.service" /lib/systemd/system - __copyfile "${_SALT_GIT_CHECKOUT_DIR}/pkg/deb/salt-${fname}.environment" "/etc/default/salt-${fname}" + if [ -f "${_SALT_GIT_CHECKOUT_DIR}/pkg/salt-${fname}.service" ]; then + __copyfile "${_SALT_GIT_CHECKOUT_DIR}/pkg/salt-${fname}.service" /lib/systemd/system + __copyfile "${_SALT_GIT_CHECKOUT_DIR}/pkg/salt-${fname}.environment" "/etc/default/salt-${fname}" else # workaround before adding Debian-specific unit files to the Salt main repo __copyfile "${_SALT_GIT_CHECKOUT_DIR}/pkg/salt-${fname}.service" /lib/systemd/system @@ -3286,9 +3232,9 @@ install_debian_git_post() { # Install initscripts for Debian 7 "Wheezy" elif [ ! -f "/etc/init.d/salt-$fname" ] || \ ([ -f "/etc/init.d/salt-$fname" ] && [ "$_FORCE_OVERWRITE" -eq $BS_TRUE ]); then - if [ -f "${_SALT_GIT_CHECKOUT_DIR}/pkg/deb/salt-$fname.init" ]; then - __copyfile "${_SALT_GIT_CHECKOUT_DIR}/pkg/deb/salt-${fname}.init" "/etc/init.d/salt-${fname}" - __copyfile "${_SALT_GIT_CHECKOUT_DIR}/pkg/deb/salt-${fname}.environment" "/etc/default/salt-${fname}" + if [ -f "${_SALT_GIT_CHECKOUT_DIR}/pkg/salt-$fname.init" ]; then + __copyfile "${_SALT_GIT_CHECKOUT_DIR}/pkg/salt-${fname}.init" "/etc/init.d/salt-${fname}" + __copyfile "${_SALT_GIT_CHECKOUT_DIR}/pkg/salt-${fname}.environment" "/etc/default/salt-${fname}" else # Make sure wget is available __check_command_exists wget || __apt_get_install_noinput wget || return 1 @@ -3442,23 +3388,7 @@ install_fedora_git_deps() { __git_clone_and_checkout || return 1 - __PACKAGES="systemd-python" - __PIP_PACKAGES="" - - if [ -f "${_SALT_GIT_CHECKOUT_DIR}/requirements/base.txt" ]; then - # We're on the develop branch, install whichever tornado is on the requirements file - __REQUIRED_TORNADO="$(grep tornado "${_SALT_GIT_CHECKOUT_DIR}/requirements/base.txt")" - if [ "${__REQUIRED_TORNADO}" != "" ]; then - __check_pip_allowed "You need to allow pip based installations (-P) in order to install tornado" - - # Install pip and pip dependencies - if ! __check_command_exists pip; then - __PACKAGES="${__PACKAGES} python-setuptools python-pip gcc python-devel" - fi - - __PIP_PACKAGES="${__PIP_PACKAGES} tornado" - fi - fi + __PACKAGES="python2-tornado systemd-python" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ]; then __PACKAGES="${__PACKAGES} python-libcloud python-netaddr" @@ -3467,11 +3397,6 @@ install_fedora_git_deps() { # shellcheck disable=SC2086 dnf install -y ${__PACKAGES} || return 1 - if [ "${__PIP_PACKAGES}" != "" ]; then - # shellcheck disable=SC2086,SC2090 - pip install -U ${__PIP_PACKAGES} || return 1 - fi - # Let's trigger config_salt() if [ "$_TEMP_CONFIG_DIR" = "null" ]; then _TEMP_CONFIG_DIR="${_SALT_GIT_CHECKOUT_DIR}/conf/" @@ -3563,13 +3488,6 @@ __install_epel_repository() { return 0 fi - # Check if epel-release is already installed and flag it accordingly - rpm --nodigest --nosignature -q epel-release > /dev/null 2>&1 - if [ $? -eq 0 ]; then - _EPEL_REPOS_INSTALLED=$BS_TRUE - return 0 - fi - # Download latest 'epel-release' package for the distro version directly epel_repo_url="${HTTP_VAL}://dl.fedoraproject.org/pub/epel/epel-release-latest-${DISTRO_MAJOR_VERSION}.noarch.rpm" rpm -Uvh --force "$epel_repo_url" || return 1 @@ -3754,8 +3672,29 @@ install_centos_git_deps() { __PACKAGES="${__PACKAGES} python-libcloud" fi - if [ "${_INSTALL_PY}" = "${BS_TRUE}" ]; then - __install_python_and_deps || return 1 + if [ "${_INSTALL_PY}" -eq "${BS_TRUE}" ]; then + # Install Python if "-y" was passed in. + __install_python || return 1 + fi + + if [ "${_PY_EXE}" != "" ]; then + # If "-x" is defined, install dependencies with pip based on the Python version given. + _PIP_PACKAGES="jinja2 msgpack-python pycrypto PyYAML tornado zmq" + + if [ -f "${_SALT_GIT_CHECKOUT_DIR}/requirements/base.txt" ]; then + for SINGLE_PACKAGE in $_PIP_PACKAGES; do + __REQUIRED_VERSION="$(grep "${SINGLE_PACKAGE}" "${_SALT_GIT_CHECKOUT_DIR}/requirements/base.txt")" + if [ "${__REQUIRED_VERSION}" != "" ]; then + _PIP_PACKAGES=$(echo "$_PIP_PACKAGES" | sed "s/${SINGLE_PACKAGE}/${__REQUIRED_VERSION}/") + fi + done + fi + + if [ "$_INSTALL_CLOUD" -eq "${BS_TRUE}" ]; then + _PIP_PACKAGES="${_PIP_PACKAGES} apache-libcloud" + fi + + __install_pip_pkgs "${_PIP_PACKAGES}" "${_PY_EXE}" || return 1 else # shellcheck disable=SC2086 __yum_install_noinput ${__PACKAGES} || return 1 @@ -4460,44 +4399,37 @@ daemons_running_alpine_linux() { install_amazon_linux_ami_deps() { # Shim to figure out if we're using old (rhel) or new (aws) rpms. _USEAWS=$BS_FALSE + pkg_append="python" repo_rev="$(echo "${STABLE_REV}" | sed 's|.*\/||g')" - if echo "$repo_rev" | egrep -q '^(latest|2016\.11)$'; then - _USEAWS=$BS_TRUE - elif echo "$repo_rev" | egrep -q '^[0-9]+$' && [ "$(echo "$repo_rev" | cut -c1-4)" -gt 2016 ]; then + if echo "$repo_rev" | egrep -q '^(latest|2016\.11)$' || \ + ( echo "$repo_rev" | egrep -q '^[0-9]+$' && [ "$(echo "$repo_rev" | cut -c1-4)" -gt 2016 ] ); then _USEAWS=$BS_TRUE + pkg_append="python27" fi # We need to install yum-utils before doing anything else when installing on # Amazon Linux ECS-optimized images. See issue #974. - yum -y install yum-utils + __yum_install_noinput yum-utils - ENABLE_EPEL_CMD="" - if [ $_DISABLE_REPOS -eq $BS_TRUE ]; then - ENABLE_EPEL_CMD="--enablerepo=${_EPEL_REPO}" + # Do upgrade early + if [ "$_UPGRADE_SYS" -eq $BS_TRUE ]; then + yum -y update || return 1 fi - if [ $_DISABLE_REPOS -eq $BS_FALSE ]; then - # enable the EPEL repo - /usr/bin/yum-config-manager --enable epel || return 1 - - # exclude Salt and ZeroMQ packages from EPEL - /usr/bin/yum-config-manager epel --setopt "epel.exclude=zeromq* salt* python-zmq*" --save || return 1 - + if [ $_DISABLE_REPOS -eq $BS_FALSE ] || [ "$_CUSTOM_REPO_URL" != "null" ]; then __REPO_FILENAME="saltstack-repo.repo" # Set a few vars to make life easier. if [ $_USEAWS -eq $BS_TRUE ]; then - base_url="$HTTP_VAL://repo.saltstack.com/yum/amazon/latest/\$basearch/$repo_rev/" + base_url="$HTTP_VAL://${_REPO_URL}/yum/amazon/latest/\$basearch/$repo_rev/" gpg_key="${base_url}SALTSTACK-GPG-KEY.pub" repo_name="SaltStack repo for Amazon Linux" - pkg_append="python27" else - base_url="$HTTP_VAL://repo.saltstack.com/yum/redhat/6/\$basearch/$repo_rev/" + base_url="$HTTP_VAL://${_REPO_URL}/yum/redhat/6/\$basearch/$repo_rev/" gpg_key="${base_url}SALTSTACK-GPG-KEY.pub" repo_name="SaltStack repo for RHEL/CentOS 6" - pkg_append="python" fi # This should prob be refactored to use __install_saltstack_rhel_repository() @@ -4515,21 +4447,19 @@ baseurl=$base_url _eof fi - if [ "$_UPGRADE_SYS" -eq $BS_TRUE ]; then - yum -y update || return 1 - fi fi - #ordereddict removed. - #Package python-ordereddict-1.1-2.el6.noarch is obsoleted by python26-2.6.9-2.88.amzn1.x86_64 which is already installed + + # Package python-ordereddict-1.1-2.el6.noarch is obsoleted by python26-2.6.9-2.88.amzn1.x86_64 + # which is already installed __PACKAGES="${pkg_append}-PyYAML ${pkg_append}-crypto ${pkg_append}-msgpack ${pkg_append}-zmq ${pkg_append}-jinja2 ${pkg_append}-requests" # shellcheck disable=SC2086 - yum -y install ${__PACKAGES} ${ENABLE_EPEL_CMD} || return 1 + __yum_install_noinput ${__PACKAGES} || return 1 if [ "${_EXTRA_PACKAGES}" != "" ]; then echoinfo "Installing the following extra packages as requested: ${_EXTRA_PACKAGES}" # shellcheck disable=SC2086 - yum install -y ${_EXTRA_PACKAGES} ${ENABLE_EPEL_CMD} || return 1 + __yum_install_noinput ${_EXTRA_PACKAGES} || return 1 fi } @@ -4549,13 +4479,8 @@ install_amazon_linux_ami_git_deps() { install_amazon_linux_ami_deps || return 1 - ENABLE_EPEL_CMD="" - if [ $_DISABLE_REPOS -eq $BS_TRUE ]; then - ENABLE_EPEL_CMD="--enablerepo=${_EPEL_REPO}" - fi - if ! __check_command_exists git; then - yum -y install git ${ENABLE_EPEL_CMD} || return 1 + __yum_install_noinput git || return 1 fi __git_clone_and_checkout || return 1 @@ -4579,7 +4504,7 @@ install_amazon_linux_ami_git_deps() { if [ "${__PACKAGES}" != "" ]; then # shellcheck disable=SC2086 - yum -y install ${__PACKAGES} ${ENABLE_EPEL_CMD} || return 1 + __yum_install_noinput ${__PACKAGES} || return 1 fi if [ "${__PIP_PACKAGES}" != "" ]; then @@ -4660,7 +4585,7 @@ install_arch_linux_stable_deps() { pacman -Su --noconfirm --needed python2-yaml if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ]; then - pacman -Su --noconfirm --needed apache-libcloud || return 1 + pacman -Su --noconfirm --needed python2-apache-libcloud || return 1 fi if [ "${_EXTRA_PACKAGES}" != "" ]; then @@ -5458,6 +5383,16 @@ install_smartos_restart_daemons() { # __ZYPPER_REQUIRES_REPLACE_FILES=-1 +__check_and_refresh_suse_pkg_repo() { + # Check to see if systemsmanagement_saltstack exists + __zypper repos | grep systemsmanagement_saltstack >/dev/null 2>&1 + + if [ $? -eq 1 ]; then + # zypper does not yet know anything about systemsmanagement_saltstack + __zypper addrepo --refresh "${SUSE_PKG_URL}" || return 1 + fi +} + __set_suse_pkg_repo() { suse_pkg_url_path="${DISTRO_REPO}/systemsmanagement:saltstack.repo" if [ "$_DOWNSTREAM_PKG_REPO" -eq $BS_TRUE ]; then @@ -6151,11 +6086,15 @@ install_suse_check_services() { # # Gentoo Install Functions. # +__autounmask() { + emerge --autounmask-write --autounmask-only "${@}"; return $? +} + __emerge() { if [ "$_GENTOO_USE_BINHOST" -eq $BS_TRUE ]; then - emerge --autounmask-write --getbinpkg "${@}"; return $? + emerge --getbinpkg "${@}"; return $? fi - emerge --autounmask-write "${@}"; return $? + emerge "${@}"; return $? } __gentoo_config_protection() { @@ -6165,6 +6104,9 @@ __gentoo_config_protection() { # cfg-update and then restart the bootstrapping script, so instead we allow # at this point to modify certain config files directly export CONFIG_PROTECT_MASK="${CONFIG_PROTECT_MASK:-} /etc/portage/package.accept_keywords /etc/portage/package.keywords /etc/portage/package.license /etc/portage/package.unmask /etc/portage/package.use" + + # emerge currently won't write to files that aren't there, so we need to ensure their presence + touch /etc/portage/package.accept_keywords /etc/portage/package.keywords /etc/portage/package.license /etc/portage/package.unmask /etc/portage/package.use } __gentoo_pre_dep() { @@ -6193,15 +6135,21 @@ __gentoo_post_dep() { __gentoo_config_protection if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ]; then + __autounmask 'dev-python/libcloud' __emerge -v 'dev-python/libcloud' fi + __autounmask 'dev-python/requests' + __autounmask 'app-admin/salt' + __emerge -vo 'dev-python/requests' __emerge -vo 'app-admin/salt' if [ "${_EXTRA_PACKAGES}" != "" ]; then echoinfo "Installing the following extra packages as requested: ${_EXTRA_PACKAGES}" # shellcheck disable=SC2086 + __autounmask ${_EXTRA_PACKAGES} || return 1 + # shellcheck disable=SC2086 __emerge -v ${_EXTRA_PACKAGES} || return 1 fi } @@ -6437,12 +6385,12 @@ config_salt() { # Copy the minion's keys if found if [ -f "$_TEMP_CONFIG_DIR/minion.pem" ]; then - __movefile "$_TEMP_CONFIG_DIR/minion.pem" "$_PKI_DIR/minion/" "$_CONFIG_ONLY" || return 1 + __movefile "$_TEMP_CONFIG_DIR/minion.pem" "$_PKI_DIR/minion/" "$_FORCE_OVERWRITE" || return 1 chmod 400 "$_PKI_DIR/minion/minion.pem" || return 1 CONFIGURED_ANYTHING=$BS_TRUE fi if [ -f "$_TEMP_CONFIG_DIR/minion.pub" ]; then - __movefile "$_TEMP_CONFIG_DIR/minion.pub" "$_PKI_DIR/minion/" "$_CONFIG_ONLY" || return 1 + __movefile "$_TEMP_CONFIG_DIR/minion.pub" "$_PKI_DIR/minion/" "$_FORCE_OVERWRITE" || return 1 chmod 664 "$_PKI_DIR/minion/minion.pub" || return 1 CONFIGURED_ANYTHING=$BS_TRUE fi diff --git a/salt/cloud/libcloudfuncs.py b/salt/cloud/libcloudfuncs.py index cdb0cf7fc6a..93a91e7b74f 100644 --- a/salt/cloud/libcloudfuncs.py +++ b/salt/cloud/libcloudfuncs.py @@ -9,7 +9,7 @@ from __future__ import absolute_import import os import logging from salt.ext.six import string_types -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import zip @@ -26,7 +26,7 @@ try: ) HAS_LIBCLOUD = True LIBCLOUD_VERSION_INFO = tuple([ - int(part) for part in re.compile(r"(\d+).(\d+).(\d+)").match(libcloud.__version__.replace('-', '.')).groups() + int(part) for part in libcloud.__version__.replace('-', '.').replace('rc', '.').split('.')[:3] ]) except ImportError: @@ -151,7 +151,7 @@ def avail_locations(conn=None, call=None): ret[img_name] = {} for attr in dir(img): - if attr.startswith('_'): + if attr.startswith('_') or attr == 'driver': continue attr_value = getattr(img, attr) @@ -188,7 +188,7 @@ def avail_images(conn=None, call=None): ret[img_name] = {} for attr in dir(img): - if attr.startswith('_'): + if attr.startswith('_') or attr in ('driver', 'get_uuid'): continue attr_value = getattr(img, attr) if isinstance(attr_value, string_types) and not six.PY3: @@ -223,7 +223,7 @@ def avail_sizes(conn=None, call=None): ret[size_name] = {} for attr in dir(size): - if attr.startswith('_'): + if attr.startswith('_') or attr in ('driver', 'get_uuid'): continue try: diff --git a/salt/config/__init__.py b/salt/config/__init__.py index a4dcb815581..6a89e1f4857 100644 --- a/salt/config/__init__.py +++ b/salt/config/__init__.py @@ -14,29 +14,27 @@ import getpass import time import codecs import logging -from copy import deepcopy import types - -# Import third party libs import yaml -try: - yaml.Loader = yaml.CLoader - yaml.Dumper = yaml.CDumper -except Exception: - pass +from copy import deepcopy # pylint: disable=import-error,no-name-in-module -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves.urllib.parse import urlparse # pylint: enable=import-error,no-name-in-module # Import salt libs import salt.utils import salt.utils.dictupdate +import salt.utils.files import salt.utils.network -import salt.syspaths +import salt.utils.path +import salt.utils.platform +import salt.utils.stringutils import salt.utils.validate.path import salt.utils.xdg +import salt.utils.yamlloader as yamlloader +import salt.syspaths import salt.exceptions from salt.utils.locales import sdecode import salt.defaults.exitcodes @@ -55,11 +53,11 @@ _DFLT_LOG_DATEFMT = '%H:%M:%S' _DFLT_LOG_DATEFMT_LOGFILE = '%Y-%m-%d %H:%M:%S' _DFLT_LOG_FMT_CONSOLE = '[%(levelname)-8s] %(message)s' _DFLT_LOG_FMT_LOGFILE = ( - '%(asctime)s,%(msecs)03d [%(name)-17s][%(levelname)-8s][%(process)d] %(message)s' + '%(asctime)s,%(msecs)03d [%(name)-17s:%(lineno)-4d][%(levelname)-8s][%(process)d] %(message)s' ) _DFLT_REFSPECS = ['+refs/heads/*:refs/remotes/origin/*', '+refs/tags/*:refs/tags/*'] -if salt.utils.is_windows(): +if salt.utils.platform.is_windows(): # Since an 'ipc_mode' of 'ipc' will never work on Windows due to lack of # support in ZeroMQ, we want the default to be something that has a # chance of working. @@ -113,9 +111,10 @@ VALID_OPTS = { 'master_port': (six.string_types, int), # The behaviour of the minion when connecting to a master. Can specify 'failover', - # 'disable' or 'func'. If 'func' is specified, the 'master' option should be set to an - # exec module function to run to determine the master hostname. If 'disable' is specified - # the minion will run, but will not try to connect to a master. + # 'disable', 'distributed', or 'func'. If 'func' is specified, the 'master' option should be + # set to an exec module function to run to determine the master hostname. If 'disable' is + # specified the minion will run, but will not try to connect to a master. If 'distributed' + # is specified the minion will try to deterministically pick a master based on its' id. 'master_type': str, # Specify the format in which the master address will be specified. Can @@ -188,6 +187,16 @@ VALID_OPTS = { # A unique identifier for this daemon 'id': str, + # Use a module function to determine the unique identifier. If this is + # set and 'id' is not set, it will allow invocation of a module function + # to determine the value of 'id'. For simple invocations without function + # arguments, this may be a string that is the function name. For + # invocations with function arguments, this may be a dictionary with the + # key being the function name, and the value being an embedded dictionary + # where each key is a function argument name and each value is the + # corresponding argument value. + 'id_function': (dict, str), + # The directory to store all cache files. 'cachedir': str, @@ -205,6 +214,9 @@ VALID_OPTS = { # The directory containing unix sockets for things like the event bus 'sock_dir': str, + # The pool size of unix sockets, it is necessary to avoid blocking waiting for zeromq and tcp communications. + 'sock_pool_size': int, + # Specifies how the file server should backup files, if enabled. The backups # live in the cache dir. 'backup_mode': str, @@ -331,7 +343,7 @@ VALID_OPTS = { # Whether or not scheduled mine updates should be accompanied by a job return for the job cache 'mine_return_job': bool, - # Schedule a mine update every n number of seconds + # The number of minutes between mine updates. 'mine_interval': int, # The ipc strategy. (i.e., sockets versus tcp, etc) @@ -349,7 +361,7 @@ VALID_OPTS = { # The TCP port on which minion events should be pulled if ipc_mode is TCP 'tcp_pull_port': int, - # The TCP port on which events for the master should be pulled if ipc_mode is TCP + # The TCP port on which events for the master should be published if ipc_mode is TCP 'tcp_master_pub_port': int, # The TCP port on which events for the master should be pulled if ipc_mode is TCP @@ -416,6 +428,12 @@ VALID_OPTS = { # Tell the client to display the jid when a job is published 'show_jid': bool, + # Ensure that a generated jid is always unique. If this is set, the jid + # format is different due to an underscore and process id being appended + # to the jid. WARNING: A change to the jid format may break external + # applications that depend on the original format. + 'unique_jid': bool, + # Tells the highstate outputter to show successful states. False will omit successes. 'state_verbose': bool, @@ -454,6 +472,12 @@ VALID_OPTS = { # Allow a daemon to function even if the key directories are not secured 'permissive_pki_access': bool, + # The passphrase of the master's private key + 'key_pass': str, + + # The passphrase of the master's private signing key + 'signing_key_pass': str, + # The path to a directory to pull in configuration file includes 'default_include': str, @@ -566,6 +590,23 @@ VALID_OPTS = { # False in 2016.3.0 'add_proxymodule_to_opts': bool, + # Merge pillar data into configuration opts. + # As multiple proxies can run on the same server, we may need different + # configuration options for each, while there's one single configuration file. + # The solution is merging the pillar data of each proxy minion into the opts. + 'proxy_merge_pillar_in_opts': bool, + + # Deep merge of pillar data into configuration opts. + # Evaluated only when `proxy_merge_pillar_in_opts` is True. + 'proxy_deep_merge_pillar_in_opts': bool, + + # The strategy used when merging pillar into opts. + # Considered only when `proxy_merge_pillar_in_opts` is True. + 'proxy_merge_pillar_in_opts_strategy': str, + + # Allow enabling mine details using pillar data. + 'proxy_mines_pillar': bool, + # In some particular cases, always alive proxies are not beneficial. # This option can be used in those less dynamic environments: # the user can request the connection @@ -611,7 +652,9 @@ VALID_OPTS = { 'gitfs_ssl_verify': bool, 'gitfs_global_lock': bool, 'gitfs_saltenv': list, + 'gitfs_ref_types': list, 'gitfs_refspecs': list, + 'gitfs_disable_saltenv_mapping': bool, 'hgfs_remotes': list, 'hgfs_mountpoint': str, 'hgfs_root': str, @@ -710,6 +753,10 @@ VALID_OPTS = { # same module used for external authentication. 'eauth_acl_module': str, + # Subsystem to use to maintain eauth tokens. By default, tokens are stored on the local + # filesystem + 'eauth_tokens': str, + # The number of open files a daemon is allowed to have open. Frequently needs to be increased # higher than the system default in order to account for the way zeromq consumes file handles. 'max_open_files': int, @@ -723,6 +770,10 @@ VALID_OPTS = { # A mapping of external systems that can be used to generate topfile data. 'master_tops': dict, + # Whether or not matches from master_tops should be executed before or + # after those from the top file(s). + 'master_tops_first': bool, + # A flag that should be set on a top-level master when it is ordering around subordinate masters # via the use of a salt syndic 'order_masters': bool, @@ -891,6 +942,7 @@ VALID_OPTS = { 'ssh_scan_timeout': float, 'ssh_identities_only': bool, 'ssh_log_file': str, + 'ssh_config_file': str, # Enable ioflo verbose logging. Warning! Very verbose! 'ioflo_verbose': int, @@ -919,7 +971,7 @@ VALID_OPTS = { 'queue_dirs': list, - # Instructs the minion to ping its master(s) ever n number of seconds. Used + # Instructs the minion to ping its master(s) every n number of seconds. Used # primarily as a mitigation technique against minion disconnects. 'ping_interval': int, @@ -1062,6 +1114,11 @@ VALID_OPTS = { # (in other words, require that minions have 'minion_sign_messages' # turned on) 'require_minion_sign_messages': bool, + + # The list of config entries to be passed to external pillar function as + # part of the extra_minion_data param + # Subconfig entries can be specified by using the ':' notation (e.g. key:subkey) + 'pass_to_ext_pillars': (six.string_types, list), } # default configurations @@ -1085,6 +1142,7 @@ DEFAULT_MINION_OPTS = { 'root_dir': salt.syspaths.ROOT_DIR, 'pki_dir': os.path.join(salt.syspaths.CONFIG_DIR, 'pki', 'minion'), 'id': '', + 'id_function': {}, 'cachedir': os.path.join(salt.syspaths.CACHE_DIR, 'minion'), 'append_minionid_config_dirs': [], 'cache_jobs': False, @@ -1093,6 +1151,7 @@ DEFAULT_MINION_OPTS = { 'grains_deep_merge': False, 'conf_file': os.path.join(salt.syspaths.CONFIG_DIR, 'minion'), 'sock_dir': os.path.join(salt.syspaths.SOCK_DIR, 'minion'), + 'sock_pool_size': 1, 'backup_mode': '', 'renderer': 'yaml_jinja', 'renderer_whitelist': [], @@ -1176,7 +1235,10 @@ DEFAULT_MINION_OPTS = { 'gitfs_global_lock': True, 'gitfs_ssl_verify': True, 'gitfs_saltenv': [], + 'gitfs_ref_types': ['branch', 'tag', 'sha'], 'gitfs_refspecs': _DFLT_REFSPECS, + 'gitfs_disable_saltenv_mapping': False, + 'unique_jid': False, 'hash_type': 'sha256', 'disable_modules': [], 'disable_returners': [], @@ -1283,6 +1345,7 @@ DEFAULT_MINION_OPTS = { 'auth_timeout': 5, 'auth_tries': 7, 'master_tries': _MASTER_TRIES, + 'master_tops_first': False, 'auth_safemode': False, 'random_master': False, 'minion_floscript': os.path.join(FLO_DIR, 'minion.flo'), @@ -1340,6 +1403,7 @@ DEFAULT_MASTER_OPTS = { 'user': _MASTER_USER, 'worker_threads': 5, 'sock_dir': os.path.join(salt.syspaths.SOCK_DIR, 'master'), + 'sock_pool_size': 1, 'ret_port': 4506, 'timeout': 5, 'keep_jobs': 24, @@ -1405,7 +1469,9 @@ DEFAULT_MASTER_OPTS = { 'gitfs_global_lock': True, 'gitfs_ssl_verify': True, 'gitfs_saltenv': [], + 'gitfs_ref_types': ['branch', 'tag', 'sha'], 'gitfs_refspecs': _DFLT_REFSPECS, + 'gitfs_disable_saltenv_mapping': False, 'hgfs_remotes': [], 'hgfs_mountpoint': '', 'hgfs_root': '', @@ -1417,6 +1483,7 @@ DEFAULT_MASTER_OPTS = { 'hgfs_saltenv_blacklist': [], 'show_timeout': True, 'show_jid': False, + 'unique_jid': False, 'svnfs_remotes': [], 'svnfs_mountpoint': '', 'svnfs_root': '', @@ -1449,8 +1516,9 @@ DEFAULT_MASTER_OPTS = { 'syndic_forward_all_events': False, 'syndic_log_file': os.path.join(salt.syspaths.LOGS_DIR, 'syndic'), 'syndic_pidfile': os.path.join(salt.syspaths.PIDFILE_DIR, 'salt-syndic.pid'), - 'runner_dirs': [], 'outputter_dirs': [], + 'runner_dirs': [], + 'utils_dirs': [], 'client_acl_verify': True, 'publisher_acl': {}, 'publisher_acl_blacklist': {}, @@ -1460,6 +1528,7 @@ DEFAULT_MASTER_OPTS = { 'token_expire_user_override': False, 'keep_acl_in_token': False, 'eauth_acl_module': '', + 'eauth_tokens': 'localfs', 'extension_modules': os.path.join(salt.syspaths.CACHE_DIR, 'master', 'extmods'), 'file_recv': False, 'file_recv_max_size': 100, @@ -1540,6 +1609,8 @@ DEFAULT_MASTER_OPTS = { 'key_logfile': os.path.join(salt.syspaths.LOGS_DIR, 'key'), 'verify_env': True, 'permissive_pki_access': False, + 'key_pass': None, + 'signing_key_pass': None, 'default_include': 'master.d/*.conf', 'winrepo_dir': os.path.join(salt.syspaths.BASE_FILE_ROOTS_DIR, 'win', 'repo'), 'winrepo_dir_ng': os.path.join(salt.syspaths.BASE_FILE_ROOTS_DIR, 'win', 'repo-ng'), @@ -1579,6 +1650,7 @@ DEFAULT_MASTER_OPTS = { 'ssh_scan_timeout': 0.01, 'ssh_identities_only': False, 'ssh_log_file': os.path.join(salt.syspaths.LOGS_DIR, 'ssh'), + 'ssh_config_file': os.path.join(salt.syspaths.HOME_DIR, '.ssh', 'config'), 'master_floscript': os.path.join(FLO_DIR, 'master.flo'), 'worker_floscript': os.path.join(FLO_DIR, 'worker.flo'), 'maintenance_floscript': os.path.join(FLO_DIR, 'maint.flo'), @@ -1641,9 +1713,16 @@ DEFAULT_PROXY_MINION_OPTS = { 'log_file': os.path.join(salt.syspaths.LOGS_DIR, 'proxy'), 'add_proxymodule_to_opts': False, 'proxy_merge_grains_in_module': True, - 'append_minionid_config_dirs': ['cachedir', 'pidfile', 'default_include'], + 'extension_modules': os.path.join(salt.syspaths.CACHE_DIR, 'proxy', 'extmods'), + 'append_minionid_config_dirs': ['cachedir', 'pidfile', 'default_include', 'extension_modules'], 'default_include': 'proxy.d/*.conf', + 'proxy_merge_pillar_in_opts': False, + 'proxy_deep_merge_pillar_in_opts': False, + 'proxy_merge_pillar_in_opts_strategy': 'smart', + + 'proxy_mines_pillar': True, + # By default, proxies will preserve the connection. # If this option is set to False, # the connection with the remote dumb device @@ -1818,7 +1897,7 @@ def _validate_opts(opts): # We don't know what data type sdb will return at run-time so we # simply cannot check it for correctness here at start-time. - if isinstance(val, str) and val.startswith('sdb://'): + if isinstance(val, six.string_types) and val.startswith('sdb://'): continue if hasattr(VALID_OPTS[key], '__call__'): @@ -1862,7 +1941,7 @@ def _validate_opts(opts): # sock_dirs must start with '\\.\mailslot\' and not contain any colons. # We don't expect the user to know this, so we will fix up their path for # them if it isn't compliant. - if (salt.utils.is_windows() and opts.get('transport') == 'raet' and + if (salt.utils.platform.is_windows() and opts.get('transport') == 'raet' and 'sock_dir' in opts and not opts['sock_dir'].startswith('\\\\.\\mailslot\\')): opts['sock_dir'] = ( @@ -1916,9 +1995,12 @@ def _read_conf_file(path): Read in a config file from a given path and process it into a dictionary ''' log.debug('Reading configuration from {0}'.format(path)) - with salt.utils.fopen(path, 'r') as conf_file: + with salt.utils.files.fopen(path, 'r') as conf_file: try: - conf_opts = yaml.safe_load(conf_file.read()) or {} + conf_opts = yamlloader.load( + conf_file.read(), + Loader=yamlloader.SaltYamlSafeLoader, + ) or {} except yaml.YAMLError as err: message = 'Error parsing configuration file: {0} - {1}'.format(path, err) log.error(message) @@ -2008,8 +2090,8 @@ def load_config(path, env_var, default_path=None, exit_on_config_errors=True): template = '{0}.template'.format(path) if os.path.isfile(template): log.debug('Writing {0} based on {1}'.format(path, template)) - with salt.utils.fopen(path, 'w') as out: - with salt.utils.fopen(template, 'r') as ifile: + with salt.utils.files.fopen(path, 'w') as out: + with salt.utils.files.fopen(template, 'r') as ifile: ifile.readline() # skip first line out.write(ifile.read()) @@ -2126,7 +2208,7 @@ def prepend_root_dir(opts, path_options): # No prepending required continue # Prepending the root dir - opts[path_option] = salt.utils.path_join(root_dir, path) + opts[path_option] = salt.utils.path.join(root_dir, path) def insert_system_path(opts, paths): @@ -2300,6 +2382,7 @@ def syndic_config(master_config_path, 'sock_dir': os.path.join( opts['cachedir'], opts.get('syndic_sock_dir', opts['sock_dir']) ), + 'sock_pool_size': master_opts['sock_pool_size'], 'cachedir': master_opts['cachedir'], } opts.update(syndic_opts) @@ -2308,7 +2391,7 @@ def syndic_config(master_config_path, 'pki_dir', 'cachedir', 'pidfile', 'sock_dir', 'extension_modules', 'autosign_file', 'autoreject_file', 'token_dir' ] - for config_key in ('syndic_log_file', 'log_file', 'key_logfile'): + for config_key in ('log_file', 'key_logfile', 'syndic_log_file'): # If this is not a URI and instead a local path if urlparse(opts.get(config_key, '')).scheme == '': prepend_root_dirs.append(config_key) @@ -2638,7 +2721,7 @@ def old_to_new(opts): providers = ( 'AWS', 'CLOUDSTACK', - 'DIGITAL_OCEAN', + 'DIGITALOCEAN', 'EC2', 'GOGRID', 'IBMSCE', @@ -3199,12 +3282,12 @@ def is_profile_configured(opts, provider, profile_name, vm_=None): alias, driver = provider.split(':') # Most drivers need an image to be specified, but some do not. - non_image_drivers = ['nova', 'virtualbox', 'libvirt', 'softlayer'] + non_image_drivers = ['nova', 'virtualbox', 'libvirt', 'softlayer', 'oneandone'] # Most drivers need a size, but some do not. non_size_drivers = ['opennebula', 'parallels', 'proxmox', 'scaleway', 'softlayer', 'softlayer_hw', 'vmware', 'vsphere', - 'virtualbox', 'profitbricks', 'libvirt'] + 'virtualbox', 'profitbricks', 'libvirt', 'oneandone'] provider_key = opts['providers'][alias][driver] profile_key = opts['providers'][alias][driver]['profiles'][profile_name] @@ -3296,12 +3379,63 @@ def _cache_id(minion_id, cache_file): Helper function, writes minion id to a cache file. ''' try: - with salt.utils.fopen(cache_file, 'w') as idf: + with salt.utils.files.fopen(cache_file, 'w') as idf: idf.write(minion_id) except (IOError, OSError) as exc: log.error('Could not cache minion ID: {0}'.format(exc)) +def call_id_function(opts): + ''' + Evaluate the function that determines the ID if the 'id_function' + option is set and return the result + ''' + if opts.get('id'): + return opts['id'] + + # Import 'salt.loader' here to avoid a circular dependency + import salt.loader as loader + + if isinstance(opts['id_function'], str): + mod_fun = opts['id_function'] + fun_kwargs = {} + elif isinstance(opts['id_function'], dict): + mod_fun, fun_kwargs = six.next(six.iteritems(opts['id_function'])) + if fun_kwargs is None: + fun_kwargs = {} + else: + log.error('\'id_function\' option is neither a string nor a dictionary') + sys.exit(salt.defaults.exitcodes.EX_GENERIC) + + # split module and function and try loading the module + mod, fun = mod_fun.split('.') + if not opts.get('grains'): + # Get grains for use by the module + opts['grains'] = loader.grains(opts) + + try: + id_mod = loader.raw_mod(opts, mod, fun) + if not id_mod: + raise KeyError + # we take whatever the module returns as the minion ID + newid = id_mod[mod_fun](**fun_kwargs) + if not isinstance(newid, str) or not newid: + log.error('Function {0} returned value "{1}" of type {2} instead of string'.format( + mod_fun, newid, type(newid)) + ) + sys.exit(salt.defaults.exitcodes.EX_GENERIC) + log.info('Evaluated minion ID from module: {0}'.format(mod_fun)) + return newid + except TypeError: + log.error('Function arguments {0} are incorrect for function {1}'.format( + fun_kwargs, mod_fun) + ) + sys.exit(salt.defaults.exitcodes.EX_GENERIC) + except KeyError: + log.error('Failed to load module {0}'.format(mod_fun)) + sys.exit(salt.defaults.exitcodes.EX_GENERIC) + + def get_id(opts, cache_minion_id=False): ''' Guess the id of the minion. @@ -3329,11 +3463,11 @@ def get_id(opts, cache_minion_id=False): if opts.get('minion_id_caching', True): try: - with salt.utils.fopen(id_cache) as idf: + with salt.utils.files.fopen(id_cache) as idf: name = idf.readline().strip() - bname = salt.utils.to_bytes(name) + bname = salt.utils.stringutils.to_bytes(name) if bname.startswith(codecs.BOM): # Remove BOM if exists - name = salt.utils.to_str(bname.replace(codecs.BOM, '', 1)) + name = salt.utils.stringutils.to_str(bname.replace(codecs.BOM, '', 1)) if name and name != 'localhost': log.debug('Using cached minion ID from {0}: {1}'.format(id_cache, name)) return name, False @@ -3343,13 +3477,21 @@ def get_id(opts, cache_minion_id=False): log.debug('Guessing ID. The id can be explicitly set in {0}' .format(os.path.join(salt.syspaths.CONFIG_DIR, 'minion'))) - newid = salt.utils.network.generate_minion_id() + if opts.get('id_function'): + newid = call_id_function(opts) + else: + newid = salt.utils.network.generate_minion_id() if opts.get('minion_id_lowercase'): newid = newid.lower() log.debug('Changed minion id {0} to lowercase.'.format(newid)) if '__role' in opts and opts.get('__role') == 'minion': - log.debug('Found minion id from generate_minion_id(): {0}'.format(newid)) + if opts.get('id_function'): + log.debug('Found minion id from external function {0}: {1}'.format( + opts['id_function'], newid)) + else: + log.debug('Found minion id from generate_minion_id(): {0}'.format( + newid)) if cache_minion_id and opts.get('minion_id_caching', True): _cache_id(newid, id_cache) is_ipv4 = salt.utils.network.is_ipv4(newid) @@ -3569,12 +3711,23 @@ def apply_master_config(overrides=None, defaults=None): if len(opts['sock_dir']) > len(opts['cachedir']) + 10: opts['sock_dir'] = os.path.join(opts['cachedir'], '.salt-unix') + opts['token_dir'] = os.path.join(opts['cachedir'], 'tokens') + opts['syndic_dir'] = os.path.join(opts['cachedir'], 'syndics') + # Make sure ext_mods gets set if it is an untrue value + # (here to catch older bad configs) opts['extension_modules'] = ( opts.get('extension_modules') or os.path.join(opts['cachedir'], 'extmods') ) - opts['token_dir'] = os.path.join(opts['cachedir'], 'tokens') - opts['syndic_dir'] = os.path.join(opts['cachedir'], 'syndics') + # Set up the utils_dirs location from the extension_modules location + opts['utils_dirs'] = ( + opts.get('utils_dirs') or + [os.path.join(opts['extension_modules'], 'utils')] + ) + + # Insert all 'utils_dirs' directories to the system path + insert_system_path(opts, opts['utils_dirs']) + if (overrides or {}).get('ipc_write_buffer', '') == 'dynamic': opts['ipc_write_buffer'] = _DFLT_IPC_WBUFFER if 'ipc_write_buffer' not in overrides: @@ -3621,7 +3774,7 @@ def apply_master_config(overrides=None, defaults=None): if opts['file_ignore_regex']: # If file_ignore_regex was given, make sure it's wrapped in a list. # Only keep valid regex entries for improved performance later on. - if isinstance(opts['file_ignore_regex'], str): + if isinstance(opts['file_ignore_regex'], six.string_types): ignore_regex = [opts['file_ignore_regex']] elif isinstance(opts['file_ignore_regex'], list): ignore_regex = opts['file_ignore_regex'] @@ -3642,7 +3795,7 @@ def apply_master_config(overrides=None, defaults=None): if opts['file_ignore_glob']: # If file_ignore_glob was given, make sure it's wrapped in a list. - if isinstance(opts['file_ignore_glob'], str): + if isinstance(opts['file_ignore_glob'], six.string_types): opts['file_ignore_glob'] = [opts['file_ignore_glob']] # Let's make sure `worker_threads` does not drop below 3 which has proven @@ -3731,7 +3884,7 @@ def client_config(path, env_var='SALT_CLIENT_CONFIG', defaults=None): # Make sure token is still valid expire = opts.get('token_expire', 43200) if os.stat(opts['token_file']).st_mtime + expire > time.mktime(time.localtime()): - with salt.utils.fopen(opts['token_file']) as fp_: + with salt.utils.files.fopen(opts['token_file']) as fp_: opts['token'] = fp_.read().strip() # On some platforms, like OpenBSD, 0.0.0.0 won't catch a master running on localhost if opts['interface'] == '0.0.0.0': diff --git a/salt/config/schemas/esxcluster.py b/salt/config/schemas/esxcluster.py new file mode 100644 index 00000000000..f9eb70fb677 --- /dev/null +++ b/salt/config/schemas/esxcluster.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +''' + :codeauthor: :email:`Alexandru Bleotu (alexandru.bleotu@morganstanley.com)` + + + salt.config.schemas.esxcluster + ~~~~~~~~~~~~~~~~~~~~~~~ + + ESX Cluster configuration schemas +''' + +# Import Python libs +from __future__ import absolute_import + +# Import Salt libs +from salt.utils.schema import (Schema, + ArrayItem, + IntegerItem, + StringItem) + + +class EsxclusterProxySchema(Schema): + ''' + Schema of the esxcluster proxy input + ''' + + title = 'Esxcluster Proxy Schema' + description = 'Esxcluster proxy schema' + additional_properties = False + proxytype = StringItem(required=True, + enum=['esxcluster']) + vcenter = StringItem(required=True, pattern=r'[^\s]+') + datacenter = StringItem(required=True) + cluster = StringItem(required=True) + mechanism = StringItem(required=True, enum=['userpass', 'sspi']) + username = StringItem() + passwords = ArrayItem(min_items=1, + items=StringItem(), + unique_items=True) + # TODO Should be changed when anyOf is supported for schemas + domain = StringItem() + principal = StringItem() + protocol = StringItem() + port = IntegerItem(minimum=1) diff --git a/salt/config/schemas/esxdatacenter.py b/salt/config/schemas/esxdatacenter.py new file mode 100644 index 00000000000..eea4c3e833a --- /dev/null +++ b/salt/config/schemas/esxdatacenter.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +''' + :codeauthor: :email:`Alexandru Bleotu (alexandru.bleotu@morganstanley.com)` + + + salt.config.schemas.esxdatacenter + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + ESX Datacenter configuration schemas +''' + +# Import Python libs +from __future__ import absolute_import + +# Import Salt libs +from salt.utils.schema import (Schema, + ArrayItem, + IntegerItem, + StringItem) + + +class EsxdatacenterProxySchema(Schema): + ''' + Schema of the esxdatacenter proxy input + ''' + + title = 'Esxdatacenter Proxy Schema' + description = 'Esxdatacenter proxy schema' + additional_properties = False + proxytype = StringItem(required=True, + enum=['esxdatacenter']) + vcenter = StringItem(required=True, pattern=r'[^\s]+') + datacenter = StringItem(required=True) + mechanism = StringItem(required=True, enum=['userpass', 'sspi']) + username = StringItem() + passwords = ArrayItem(min_items=1, + items=StringItem(), + unique_items=True) + # TODO Should be changed when anyOf is supported for schemas + domain = StringItem() + principal = StringItem() + protocol = StringItem() + port = IntegerItem(minimum=1) diff --git a/salt/crypt.py b/salt/crypt.py index c51077409f9..f5f945a7da2 100644 --- a/salt/crypt.py +++ b/salt/crypt.py @@ -19,10 +19,11 @@ import traceback import binascii import weakref import getpass +import tornado.gen # Import third party libs -import salt.ext.six as six from salt.ext.six.moves import zip # pylint: disable=import-error,redefined-builtin +from salt.ext import six try: from Cryptodome.Cipher import AES, PKCS1_OAEP from Cryptodome.Hash import SHA @@ -46,20 +47,22 @@ if not CDOME: # Import salt libs import salt.defaults.exitcodes -import salt.utils -import salt.utils.decorators import salt.payload import salt.transport.client import salt.transport.frame +import salt.utils +import salt.utils.decorators +import salt.utils.event +import salt.utils.files import salt.utils.rsax931 +import salt.utils.sdb +import salt.utils.stringutils import salt.utils.verify import salt.version from salt.exceptions import ( - AuthenticationError, SaltClientError, SaltReqTimeoutError + AuthenticationError, SaltClientError, SaltReqTimeoutError, MasterExit ) -import tornado.gen - log = logging.getLogger(__name__) @@ -67,19 +70,19 @@ def dropfile(cachedir, user=None): ''' Set an AES dropfile to request the master update the publish session key ''' - dfn = os.path.join(cachedir, '.dfn') + dfn = os.path.join(cachedir, u'.dfn') # set a mask (to avoid a race condition on file creation) and store original. mask = os.umask(191) try: - log.info('Rotating AES key') + log.info(u'Rotating AES key') if os.path.isfile(dfn): - log.info('AES key rotation already requested') + log.info(u'AES key rotation already requested') return if os.path.isfile(dfn) and not os.access(dfn, os.W_OK): os.chmod(dfn, stat.S_IRUSR | stat.S_IWUSR) - with salt.utils.fopen(dfn, 'wb+') as fp_: - fp_.write(b'') + with salt.utils.files.fopen(dfn, u'wb+') as fp_: + fp_.write(b'') # future lint: disable=non-unicode-string os.chmod(dfn, stat.S_IRUSR) if user: try: @@ -92,7 +95,7 @@ def dropfile(cachedir, user=None): os.umask(mask) # restore original umask -def gen_keys(keydir, keyname, keysize, user=None): +def gen_keys(keydir, keyname, keysize, user=None, passphrase=None): ''' Generate a RSA public keypair for use with salt @@ -100,13 +103,14 @@ def gen_keys(keydir, keyname, keysize, user=None): :param str keyname: The type of salt server for whom this key should be written. (i.e. 'master' or 'minion') :param int keysize: The number of bits in the key :param str user: The user on the system who should own this keypair + :param str passphrase: The passphrase which should be used to encrypt the private key :rtype: str :return: Path on the filesystem to the RSA private key ''' base = os.path.join(keydir, keyname) - priv = '{0}.pem'.format(base) - pub = '{0}.pub'.format(base) + priv = u'{0}.pem'.format(base) + pub = u'{0}.pub'.format(base) salt.utils.reinit_crypto() gen = RSA.generate(bits=keysize, e=65537) @@ -117,14 +121,14 @@ def gen_keys(keydir, keyname, keysize, user=None): # Do not try writing anything, if directory has no permissions. if not os.access(keydir, os.W_OK): - raise IOError('Write access denied to "{0}" for user "{1}".'.format(os.path.abspath(keydir), getpass.getuser())) + raise IOError(u'Write access denied to "{0}" for user "{1}".'.format(os.path.abspath(keydir), getpass.getuser())) cumask = os.umask(191) - with salt.utils.fopen(priv, 'wb+') as f: - f.write(gen.exportKey('PEM')) + with salt.utils.files.fopen(priv, u'wb+') as f: + f.write(gen.exportKey(u'PEM', passphrase)) os.umask(cumask) - with salt.utils.fopen(pub, 'wb+') as f: - f.write(gen.publickey().exportKey('PEM')) + with salt.utils.files.fopen(pub, u'wb+') as f: + f.write(gen.publickey().exportKey(u'PEM')) os.chmod(priv, 256) if user: try: @@ -140,7 +144,7 @@ def gen_keys(keydir, keyname, keysize, user=None): @salt.utils.decorators.memoize -def _get_key_with_evict(path, timestamp): +def _get_key_with_evict(path, timestamp, passphrase): ''' Load a key from disk. `timestamp` above is intended to be the timestamp of the file's last modification. This fn is memoized so if it is called with the @@ -148,13 +152,13 @@ def _get_key_with_evict(path, timestamp): the result is returned from the memoiziation. If the file gets modified then the params are different and the key is loaded from disk. ''' - log.debug('salt.crypt._get_key_with_evict: Loading private key') - with salt.utils.fopen(path) as f: - key = RSA.importKey(f.read()) + log.debug(u'salt.crypt._get_key_with_evict: Loading private key') + with salt.utils.files.fopen(path) as f: + key = RSA.importKey(f.read(), passphrase) return key -def _get_rsa_key(path): +def _get_rsa_key(path, passphrase): ''' Read a key off the disk. Poor man's simple cache in effect here, we memoize the result of calling _get_rsa_with_evict. This means @@ -165,16 +169,16 @@ def _get_rsa_key(path): is called it is called with different parameters and the fn is run fully to retrieve the key from disk. ''' - log.debug('salt.crypt._get_rsa_key: Loading private key') - return _get_key_with_evict(path, str(os.path.getmtime(path))) + log.debug(u'salt.crypt._get_rsa_key: Loading private key') + return _get_key_with_evict(path, str(os.path.getmtime(path)), passphrase) -def sign_message(privkey_path, message): +def sign_message(privkey_path, message, passphrase=None): ''' Use Crypto.Signature.PKCS1_v1_5 to sign a message. Returns the signature. ''' - key = _get_rsa_key(privkey_path) - log.debug('salt.crypt.sign_message: Signing message.') + key = _get_rsa_key(privkey_path, passphrase) + log.debug(u'salt.crypt.sign_message: Signing message.') signer = PKCS1_v1_5.new(key) return signer.sign(SHA.new(message)) @@ -184,38 +188,41 @@ def verify_signature(pubkey_path, message, signature): Use Crypto.Signature.PKCS1_v1_5 to verify the signature on a message. Returns True for valid signature. ''' - log.debug('salt.crypt.verify_signature: Loading public key') - with salt.utils.fopen(pubkey_path) as f: + log.debug(u'salt.crypt.verify_signature: Loading public key') + with salt.utils.files.fopen(pubkey_path) as f: pubkey = RSA.importKey(f.read()) - log.debug('salt.crypt.verify_signature: Verifying signature') + log.debug(u'salt.crypt.verify_signature: Verifying signature') verifier = PKCS1_v1_5.new(pubkey) return verifier.verify(SHA.new(message), signature) -def gen_signature(priv_path, pub_path, sign_path): +def gen_signature(priv_path, pub_path, sign_path, passphrase=None): ''' creates a signature for the given public-key with the given private key and writes it to sign_path ''' - with salt.utils.fopen(pub_path) as fp_: + with salt.utils.files.fopen(pub_path) as fp_: mpub_64 = fp_.read() - mpub_sig = sign_message(priv_path, mpub_64) + mpub_sig = sign_message(priv_path, mpub_64, passphrase) mpub_sig_64 = binascii.b2a_base64(mpub_sig) if os.path.isfile(sign_path): return False - log.trace('Calculating signature for {0} with {1}' - .format(os.path.basename(pub_path), - os.path.basename(priv_path))) + log.trace( + u'Calculating signature for %s with %s', + os.path.basename(pub_path), os.path.basename(priv_path) + ) if os.path.isfile(sign_path): - log.trace('Signature file {0} already exists, please ' - 'remove it first and try again'.format(sign_path)) + log.trace( + u'Signature file %s already exists, please remove it first and ' + u'try again', sign_path + ) else: - with salt.utils.fopen(sign_path, 'wb+') as sig_f: - sig_f.write(salt.utils.to_bytes(mpub_sig_64)) - log.trace('Wrote signature to {0}'.format(sign_path)) + with salt.utils.files.fopen(sign_path, u'wb+') as sig_f: + sig_f.write(salt.utils.stringutils.to_bytes(mpub_sig_64)) + log.trace(u'Wrote signature to %s', sign_path) return True @@ -228,7 +235,7 @@ def private_encrypt(key, message): :rtype: str :return: The signature, or an empty string if the signature operation failed ''' - signer = salt.utils.rsax931.RSAX931Signer(key.exportKey('PEM')) + signer = salt.utils.rsax931.RSAX931Signer(key.exportKey(u'PEM')) return signer.sign(message) @@ -242,7 +249,7 @@ def public_decrypt(pub, message): :return: The message (or digest) recovered from the signature, or an empty string if the verification failed ''' - verifier = salt.utils.rsax931.RSAX931Verifier(pub.exportKey('PEM')) + verifier = salt.utils.rsax931.RSAX931Verifier(pub.exportKey(u'PEM')) return verifier.verify(message) @@ -256,41 +263,50 @@ class MasterKeys(dict): def __init__(self, opts): super(MasterKeys, self).__init__() self.opts = opts - self.pub_path = os.path.join(self.opts['pki_dir'], 'master.pub') - self.rsa_path = os.path.join(self.opts['pki_dir'], 'master.pem') + self.pub_path = os.path.join(self.opts[u'pki_dir'], u'master.pub') + self.rsa_path = os.path.join(self.opts[u'pki_dir'], u'master.pem') + + key_pass = salt.utils.sdb.sdb_get(self.opts['key_pass'], self.opts) + self.key = self.__get_keys(passphrase=key_pass) - self.key = self.__get_keys() self.pub_signature = None # set names for the signing key-pairs - if opts['master_sign_pubkey']: + if opts[u'master_sign_pubkey']: # if only the signature is available, use that - if opts['master_use_pubkey_signature']: - self.sig_path = os.path.join(self.opts['pki_dir'], - opts['master_pubkey_signature']) + if opts[u'master_use_pubkey_signature']: + self.sig_path = os.path.join(self.opts[u'pki_dir'], + opts[u'master_pubkey_signature']) if os.path.isfile(self.sig_path): - with salt.utils.fopen(self.sig_path) as fp_: + with salt.utils.files.fopen(self.sig_path) as fp_: self.pub_signature = fp_.read() - log.info('Read {0}\'s signature from {1}' - ''.format(os.path.basename(self.pub_path), - self.opts['master_pubkey_signature'])) + log.info( + u'Read %s\'s signature from %s', + os.path.basename(self.pub_path), + self.opts[u'master_pubkey_signature'] + ) else: - log.error('Signing the master.pub key with a signature is enabled ' - 'but no signature file found at the defined location ' - '{0}'.format(self.sig_path)) - log.error('The signature-file may be either named differently ' - 'or has to be created with \'salt-key --gen-signature\'') + log.error( + u'Signing the master.pub key with a signature is ' + u'enabled but no signature file found at the defined ' + u'location %s', self.sig_path + ) + log.error( + u'The signature-file may be either named differently ' + u'or has to be created with \'salt-key --gen-signature\'' + ) sys.exit(1) # create a new signing key-pair to sign the masters # auth-replies when a minion tries to connect else: - self.pub_sign_path = os.path.join(self.opts['pki_dir'], - opts['master_sign_key_name'] + '.pub') - self.rsa_sign_path = os.path.join(self.opts['pki_dir'], - opts['master_sign_key_name'] + '.pem') - self.sign_key = self.__get_keys(name=opts['master_sign_key_name']) + key_pass = salt.utils.sdb.sdb_get(self.opts[u'signing_key_pass'], self.opts) + self.pub_sign_path = os.path.join(self.opts[u'pki_dir'], + opts[u'master_sign_key_name'] + u'.pub') + self.rsa_sign_path = os.path.join(self.opts[u'pki_dir'], + opts[u'master_sign_key_name'] + u'.pem') + self.sign_key = self.__get_keys(name=opts[u'master_sign_key_name']) # We need __setstate__ and __getstate__ to avoid pickling errors since # some of the member variables correspond to Cython objects which are @@ -298,43 +314,49 @@ class MasterKeys(dict): # These methods are only used when pickling so will not be used on # non-Windows platforms. def __setstate__(self, state): - self.__init__(state['opts']) + self.__init__(state[u'opts']) def __getstate__(self): - return {'opts': self.opts} + return {u'opts': self.opts} - def __get_keys(self, name='master'): + def __get_keys(self, name=u'master', passphrase=None): ''' Returns a key object for a key in the pki-dir ''' - path = os.path.join(self.opts['pki_dir'], - name + '.pem') + path = os.path.join(self.opts[u'pki_dir'], + name + u'.pem') if os.path.exists(path): - with salt.utils.fopen(path) as f: - key = RSA.importKey(f.read()) - log.debug('Loaded {0} key: {1}'.format(name, path)) + with salt.utils.files.fopen(path) as f: + try: + key = RSA.importKey(f.read(), passphrase) + except ValueError as e: + message = u'Unable to read key: {0}; passphrase may be incorrect'.format(path) + log.error(message) + raise MasterExit(message) + log.debug(u'Loaded %s key: %s', name, path) else: - log.info('Generating {0} keys: {1}'.format(name, self.opts['pki_dir'])) - gen_keys(self.opts['pki_dir'], + log.info(u'Generating %s keys: %s', name, self.opts[u'pki_dir']) + gen_keys(self.opts[u'pki_dir'], name, - self.opts['keysize'], - self.opts.get('user')) - with salt.utils.fopen(self.rsa_path) as f: - key = RSA.importKey(f.read()) + self.opts[u'keysize'], + self.opts.get(u'user'), + passphrase) + with salt.utils.files.fopen(self.rsa_path) as f: + key = RSA.importKey(f.read(), passphrase) return key - def get_pub_str(self, name='master'): + def get_pub_str(self, name=u'master'): ''' Return the string representation of a public key in the pki-directory ''' - path = os.path.join(self.opts['pki_dir'], - name + '.pub') + path = os.path.join(self.opts[u'pki_dir'], + name + u'.pub') if not os.path.isfile(path): key = self.__get_keys() - with salt.utils.fopen(path, 'wb+') as wfh: - wfh.write(key.publickey().exportKey('PEM')) - with salt.utils.fopen(path) as rfh: + with salt.utils.files.fopen(path, u'wb+') as wfh: + wfh.write(key.publickey().exportKey(u'PEM')) + with salt.utils.files.fopen(path) as rfh: return rfh.read() def get_mkey_paths(self): @@ -373,23 +395,24 @@ class AsyncAuth(object): loop_instance_map = AsyncAuth.instance_map[io_loop] key = cls.__key(opts) - if key not in loop_instance_map: - log.debug('Initializing new AsyncAuth for {0}'.format(key)) + auth = loop_instance_map.get(key) + if auth is None: + log.debug(u'Initializing new AsyncAuth for %s', key) # we need to make a local variable for this, as we are going to store # it in a WeakValueDictionary-- which will remove the item if no one # references it-- this forces a reference while we return to the caller - new_auth = object.__new__(cls) - new_auth.__singleton_init__(opts, io_loop=io_loop) - loop_instance_map[key] = new_auth + auth = object.__new__(cls) + auth.__singleton_init__(opts, io_loop=io_loop) + loop_instance_map[key] = auth else: - log.debug('Re-using AsyncAuth for {0}'.format(key)) - return loop_instance_map[key] + log.debug(u'Re-using AsyncAuth for %s', key) + return auth @classmethod def __key(cls, opts, io_loop=None): - return (opts['pki_dir'], # where the keys are stored - opts['id'], # minion ID - opts['master_uri'], # master ID + return (opts[u'pki_dir'], # where the keys are stored + opts[u'id'], # minion ID + opts[u'master_uri'], # master ID ) # has to remain empty for singletons, since __init__ will *always* be called @@ -409,14 +432,14 @@ class AsyncAuth(object): if six.PY2: self.token = Crypticle.generate_key_string() else: - self.token = salt.utils.to_bytes(Crypticle.generate_key_string()) + self.token = salt.utils.stringutils.to_bytes(Crypticle.generate_key_string()) self.serial = salt.payload.Serial(self.opts) - self.pub_path = os.path.join(self.opts['pki_dir'], 'minion.pub') - self.rsa_path = os.path.join(self.opts['pki_dir'], 'minion.pem') - if self.opts['__role'] == 'syndic': - self.mpub = 'syndic_master.pub' + self.pub_path = os.path.join(self.opts[u'pki_dir'], u'minion.pub') + self.rsa_path = os.path.join(self.opts[u'pki_dir'], u'minion.pem') + if self.opts[u'__role'] == u'syndic': + self.mpub = u'syndic_master.pub' else: - self.mpub = 'minion_master.pub' + self.mpub = u'minion_master.pub' if not os.path.isfile(self.pub_path): self.get_keys() @@ -428,7 +451,7 @@ class AsyncAuth(object): if key in AsyncAuth.creds_map: creds = AsyncAuth.creds_map[key] self._creds = creds - self._crypticle = Crypticle(self.opts, creds['aes']) + self._crypticle = Crypticle(self.opts, creds[u'aes']) self._authenticate_future = tornado.concurrent.Future() self._authenticate_future.set_result(True) else: @@ -439,7 +462,7 @@ class AsyncAuth(object): result = cls.__new__(cls, copy.deepcopy(self.opts, memo), io_loop=None) memo[id(self)] = result for key in self.__dict__: - if key in ('io_loop',): + if key in (u'io_loop',): # The io_loop has a thread Lock which will fail to be deep # copied. Skip it because it will just be recreated on the # new copy. @@ -457,7 +480,7 @@ class AsyncAuth(object): @property def authenticated(self): - return hasattr(self, '_authenticate_future') and \ + return hasattr(self, u'_authenticate_future') and \ self._authenticate_future.done() and \ self._authenticate_future.exception() is None @@ -476,7 +499,7 @@ class AsyncAuth(object): for the sign-in-- whis way callers can all assume there aren't others ''' # if an auth is in flight-- and not done-- just pass that back as the future to wait on - if hasattr(self, '_authenticate_future') and not self._authenticate_future.done(): + if hasattr(self, u'_authenticate_future') and not self._authenticate_future.done(): future = self._authenticate_future else: future = tornado.concurrent.Future() @@ -502,13 +525,13 @@ class AsyncAuth(object): :rtype: Crypticle :returns: A crypticle used for encryption operations ''' - acceptance_wait_time = self.opts['acceptance_wait_time'] - acceptance_wait_time_max = self.opts['acceptance_wait_time_max'] + acceptance_wait_time = self.opts[u'acceptance_wait_time'] + acceptance_wait_time_max = self.opts[u'acceptance_wait_time_max'] if not acceptance_wait_time_max: acceptance_wait_time_max = acceptance_wait_time creds = None channel = salt.transport.client.AsyncReqChannel.factory(self.opts, - crypt='clear', + crypt=u'clear', io_loop=self.io_loop) error = None while True: @@ -517,41 +540,45 @@ class AsyncAuth(object): except SaltClientError as exc: error = exc break - if creds == 'retry': - if self.opts.get('detect_mode') is True: - error = SaltClientError('Detect mode is on') + if creds == u'retry': + if self.opts.get(u'detect_mode') is True: + error = SaltClientError(u'Detect mode is on') break - if self.opts.get('caller'): - print('Minion failed to authenticate with the master, ' - 'has the minion key been accepted?') + if self.opts.get(u'caller'): + print(u'Minion failed to authenticate with the master, ' + u'has the minion key been accepted?') sys.exit(2) if acceptance_wait_time: - log.info('Waiting {0} seconds before retry.'.format(acceptance_wait_time)) + log.info( + u'Waiting %s seconds before retry.', acceptance_wait_time + ) yield tornado.gen.sleep(acceptance_wait_time) if acceptance_wait_time < acceptance_wait_time_max: acceptance_wait_time += acceptance_wait_time - log.debug('Authentication wait time is {0}'.format(acceptance_wait_time)) + log.debug( + u'Authentication wait time is %s', acceptance_wait_time + ) continue break - if not isinstance(creds, dict) or 'aes' not in creds: - if self.opts.get('detect_mode') is True: - error = SaltClientError('-|RETRY|-') + if not isinstance(creds, dict) or u'aes' not in creds: + if self.opts.get(u'detect_mode') is True: + error = SaltClientError(u'-|RETRY|-') try: del AsyncAuth.creds_map[self.__key(self.opts)] except KeyError: pass if not error: - error = SaltClientError('Attempt to authenticate with the salt master failed') + error = SaltClientError(u'Attempt to authenticate with the salt master failed') self._authenticate_future.set_exception(error) else: key = self.__key(self.opts) AsyncAuth.creds_map[key] = creds self._creds = creds - self._crypticle = Crypticle(self.opts, creds['aes']) + self._crypticle = Crypticle(self.opts, creds[u'aes']) self._authenticate_future.set_result(True) # mark the sign-in as complete # Notify the bus about creds change - event = salt.utils.event.get_event(self.opts.get('__role'), opts=self.opts, listen=False) - event.fire_event({'key': key, 'creds': creds}, salt.utils.event.tagify(prefix='auth', suffix='creds')) + event = salt.utils.event.get_event(self.opts.get(u'__role'), opts=self.opts, listen=False) + event.fire_event({u'key': key, u'creds': creds}, salt.utils.event.tagify(prefix=u'auth', suffix=u'creds')) @tornado.gen.coroutine def sign_in(self, timeout=60, safe=True, tries=1, channel=None): @@ -572,23 +599,23 @@ class AsyncAuth(object): ''' auth = {} - auth_timeout = self.opts.get('auth_timeout', None) + auth_timeout = self.opts.get(u'auth_timeout', None) if auth_timeout is not None: timeout = auth_timeout - auth_safemode = self.opts.get('auth_safemode', None) + auth_safemode = self.opts.get(u'auth_safemode', None) if auth_safemode is not None: safe = auth_safemode - auth_tries = self.opts.get('auth_tries', None) + auth_tries = self.opts.get(u'auth_tries', None) if auth_tries is not None: tries = auth_tries - m_pub_fn = os.path.join(self.opts['pki_dir'], self.mpub) + m_pub_fn = os.path.join(self.opts[u'pki_dir'], self.mpub) - auth['master_uri'] = self.opts['master_uri'] + auth[u'master_uri'] = self.opts[u'master_uri'] if not channel: channel = salt.transport.client.AsyncReqChannel.factory(self.opts, - crypt='clear', + crypt=u'clear', io_loop=self.io_loop) sign_in_payload = self.minion_sign_in_payload() @@ -600,66 +627,65 @@ class AsyncAuth(object): ) except SaltReqTimeoutError as e: if safe: - log.warning('SaltReqTimeoutError: {0}'.format(e)) - raise tornado.gen.Return('retry') - if self.opts.get('detect_mode') is True: - raise tornado.gen.Return('retry') + log.warning(u'SaltReqTimeoutError: %s', e) + raise tornado.gen.Return(u'retry') + if self.opts.get(u'detect_mode') is True: + raise tornado.gen.Return(u'retry') else: - raise SaltClientError('Attempt to authenticate with the salt master failed with timeout error') - if 'load' in payload: - if 'ret' in payload['load']: - if not payload['load']['ret']: - if self.opts['rejected_retry']: + raise SaltClientError(u'Attempt to authenticate with the salt master failed with timeout error') + if u'load' in payload: + if u'ret' in payload[u'load']: + if not payload[u'load'][u'ret']: + if self.opts[u'rejected_retry']: log.error( - 'The Salt Master has rejected this minion\'s public ' - 'key.\nTo repair this issue, delete the public key ' - 'for this minion on the Salt Master.\nThe Salt ' - 'Minion will attempt to to re-authenicate.' + u'The Salt Master has rejected this minion\'s public ' + u'key.\nTo repair this issue, delete the public key ' + u'for this minion on the Salt Master.\nThe Salt ' + u'Minion will attempt to to re-authenicate.' ) - raise tornado.gen.Return('retry') + raise tornado.gen.Return(u'retry') else: log.critical( - 'The Salt Master has rejected this minion\'s public ' - 'key!\nTo repair this issue, delete the public key ' - 'for this minion on the Salt Master and restart this ' - 'minion.\nOr restart the Salt Master in open mode to ' - 'clean out the keys. The Salt Minion will now exit.' + u'The Salt Master has rejected this minion\'s public ' + u'key!\nTo repair this issue, delete the public key ' + u'for this minion on the Salt Master and restart this ' + u'minion.\nOr restart the Salt Master in open mode to ' + u'clean out the keys. The Salt Minion will now exit.' ) sys.exit(salt.defaults.exitcodes.EX_NOPERM) # has the master returned that its maxed out with minions? - elif payload['load']['ret'] == 'full': - raise tornado.gen.Return('full') + elif payload[u'load'][u'ret'] == u'full': + raise tornado.gen.Return(u'full') else: log.error( - 'The Salt Master has cached the public key for this ' - 'node, this salt minion will wait for {0} seconds ' - 'before attempting to re-authenticate'.format( - self.opts['acceptance_wait_time'] - ) + u'The Salt Master has cached the public key for this ' + u'node, this salt minion will wait for %s seconds ' + u'before attempting to re-authenticate', + self.opts[u'acceptance_wait_time'] ) - raise tornado.gen.Return('retry') - auth['aes'] = self.verify_master(payload, master_pub='token' in sign_in_payload) - if not auth['aes']: + raise tornado.gen.Return(u'retry') + auth[u'aes'] = self.verify_master(payload, master_pub=u'token' in sign_in_payload) + if not auth[u'aes']: log.critical( - 'The Salt Master server\'s public key did not authenticate!\n' - 'The master may need to be updated if it is a version of Salt ' - 'lower than {0}, or\n' - 'If you are confident that you are connecting to a valid Salt ' - 'Master, then remove the master public key and restart the ' - 'Salt Minion.\nThe master public key can be found ' - 'at:\n{1}'.format(salt.version.__version__, m_pub_fn) + u'The Salt Master server\'s public key did not authenticate!\n' + u'The master may need to be updated if it is a version of Salt ' + u'lower than %s, or\n' + u'If you are confident that you are connecting to a valid Salt ' + u'Master, then remove the master public key and restart the ' + u'Salt Minion.\nThe master public key can be found ' + u'at:\n%s', salt.version.__version__, m_pub_fn ) - raise SaltClientError('Invalid master key') - if self.opts.get('syndic_master', False): # Is syndic - syndic_finger = self.opts.get('syndic_finger', self.opts.get('master_finger', False)) + raise SaltClientError(u'Invalid master key') + if self.opts.get(u'syndic_master', False): # Is syndic + syndic_finger = self.opts.get(u'syndic_finger', self.opts.get(u'master_finger', False)) if syndic_finger: - if salt.utils.pem_finger(m_pub_fn, sum_type=self.opts['hash_type']) != syndic_finger: + if salt.utils.pem_finger(m_pub_fn, sum_type=self.opts[u'hash_type']) != syndic_finger: self._finger_fail(syndic_finger, m_pub_fn) else: - if self.opts.get('master_finger', False): - if salt.utils.pem_finger(m_pub_fn, sum_type=self.opts['hash_type']) != self.opts['master_finger']: - self._finger_fail(self.opts['master_finger'], m_pub_fn) - auth['publish_port'] = payload['publish_port'] + if self.opts.get(u'master_finger', False): + if salt.utils.pem_finger(m_pub_fn, sum_type=self.opts[u'hash_type']) != self.opts[u'master_finger']: + self._finger_fail(self.opts[u'master_finger'], m_pub_fn) + auth[u'publish_port'] = payload[u'publish_port'] raise tornado.gen.Return(auth) def get_keys(self): @@ -670,20 +696,20 @@ class AsyncAuth(object): :return: The RSA keypair ''' # Make sure all key parent directories are accessible - user = self.opts.get('user', 'root') - salt.utils.verify.check_path_traversal(self.opts['pki_dir'], user) + user = self.opts.get(u'user', u'root') + salt.utils.verify.check_path_traversal(self.opts[u'pki_dir'], user) if os.path.exists(self.rsa_path): - with salt.utils.fopen(self.rsa_path) as f: + with salt.utils.files.fopen(self.rsa_path) as f: key = RSA.importKey(f.read()) - log.debug('Loaded minion key: {0}'.format(self.rsa_path)) + log.debug(u'Loaded minion key: %s', self.rsa_path) else: - log.info('Generating keys: {0}'.format(self.opts['pki_dir'])) - gen_keys(self.opts['pki_dir'], - 'minion', - self.opts['keysize'], - self.opts.get('user')) - with salt.utils.fopen(self.rsa_path) as f: + log.info(u'Generating keys: %s', self.opts[u'pki_dir']) + gen_keys(self.opts[u'pki_dir'], + u'minion', + self.opts[u'keysize'], + self.opts.get(u'user')) + with salt.utils.files.fopen(self.rsa_path) as f: key = RSA.importKey(f.read()) return key @@ -708,18 +734,18 @@ class AsyncAuth(object): :rtype: dict ''' payload = {} - payload['cmd'] = '_auth' - payload['id'] = self.opts['id'] + payload[u'cmd'] = u'_auth' + payload[u'id'] = self.opts[u'id'] try: - pubkey_path = os.path.join(self.opts['pki_dir'], self.mpub) - with salt.utils.fopen(pubkey_path) as f: + pubkey_path = os.path.join(self.opts[u'pki_dir'], self.mpub) + with salt.utils.files.fopen(pubkey_path) as f: pub = RSA.importKey(f.read()) cipher = PKCS1_OAEP.new(pub) - payload['token'] = cipher.encrypt(self.token) + payload[u'token'] = cipher.encrypt(self.token) except Exception: pass - with salt.utils.fopen(self.pub_path) as f: - payload['pub'] = f.read() + with salt.utils.files.fopen(self.pub_path) as f: + payload[u'pub'] = f.read() return payload def decrypt_aes(self, payload, master_pub=True): @@ -745,46 +771,42 @@ class AsyncAuth(object): :rtype: str :return: The decrypted AES seed key ''' - if self.opts.get('auth_trb', False): - log.warning( - 'Auth Called: {0}'.format( - ''.join(traceback.format_stack()) - ) - ) + if self.opts.get(u'auth_trb', False): + log.warning(u'Auth Called: %s', u''.join(traceback.format_stack())) else: - log.debug('Decrypting the current master AES key') + log.debug(u'Decrypting the current master AES key') key = self.get_keys() cipher = PKCS1_OAEP.new(key) - key_str = cipher.decrypt(payload['aes']) - if 'sig' in payload: - m_path = os.path.join(self.opts['pki_dir'], self.mpub) + key_str = cipher.decrypt(payload[u'aes']) + if u'sig' in payload: + m_path = os.path.join(self.opts[u'pki_dir'], self.mpub) if os.path.exists(m_path): try: - with salt.utils.fopen(m_path) as f: + with salt.utils.files.fopen(m_path) as f: mkey = RSA.importKey(f.read()) except Exception: - return '', '' + return u'', u'' digest = hashlib.sha256(key_str).hexdigest() if six.PY3: - digest = salt.utils.to_bytes(digest) - m_digest = public_decrypt(mkey.publickey(), payload['sig']) + digest = salt.utils.stringutils.to_bytes(digest) + m_digest = public_decrypt(mkey.publickey(), payload[u'sig']) if m_digest != digest: - return '', '' + return u'', u'' else: - return '', '' + return u'', u'' if six.PY3: - key_str = salt.utils.to_str(key_str) + key_str = salt.utils.stringutils.to_str(key_str) - if '_|-' in key_str: - return key_str.split('_|-') + if u'_|-' in key_str: + return key_str.split(u'_|-') else: - if 'token' in payload: - token = cipher.decrypt(payload['token']) + if u'token' in payload: + token = cipher.decrypt(payload[u'token']) return key_str, token elif not master_pub: - return key_str, '' - return '', '' + return key_str, u'' + return u'', u'' def verify_pubkey_sig(self, message, sig): ''' @@ -794,50 +816,63 @@ class AsyncAuth(object): :rtype: bool :return: Success or failure of public key verification ''' - if self.opts['master_sign_key_name']: - path = os.path.join(self.opts['pki_dir'], - self.opts['master_sign_key_name'] + '.pub') + if self.opts[u'master_sign_key_name']: + path = os.path.join(self.opts[u'pki_dir'], + self.opts[u'master_sign_key_name'] + u'.pub') if os.path.isfile(path): res = verify_signature(path, message, binascii.a2b_base64(sig)) else: - log.error('Verification public key {0} does not exist. You ' - 'need to copy it from the master to the minions ' - 'pki directory'.format(os.path.basename(path))) + log.error( + u'Verification public key %s does not exist. You need to ' + u'copy it from the master to the minions pki directory', + os.path.basename(path) + ) return False if res: - log.debug('Successfully verified signature of master ' - 'public key with verification public key ' - '{0}'.format(self.opts['master_sign_key_name'] + '.pub')) + log.debug( + u'Successfully verified signature of master public key ' + u'with verification public key %s', + self.opts[u'master_sign_key_name'] + u'.pub' + ) return True else: - log.debug('Failed to verify signature of public key') + log.debug(u'Failed to verify signature of public key') return False else: - log.error('Failed to verify the signature of the message because ' - 'the verification key-pairs name is not defined. Please ' - 'make sure that master_sign_key_name is defined.') + log.error( + u'Failed to verify the signature of the message because the ' + u'verification key-pairs name is not defined. Please make ' + u'sure that master_sign_key_name is defined.' + ) return False def verify_signing_master(self, payload): try: - if self.verify_pubkey_sig(payload['pub_key'], - payload['pub_sig']): - log.info('Received signed and verified master pubkey ' - 'from master {0}'.format(self.opts['master'])) - m_pub_fn = os.path.join(self.opts['pki_dir'], self.mpub) - uid = salt.utils.get_uid(self.opts.get('user', None)) - with salt.utils.fpopen(m_pub_fn, 'wb+', uid=uid) as wfh: - wfh.write(salt.utils.to_bytes(payload['pub_key'])) + if self.verify_pubkey_sig(payload[u'pub_key'], + payload[u'pub_sig']): + log.info( + u'Received signed and verified master pubkey from master %s', + self.opts[u'master'] + ) + m_pub_fn = os.path.join(self.opts[u'pki_dir'], self.mpub) + uid = salt.utils.get_uid(self.opts.get(u'user', None)) + with salt.utils.files.fpopen(m_pub_fn, u'wb+', uid=uid) as wfh: + wfh.write(salt.utils.stringutils.to_bytes(payload[u'pub_key'])) return True else: - log.error('Received signed public-key from master {0} ' - 'but signature verification failed!'.format(self.opts['master'])) + log.error( + u'Received signed public-key from master %s but signature ' + u'verification failed!', self.opts[u'master'] + ) return False except Exception as sign_exc: - log.error('There was an error while verifying the masters public-key signature') + log.error( + u'There was an error while verifying the masters public-key ' + u'signature' + ) raise Exception(sign_exc) def check_auth_deps(self, payload): @@ -853,25 +888,25 @@ class AsyncAuth(object): 'pub_key': The RSA public key of the sender. ''' # master and minion sign and verify - if 'pub_sig' in payload and self.opts['verify_master_pubkey_sign']: + if u'pub_sig' in payload and self.opts[u'verify_master_pubkey_sign']: return True # master and minion do NOT sign and do NOT verify - elif 'pub_sig' not in payload and not self.opts['verify_master_pubkey_sign']: + elif u'pub_sig' not in payload and not self.opts[u'verify_master_pubkey_sign']: return True # master signs, but minion does NOT verify - elif 'pub_sig' in payload and not self.opts['verify_master_pubkey_sign']: - log.error('The masters sent its public-key signature, but signature ' - 'verification is not enabled on the minion. Either enable ' - 'signature verification on the minion or disable signing ' - 'the public key on the master!') + elif u'pub_sig' in payload and not self.opts[u'verify_master_pubkey_sign']: + log.error(u'The masters sent its public-key signature, but signature ' + u'verification is not enabled on the minion. Either enable ' + u'signature verification on the minion or disable signing ' + u'the public key on the master!') return False # master does NOT sign but minion wants to verify - elif 'pub_sig' not in payload and self.opts['verify_master_pubkey_sign']: - log.error('The master did not send its public-key signature, but ' - 'signature verification is enabled on the minion. Either ' - 'disable signature verification on the minion or enable ' - 'signing the public on the master!') + elif u'pub_sig' not in payload and self.opts[u'verify_master_pubkey_sign']: + log.error(u'The master did not send its public-key signature, but ' + u'signature verification is enabled on the minion. Either ' + u'disable signature verification on the minion or enable ' + u'signing the public on the master!') return False def extract_aes(self, payload, master_pub=True): @@ -894,14 +929,14 @@ class AsyncAuth(object): aes, token = self.decrypt_aes(payload, master_pub) if token != self.token: log.error( - 'The master failed to decrypt the random minion token' + u'The master failed to decrypt the random minion token' ) - return '' + return u'' except Exception: log.error( - 'The master failed to decrypt the random minion token' + u'The master failed to decrypt the random minion token' ) - return '' + return u'' return aes else: aes, token = self.decrypt_aes(payload, master_pub) @@ -923,75 +958,77 @@ class AsyncAuth(object): :rtype: str :return: An empty string on verification failure. On success, the decrypted AES message in the payload. ''' - m_pub_fn = os.path.join(self.opts['pki_dir'], self.mpub) + m_pub_fn = os.path.join(self.opts[u'pki_dir'], self.mpub) m_pub_exists = os.path.isfile(m_pub_fn) - if m_pub_exists and master_pub and not self.opts['open_mode']: - with salt.utils.fopen(m_pub_fn) as fp_: + if m_pub_exists and master_pub and not self.opts[u'open_mode']: + with salt.utils.files.fopen(m_pub_fn) as fp_: local_master_pub = fp_.read() - if payload['pub_key'].replace('\n', '').replace('\r', '') != \ - local_master_pub.replace('\n', '').replace('\r', ''): + if payload[u'pub_key'].replace(u'\n', u'').replace(u'\r', u'') != \ + local_master_pub.replace(u'\n', u'').replace(u'\r', u''): if not self.check_auth_deps(payload): - return '' + return u'' - if self.opts['verify_master_pubkey_sign']: + if self.opts[u'verify_master_pubkey_sign']: if self.verify_signing_master(payload): return self.extract_aes(payload, master_pub=False) else: - return '' + return u'' else: # This is not the last master we connected to - log.error('The master key has changed, the salt master could ' - 'have been subverted, verify salt master\'s public ' - 'key') - return '' + log.error( + u'The master key has changed, the salt master could ' + u'have been subverted, verify salt master\'s public ' + u'key' + ) + return u'' else: if not self.check_auth_deps(payload): - return '' + return u'' # verify the signature of the pubkey even if it has # not changed compared with the one we already have - if self.opts['always_verify_signature']: + if self.opts[u'always_verify_signature']: if self.verify_signing_master(payload): return self.extract_aes(payload) else: - log.error('The masters public could not be verified. Is the ' - 'verification pubkey {0} up to date?' - ''.format(self.opts['master_sign_key_name'] + '.pub')) - return '' + log.error( + u'The masters public could not be verified. Is the ' + u'verification pubkey %s up to date?', + self.opts[u'master_sign_key_name'] + u'.pub' + ) + return u'' else: return self.extract_aes(payload) else: if not self.check_auth_deps(payload): - return '' + return u'' # verify the masters pubkey signature if the minion # has not received any masters pubkey before - if self.opts['verify_master_pubkey_sign']: + if self.opts[u'verify_master_pubkey_sign']: if self.verify_signing_master(payload): return self.extract_aes(payload, master_pub=False) else: - return '' + return u'' else: if not m_pub_exists: # the minion has not received any masters pubkey yet, write # the newly received pubkey to minion_master.pub - with salt.utils.fopen(m_pub_fn, 'wb+') as fp_: - fp_.write(salt.utils.to_bytes(payload['pub_key'])) + with salt.utils.files.fopen(m_pub_fn, u'wb+') as fp_: + fp_.write(salt.utils.stringutils.to_bytes(payload[u'pub_key'])) return self.extract_aes(payload, master_pub=False) def _finger_fail(self, finger, master_key): log.critical( - 'The specified fingerprint in the master configuration ' - 'file:\n{0}\nDoes not match the authenticating master\'s ' - 'key:\n{1}\nVerify that the configured fingerprint ' - 'matches the fingerprint of the correct master and that ' - 'this minion is not subject to a man-in-the-middle attack.' - .format( - finger, - salt.utils.pem_finger(master_key, sum_type=self.opts['hash_type']) - ) + u'The specified fingerprint in the master configuration ' + u'file:\n%s\nDoes not match the authenticating master\'s ' + u'key:\n%s\nVerify that the configured fingerprint ' + u'matches the fingerprint of the correct master and that ' + u'this minion is not subject to a man-in-the-middle attack.', + finger, + salt.utils.pem_finger(master_key, sum_type=self.opts[u'hash_type']) ) sys.exit(42) @@ -1009,20 +1046,21 @@ class SAuth(AsyncAuth): Only create one instance of SAuth per __key() ''' key = cls.__key(opts) - if key not in SAuth.instances: - log.debug('Initializing new SAuth for {0}'.format(key)) - new_auth = object.__new__(cls) - new_auth.__singleton_init__(opts) - SAuth.instances[key] = new_auth + auth = SAuth.instances.get(key) + if auth is None: + log.debug(u'Initializing new SAuth for %s', key) + auth = object.__new__(cls) + auth.__singleton_init__(opts) + SAuth.instances[key] = auth else: - log.debug('Re-using SAuth for {0}'.format(key)) - return SAuth.instances[key] + log.debug(u'Re-using SAuth for %s', key) + return auth @classmethod def __key(cls, opts, io_loop=None): - return (opts['pki_dir'], # where the keys are stored - opts['id'], # minion ID - opts['master_uri'], # master ID + return (opts[u'pki_dir'], # where the keys are stored + opts[u'id'], # minion ID + opts[u'master_uri'], # master ID ) # has to remain empty for singletons, since __init__ will *always* be called @@ -1042,28 +1080,28 @@ class SAuth(AsyncAuth): if six.PY2: self.token = Crypticle.generate_key_string() else: - self.token = salt.utils.to_bytes(Crypticle.generate_key_string()) + self.token = salt.utils.stringutils.to_bytes(Crypticle.generate_key_string()) self.serial = salt.payload.Serial(self.opts) - self.pub_path = os.path.join(self.opts['pki_dir'], 'minion.pub') - self.rsa_path = os.path.join(self.opts['pki_dir'], 'minion.pem') - if 'syndic_master' in self.opts: - self.mpub = 'syndic_master.pub' - elif 'alert_master' in self.opts: - self.mpub = 'monitor_master.pub' + self.pub_path = os.path.join(self.opts[u'pki_dir'], u'minion.pub') + self.rsa_path = os.path.join(self.opts[u'pki_dir'], u'minion.pem') + if u'syndic_master' in self.opts: + self.mpub = u'syndic_master.pub' + elif u'alert_master' in self.opts: + self.mpub = u'monitor_master.pub' else: - self.mpub = 'minion_master.pub' + self.mpub = u'minion_master.pub' if not os.path.isfile(self.pub_path): self.get_keys() @property def creds(self): - if not hasattr(self, '_creds'): + if not hasattr(self, u'_creds'): self.authenticate() return self._creds @property def crypticle(self): - if not hasattr(self, '_crypticle'): + if not hasattr(self, u'_crypticle'): self.authenticate() return self._crypticle @@ -1077,28 +1115,28 @@ class SAuth(AsyncAuth): :rtype: Crypticle :returns: A crypticle used for encryption operations ''' - acceptance_wait_time = self.opts['acceptance_wait_time'] - acceptance_wait_time_max = self.opts['acceptance_wait_time_max'] - channel = salt.transport.client.ReqChannel.factory(self.opts, crypt='clear') + acceptance_wait_time = self.opts[u'acceptance_wait_time'] + acceptance_wait_time_max = self.opts[u'acceptance_wait_time_max'] + channel = salt.transport.client.ReqChannel.factory(self.opts, crypt=u'clear') if not acceptance_wait_time_max: acceptance_wait_time_max = acceptance_wait_time while True: creds = self.sign_in(channel=channel) - if creds == 'retry': - if self.opts.get('caller'): - print('Minion failed to authenticate with the master, ' - 'has the minion key been accepted?') + if creds == u'retry': + if self.opts.get(u'caller'): + print(u'Minion failed to authenticate with the master, ' + u'has the minion key been accepted?') sys.exit(2) if acceptance_wait_time: - log.info('Waiting {0} seconds before retry.'.format(acceptance_wait_time)) + log.info(u'Waiting %s seconds before retry.', acceptance_wait_time) time.sleep(acceptance_wait_time) if acceptance_wait_time < acceptance_wait_time_max: acceptance_wait_time += acceptance_wait_time - log.debug('Authentication wait time is {0}'.format(acceptance_wait_time)) + log.debug(u'Authentication wait time is %s', acceptance_wait_time) continue break self._creds = creds - self._crypticle = Crypticle(self.opts, creds['aes']) + self._crypticle = Crypticle(self.opts, creds[u'aes']) def sign_in(self, timeout=60, safe=True, tries=1, channel=None): ''' @@ -1118,22 +1156,22 @@ class SAuth(AsyncAuth): ''' auth = {} - auth_timeout = self.opts.get('auth_timeout', None) + auth_timeout = self.opts.get(u'auth_timeout', None) if auth_timeout is not None: timeout = auth_timeout - auth_safemode = self.opts.get('auth_safemode', None) + auth_safemode = self.opts.get(u'auth_safemode', None) if auth_safemode is not None: safe = auth_safemode - auth_tries = self.opts.get('auth_tries', None) + auth_tries = self.opts.get(u'auth_tries', None) if auth_tries is not None: tries = auth_tries - m_pub_fn = os.path.join(self.opts['pki_dir'], self.mpub) + m_pub_fn = os.path.join(self.opts[u'pki_dir'], self.mpub) - auth['master_uri'] = self.opts['master_uri'] + auth[u'master_uri'] = self.opts[u'master_uri'] if not channel: - channel = salt.transport.client.ReqChannel.factory(self.opts, crypt='clear') + channel = salt.transport.client.ReqChannel.factory(self.opts, crypt=u'clear') sign_in_payload = self.minion_sign_in_payload() try: @@ -1144,67 +1182,66 @@ class SAuth(AsyncAuth): ) except SaltReqTimeoutError as e: if safe: - log.warning('SaltReqTimeoutError: {0}'.format(e)) - return 'retry' - raise SaltClientError('Attempt to authenticate with the salt master failed with timeout error') + log.warning(u'SaltReqTimeoutError: %s', e) + return u'retry' + raise SaltClientError(u'Attempt to authenticate with the salt master failed with timeout error') - if 'load' in payload: - if 'ret' in payload['load']: - if not payload['load']['ret']: - if self.opts['rejected_retry']: + if u'load' in payload: + if u'ret' in payload[u'load']: + if not payload[u'load'][u'ret']: + if self.opts[u'rejected_retry']: log.error( - 'The Salt Master has rejected this minion\'s public ' - 'key.\nTo repair this issue, delete the public key ' - 'for this minion on the Salt Master.\nThe Salt ' - 'Minion will attempt to to re-authenicate.' + u'The Salt Master has rejected this minion\'s public ' + u'key.\nTo repair this issue, delete the public key ' + u'for this minion on the Salt Master.\nThe Salt ' + u'Minion will attempt to to re-authenicate.' ) - return 'retry' + return u'retry' else: log.critical( - 'The Salt Master has rejected this minion\'s public ' - 'key!\nTo repair this issue, delete the public key ' - 'for this minion on the Salt Master and restart this ' - 'minion.\nOr restart the Salt Master in open mode to ' - 'clean out the keys. The Salt Minion will now exit.' + u'The Salt Master has rejected this minion\'s public ' + u'key!\nTo repair this issue, delete the public key ' + u'for this minion on the Salt Master and restart this ' + u'minion.\nOr restart the Salt Master in open mode to ' + u'clean out the keys. The Salt Minion will now exit.' ) sys.exit(salt.defaults.exitcodes.EX_NOPERM) # has the master returned that its maxed out with minions? - elif payload['load']['ret'] == 'full': - return 'full' + elif payload[u'load'][u'ret'] == u'full': + return u'full' else: log.error( - 'The Salt Master has cached the public key for this ' - 'node. If this is the first time connecting to this master ' - 'then this key may need to be accepted using \'salt-key -a {0}\' on ' - 'the salt master. This salt minion will wait for {1} seconds ' - 'before attempting to re-authenticate.'.format( - self.opts['id'], - self.opts['acceptance_wait_time'] - ) + u'The Salt Master has cached the public key for this ' + u'node. If this is the first time connecting to this ' + u'master then this key may need to be accepted using ' + u'\'salt-key -a %s\' on the salt master. This salt ' + u'minion will wait for %s seconds before attempting ' + u'to re-authenticate.', + self.opts[u'id'], self.opts[u'acceptance_wait_time'] ) - return 'retry' - auth['aes'] = self.verify_master(payload, master_pub='token' in sign_in_payload) - if not auth['aes']: + return u'retry' + auth[u'aes'] = self.verify_master(payload, master_pub=u'token' in sign_in_payload) + if not auth[u'aes']: log.critical( - 'The Salt Master server\'s public key did not authenticate!\n' - 'The master may need to be updated if it is a version of Salt ' - 'lower than {0}, or\n' - 'If you are confident that you are connecting to a valid Salt ' - 'Master, then remove the master public key and restart the ' - 'Salt Minion.\nThe master public key can be found ' - 'at:\n{1}'.format(salt.version.__version__, m_pub_fn) + u'The Salt Master server\'s public key did not authenticate!\n' + u'The master may need to be updated if it is a version of Salt ' + u'lower than %s, or\n' + u'If you are confident that you are connecting to a valid Salt ' + u'Master, then remove the master public key and restart the ' + u'Salt Minion.\nThe master public key can be found ' + u'at:\n%s', salt.version.__version__, m_pub_fn ) sys.exit(42) - if self.opts.get('syndic_master', False): # Is syndic - syndic_finger = self.opts.get('syndic_finger', self.opts.get('master_finger', False)) + if self.opts.get(u'syndic_master', False): # Is syndic + syndic_finger = self.opts.get(u'syndic_finger', self.opts.get(u'master_finger', False)) if syndic_finger: - if salt.utils.pem_finger(m_pub_fn, sum_type=self.opts['hash_type']) != syndic_finger: + if salt.utils.pem_finger(m_pub_fn, sum_type=self.opts[u'hash_type']) != syndic_finger: self._finger_fail(syndic_finger, m_pub_fn) else: - if self.opts.get('master_finger', False): - if salt.utils.pem_finger(m_pub_fn, sum_type=self.opts['hash_type']) != self.opts['master_finger']: - self._finger_fail(self.opts['master_finger'], m_pub_fn) - auth['publish_port'] = payload['publish_port'] + if self.opts.get(u'master_finger', False): + if salt.utils.pem_finger(m_pub_fn, sum_type=self.opts[u'hash_type']) != self.opts[u'master_finger']: + self._finger_fail(self.opts[u'master_finger'], m_pub_fn) + auth[u'publish_port'] = payload[u'publish_port'] return auth @@ -1216,7 +1253,7 @@ class Crypticle(object): Signing algorithm: HMAC-SHA256 ''' - PICKLE_PAD = b'pickle::' + PICKLE_PAD = b'pickle::' # future lint: disable=non-unicode-string AES_BLOCK_SIZE = 16 SIG_SIZE = hashlib.sha256().digest_size @@ -1231,16 +1268,17 @@ class Crypticle(object): key = os.urandom(key_size // 8 + cls.SIG_SIZE) b64key = base64.b64encode(key) if six.PY3: - b64key = b64key.decode('utf-8') - return b64key.replace('\n', '') + b64key = b64key.decode(u'utf-8') + # Return data must be a base64-encoded string, not a unicode type + return b64key.replace('\n', '') # future lint: disable=non-unicode-string @classmethod def extract_keys(cls, key_string, key_size): if six.PY2: - key = key_string.decode('base64') + key = key_string.decode(u'base64') else: - key = salt.utils.to_bytes(base64.b64decode(key_string)) - assert len(key) == key_size / 8 + cls.SIG_SIZE, 'invalid key' + key = salt.utils.stringutils.to_bytes(base64.b64decode(key_string)) + assert len(key) == key_size / 8 + cls.SIG_SIZE, u'invalid key' return key[:-cls.SIG_SIZE], key[-cls.SIG_SIZE:] def encrypt(self, data): @@ -1252,7 +1290,7 @@ class Crypticle(object): if six.PY2: data = data + pad * chr(pad) else: - data = data + salt.utils.to_bytes(pad * chr(pad)) + data = data + salt.utils.stringutils.to_bytes(pad * chr(pad)) iv_bytes = os.urandom(self.AES_BLOCK_SIZE) cypher = AES.new(aes_key, AES.MODE_CBC, iv_bytes) data = iv_bytes + cypher.encrypt(data) @@ -1267,11 +1305,11 @@ class Crypticle(object): sig = data[-self.SIG_SIZE:] data = data[:-self.SIG_SIZE] if six.PY3 and not isinstance(data, bytes): - data = salt.utils.to_bytes(data) + data = salt.utils.stringutils.to_bytes(data) mac_bytes = hmac.new(hmac_key, data, hashlib.sha256).digest() if len(mac_bytes) != len(sig): - log.debug('Failed to authenticate message') - raise AuthenticationError('message authentication failed') + log.debug(u'Failed to authenticate message') + raise AuthenticationError(u'message authentication failed') result = 0 if six.PY2: @@ -1281,8 +1319,8 @@ class Crypticle(object): for zipped_x, zipped_y in zip(mac_bytes, sig): result |= zipped_x ^ zipped_y if result != 0: - log.debug('Failed to authenticate message') - raise AuthenticationError('message authentication failed') + log.debug(u'Failed to authenticate message') + raise AuthenticationError(u'message authentication failed') iv_bytes = data[:self.AES_BLOCK_SIZE] data = data[self.AES_BLOCK_SIZE:] cypher = AES.new(aes_key, AES.MODE_CBC, iv_bytes) diff --git a/salt/daemons/__init__.py b/salt/daemons/__init__.py index 17460f4855c..f28a612762f 100644 --- a/salt/daemons/__init__.py +++ b/salt/daemons/__init__.py @@ -11,7 +11,7 @@ import logging # Import Salt Libs from salt.utils.odict import OrderedDict -import salt.ext.six as six +from salt.ext import six log = logging.getLogger(__name__) diff --git a/salt/daemons/flo/__init__.py b/salt/daemons/flo/__init__.py index 4cee14e2899..bd8e3dcec34 100644 --- a/salt/daemons/flo/__init__.py +++ b/salt/daemons/flo/__init__.py @@ -38,7 +38,7 @@ import salt.daemons.masterapi # Import 3rd-party libs import ioflo.app.run # pylint: disable=3rd-party-module-not-gated -import salt.ext.six as six +from salt.ext import six def explode_opts(opts): diff --git a/salt/daemons/flo/core.py b/salt/daemons/flo/core.py index 4b0f2fc288b..1a11e08aedb 100644 --- a/salt/daemons/flo/core.py +++ b/salt/daemons/flo/core.py @@ -18,7 +18,9 @@ from _socket import gaierror # Import salt libs import salt.daemons.masterapi import salt.utils.args +import salt.utils.kinds as kinds import salt.utils.process +import salt.utils.stringutils import salt.transport import salt.engines @@ -32,7 +34,7 @@ from raet.lane.stacking import LaneStack from salt import daemons from salt.daemons import salting from salt.exceptions import SaltException -from salt.utils import kinds, is_windows +from salt.utils.platform import is_windows from salt.utils.event import tagify # Import ioflo libs @@ -59,7 +61,7 @@ try: except ImportError: pass # pylint: disable=no-name-in-module,redefined-builtin -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import range # pylint: enable=import-error,no-name-in-module,redefined-builtin @@ -398,7 +400,7 @@ class SaltRaetRoadStackJoiner(ioflo.base.deeding.Deed): kind=kinds.applKinds.master)) except gaierror as ex: log.warning("Unable to connect to master {0}: {1}".format(mha, ex)) - if self.opts.value.get('master_type') != 'failover': + if self.opts.value.get(u'master_type') not in (u'failover', u'distributed'): raise ex if not stack.remotes: raise ex @@ -865,7 +867,7 @@ class SaltRaetManorLaneSetup(ioflo.base.deeding.Deed): self.presence_req.value = deque() self.stats_req.value = deque() self.publish.value = deque() - self.worker_verify.value = salt.utils.rand_string() + self.worker_verify.value = salt.utils.stringutils.random() if self.opts.value.get('worker_threads'): worker_seed = [] for index in range(self.opts.value['worker_threads']): diff --git a/salt/daemons/flo/jobber.py b/salt/daemons/flo/jobber.py index 202a32ceeb8..e18edc3c661 100644 --- a/salt/daemons/flo/jobber.py +++ b/salt/daemons/flo/jobber.py @@ -17,16 +17,19 @@ import subprocess import json # Import salt libs -import salt.ext.six as six +from salt.ext import six import salt.daemons.masterapi -import salt.utils.args import salt.utils +import salt.utils.args +import salt.utils.files +import salt.utils.kinds as kinds +import salt.utils.stringutils import salt.transport from raet import raeting, nacling from raet.lane.stacking import LaneStack from raet.lane.yarding import RemoteYard -from salt.utils import kinds, is_windows +from salt.utils.platform import is_windows from salt.utils.event import tagify from salt.exceptions import ( @@ -58,7 +61,7 @@ def jobber_check(self): rms.append(jid) data = self.shells.value[jid] stdout, stderr = data['proc'].communicate() - ret = json.loads(salt.utils.to_str(stdout), object_hook=salt.utils.decode_dict)['local'] + ret = json.loads(salt.utils.stringutils.to_str(stdout), object_hook=salt.utils.decode_dict)['local'] route = {'src': (self.stack.value.local.name, 'manor', 'jid_ret'), 'dst': (data['msg']['route']['src'][0], None, 'remote_cmd')} ret['cmd'] = '_return' @@ -211,7 +214,7 @@ class SaltRaetNixJobber(ioflo.base.deeding.Deed): except (KeyError, AttributeError, TypeError): pass else: - if isinstance(oput, str): + if isinstance(oput, six.string_types): ret['out'] = oput msg = {'route': route, 'load': ret} stack.transmit(msg, stack.fetchUidByName('manor')) @@ -279,7 +282,7 @@ class SaltRaetNixJobber(ioflo.base.deeding.Deed): sdata = {'pid': os.getpid()} sdata.update(data) - with salt.utils.fopen(fn_, 'w+b') as fp_: + with salt.utils.files.fopen(fn_, 'w+b') as fp_: fp_.write(self.serial.dumps(sdata)) ret = {'success': False} function_name = data['fun'] diff --git a/salt/daemons/flo/worker.py b/salt/daemons/flo/worker.py index fba2e7e632d..04c636110b3 100644 --- a/salt/daemons/flo/worker.py +++ b/salt/daemons/flo/worker.py @@ -21,7 +21,7 @@ from raet import raeting from raet.lane.stacking import LaneStack from raet.lane.yarding import RemoteYard -from salt.utils import kinds +import salt.utils.kinds as kinds # Import ioflo libs import ioflo.base.deeding diff --git a/salt/daemons/masterapi.py b/salt/daemons/masterapi.py index d27dde47188..9c59194a054 100644 --- a/salt/daemons/masterapi.py +++ b/salt/daemons/masterapi.py @@ -12,7 +12,6 @@ import os import re import time import stat -import msgpack # Import salt libs import salt.crypt @@ -31,17 +30,21 @@ import salt.fileserver import salt.utils.args import salt.utils.atomicfile import salt.utils.event +import salt.utils.files +import salt.utils.gitfs import salt.utils.verify import salt.utils.minions import salt.utils.gzip_util import salt.utils.jid +import salt.utils.minions +import salt.utils.platform +import salt.utils.verify from salt.defaults import DEFAULT_TARGET_DELIM from salt.pillar import git_pillar -from salt.utils.event import tagify from salt.exceptions import FileserverConfigError, SaltMasterError # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six try: import pwd @@ -63,44 +66,19 @@ def init_git_pillar(opts): ret = [] for opts_dict in [x for x in opts.get('ext_pillar', [])]: if 'git' in opts_dict: - if isinstance(opts_dict['git'], six.string_types): - # Legacy git pillar code - try: - import git - except ImportError: - return ret - parts = opts_dict['git'].strip().split() - try: - br = parts[0] - loc = parts[1] - except IndexError: - log.critical( - 'Unable to extract external pillar data: {0}' - .format(opts_dict['git']) - ) + try: + pillar = salt.utils.gitfs.GitPillar(opts) + pillar.init_remotes( + opts_dict['git'], + git_pillar.PER_REMOTE_OVERRIDES, + git_pillar.PER_REMOTE_ONLY + ) + ret.append(pillar) + except FileserverConfigError: + if opts.get('git_pillar_verify_config', True): + raise else: - ret.append( - git_pillar._LegacyGitPillar( - br, - loc, - opts - ) - ) - else: - # New git_pillar code - try: - pillar = salt.utils.gitfs.GitPillar(opts) - pillar.init_remotes( - opts_dict['git'], - git_pillar.PER_REMOTE_OVERRIDES, - git_pillar.PER_REMOTE_ONLY - ) - ret.append(pillar) - except FileserverConfigError: - if opts.get('git_pillar_verify_config', True): - raise - else: - log.critical('Could not initialize git_pillar') + log.critical('Could not initialize git_pillar') return ret @@ -150,22 +128,11 @@ def clean_expired_tokens(opts): ''' Clean expired tokens from the master ''' - serializer = salt.payload.Serial(opts) - for (dirpath, dirnames, filenames) in os.walk(opts['token_dir']): - for token in filenames: - token_path = os.path.join(dirpath, token) - with salt.utils.fopen(token_path, 'rb') as token_file: - try: - token_data = serializer.loads(token_file.read()) - except msgpack.UnpackValueError: - # Bad token file or empty. Remove. - os.remove(token_path) - return - if 'expire' not in token_data or token_data.get('expire', 0) < time.time(): - try: - os.remove(token_path) - except (IOError, OSError): - pass + loadauth = salt.auth.LoadAuth(opts) + for tok in loadauth.list_tokens(): + token_data = loadauth.get_tok(tok) + if 'expire' not in token_data or token_data.get('expire', 0) < time.time(): + loadauth.rm_token(tok) def clean_pub_auth(opts): @@ -203,7 +170,7 @@ def clean_old_jobs(opts): def mk_key(opts, user): - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): # The username may contain '\' if it is in Windows # 'DOMAIN\username' format. Fix this for the keyfile path. keyfile = os.path.join( @@ -216,14 +183,14 @@ def mk_key(opts, user): if os.path.exists(keyfile): log.debug('Removing stale keyfile: {0}'.format(keyfile)) - if salt.utils.is_windows() and not os.access(keyfile, os.W_OK): + if salt.utils.platform.is_windows() and not os.access(keyfile, os.W_OK): # Cannot delete read-only files on Windows. os.chmod(keyfile, stat.S_IRUSR | stat.S_IWUSR) os.unlink(keyfile) key = salt.crypt.Crypticle.generate_key_string() cumask = os.umask(191) - with salt.utils.fopen(keyfile, 'w+') as fp_: + with salt.utils.files.fopen(keyfile, 'w+') as fp_: fp_.write(key) os.umask(cumask) # 600 octal: Read and write access to the owner only. @@ -303,7 +270,7 @@ class AutoKey(object): ''' Check if the specified filename has correct permissions ''' - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): return True # After we've ascertained we're not on windows @@ -360,7 +327,7 @@ class AutoKey(object): log.warning(message.format(signing_file)) return False - with salt.utils.fopen(signing_file, 'r') as fp_: + with salt.utils.files.fopen(signing_file, 'r') as fp_: for line in fp_: line = line.strip() if line.startswith('#'): @@ -583,11 +550,12 @@ class RemoteFuncs(object): if match_type.lower() == 'compound': match_type = 'compound_pillar_exact' checker = salt.utils.minions.CkMinions(self.opts) - minions = checker.check_minions( + _res = checker.check_minions( load['tgt'], match_type, greedy=False ) + minions = _res['minions'] for minion in minions: fdata = self.cache.fetch('minions/{0}'.format(minion), 'mine') if isinstance(fdata, dict): @@ -693,7 +661,7 @@ class RemoteFuncs(object): mode = 'ab' else: mode = 'wb' - with salt.utils.fopen(cpath, mode) as fp_: + with salt.utils.files.fopen(cpath, mode) as fp_: if load['loc']: fp_.seek(load['loc']) fp_.write(load['data']) @@ -714,14 +682,13 @@ class RemoteFuncs(object): load.get('saltenv', load.get('env')), load.get('ext'), self.mminion.functions, - pillar=load.get('pillar_override', {})) - pillar_dirs = {} - data = pillar.compile_pillar(pillar_dirs=pillar_dirs) + pillar_override=load.get('pillar_override', {})) + data = pillar.compile_pillar() if self.opts.get('minion_data_cache', False): self.cache.store('minions/{0}'.format(load['id']), 'data', {'grains': load['grains'], 'pillar': data}) - self.event.fire_event('Minion data cache refresh', tagify(load['id'], 'refresh', 'minion')) + self.event.fire_event('Minion data cache refresh', salt.utils.event.tagify(load['id'], 'refresh', 'minion')) return data def _minion_event(self, load): @@ -741,7 +708,7 @@ class RemoteFuncs(object): event_data = event self.event.fire_event(event_data, event['tag']) # old dup event if load.get('pretag') is not None: - self.event.fire_event(event_data, tagify(event['tag'], base=load['pretag'])) + self.event.fire_event(event_data, salt.utils.event.tagify(event['tag'], base=load['pretag'])) else: tag = load['tag'] self.event.fire_event(load, tag) @@ -752,7 +719,7 @@ class RemoteFuncs(object): Handle the return data sent from the minions ''' # Generate EndTime - endtime = salt.utils.jid.jid_to_time(salt.utils.jid.gen_jid()) + endtime = salt.utils.jid.jid_to_time(salt.utils.jid.gen_jid(self.opts)) # If the return data is invalid, just ignore it if any(key not in load for key in ('return', 'jid', 'id')): return False @@ -767,7 +734,7 @@ class RemoteFuncs(object): self.mminion.returners[saveload_fstr](load['jid'], load) log.info('Got return from {id} for job {jid}'.format(**load)) self.event.fire_event(load, load['jid']) # old dup event - self.event.fire_event(load, tagify([load['jid'], 'ret', load['id']], 'job')) + self.event.fire_event(load, salt.utils.event.tagify([load['jid'], 'ret', load['id']], 'job')) self.event.fire_ret_load(load) if not self.opts['job_cache'] or self.opts.get('ext_job_cache'): return @@ -858,7 +825,7 @@ class RemoteFuncs(object): if not os.path.isdir(auth_cache): os.makedirs(auth_cache) jid_fn = os.path.join(auth_cache, load['jid']) - with salt.utils.fopen(jid_fn, 'r') as fp_: + with salt.utils.files.fopen(jid_fn, 'r') as fp_: if not load['id'] == fp_.read(): return {} @@ -906,16 +873,17 @@ class RemoteFuncs(object): pub_load['tgt_type'] = load['tgt_type'] ret = {} ret['jid'] = self.local.cmd_async(**pub_load) - ret['minions'] = self.ckminions.check_minions( + _res = self.ckminions.check_minions( load['tgt'], pub_load['tgt_type']) + ret['minions'] = _res['minions'] auth_cache = os.path.join( self.opts['cachedir'], 'publish_auth') if not os.path.isdir(auth_cache): os.makedirs(auth_cache) jid_fn = os.path.join(auth_cache, str(ret['jid'])) - with salt.utils.fopen(jid_fn, 'w+') as fp_: + with salt.utils.files.fopen(jid_fn, 'w+') as fp_: fp_.write(load['id']) return ret @@ -1045,35 +1013,33 @@ class LocalFuncs(object): ''' Send a master control function back to the runner system ''' - if 'token' in load: - auth_type = 'token' - err_name = 'TokenAuthenticationError' - token = self.loadauth.authenticate_token(load) - if not token: - return dict(error=dict(name=err_name, - message='Authentication failure of type "token" occurred.')) - username = token['name'] - if self.opts['keep_acl_in_token'] and 'auth_list' in token: - auth_list = token['auth_list'] - else: - load['eauth'] = token['eauth'] - load['username'] = username - auth_list = self.loadauth.get_auth_list(load) - else: - auth_type = 'eauth' - err_name = 'EauthAuthenticationError' - username = load.get('username', 'UNKNOWN') - if not self.loadauth.authenticate_eauth(load): - return dict(error=dict(name=err_name, - message=('Authentication failure of type "eauth" occurred ' - 'for user {0}.').format(username))) - auth_list = self.loadauth.get_auth_list(load) + # All runner opts pass through eauth + auth_type, err_name, key = self._prep_auth_info(load) - if not self.ckminions.runner_check(auth_list, load['fun']): - return dict(error=dict(name=err_name, - message=('Authentication failure of type "{0}" occurred ' - 'for user {1}.').format(auth_type, username))) + # Authenticate + auth_check = self.loadauth.check_authentication(load, auth_type) + error = auth_check.get('error') + if error: + # Authentication error occurred: do not continue. + return {'error': error} + + # Authorize + runner_check = self.ckminions.runner_check( + auth_check.get('auth_list', []), + load['fun'], + load['kwarg'] + ) + username = auth_check.get('username') + if not runner_check: + return {'error': {'name': err_name, + 'message': 'Authentication failure of type "{0}" occurred ' + 'for user {1}.'.format(auth_type, username)}} + elif isinstance(runner_check, dict) and 'error' in runner_check: + # A dictionary with an error name/message was handled by ckminions.runner_check + return runner_check + + # Authorized. Do the job! try: fun = load.pop('fun') runner_client = salt.runner.RunnerClient(self.opts) @@ -1082,80 +1048,73 @@ class LocalFuncs(object): username) except Exception as exc: log.error('Exception occurred while ' - 'introspecting {0}: {1}'.format(fun, exc)) - return dict(error=dict(name=exc.__class__.__name__, - args=exc.args, - message=str(exc))) + 'introspecting {0}: {1}'.format(fun, exc)) + return {'error': {'name': exc.__class__.__name__, + 'args': exc.args, + 'message': str(exc)}} def wheel(self, load): ''' Send a master control function back to the wheel system ''' # All wheel ops pass through eauth - if 'token' in load: - auth_type = 'token' - err_name = 'TokenAuthenticationError' - token = self.loadauth.authenticate_token(load) - if not token: - return dict(error=dict(name=err_name, - message='Authentication failure of type "token" occurred.')) - username = token['name'] - if self.opts['keep_acl_in_token'] and 'auth_list' in token: - auth_list = token['auth_list'] - else: - load['eauth'] = token['eauth'] - load['username'] = username - auth_list = self.loadauth.get_auth_list(load) - elif 'eauth' in load: - auth_type = 'eauth' - err_name = 'EauthAuthenticationError' - username = load.get('username', 'UNKNOWN') - if not self.loadauth.authenticate_eauth(load): - return dict(error=dict(name=err_name, - message=('Authentication failure of type "eauth" occurred for ' - 'user {0}.').format(username))) - auth_list = self.loadauth.get_auth_list(load) - else: - auth_type = 'user' - err_name = 'UserAuthenticationError' - username = load.get('username', 'UNKNOWN') - if not self.loadauth.authenticate_key(load, self.key): - return dict(error=dict(name=err_name, - message=('Authentication failure of type "user" occurred for ' - 'user {0}.').format(username))) + auth_type, err_name, key = self._prep_auth_info(load) + # Authenticate + auth_check = self.loadauth.check_authentication( + load, + auth_type, + key=key, + show_username=True + ) + error = auth_check.get('error') + + if error: + # Authentication error occurred: do not continue. + return {'error': error} + + # Authorize + username = auth_check.get('username') if auth_type != 'user': - if not self.ckminions.wheel_check(auth_list, load['fun']): - return dict(error=dict(name=err_name, - message=('Authentication failure of type "{0}" occurred for ' - 'user {1}.').format(auth_type, username))) + wheel_check = self.ckminions.wheel_check( + auth_check.get('auth_list', []), + load['fun'], + load['kwarg'] + ) + if not wheel_check: + return {'error': {'name': err_name, + 'message': 'Authentication failure of type "{0}" occurred for ' + 'user {1}.'.format(auth_type, username)}} + elif isinstance(wheel_check, dict) and 'error' in wheel_check: + # A dictionary with an error name/message was handled by ckminions.wheel_check + return wheel_check # Authenticated. Do the job. - jid = salt.utils.jid.gen_jid() + jid = salt.utils.jid.gen_jid(self.opts) fun = load.pop('fun') - tag = tagify(jid, prefix='wheel') + tag = salt.utils.event.tagify(jid, prefix='wheel') data = {'fun': "wheel.{0}".format(fun), 'jid': jid, 'tag': tag, 'user': username} try: - self.event.fire_event(data, tagify([jid, 'new'], 'wheel')) + self.event.fire_event(data, salt.utils.event.tagify([jid, 'new'], 'wheel')) ret = self.wheel_.call_func(fun, **load) data['return'] = ret data['success'] = True - self.event.fire_event(data, tagify([jid, 'ret'], 'wheel')) + self.event.fire_event(data, salt.utils.event.tagify([jid, 'ret'], 'wheel')) return {'tag': tag, 'data': data} except Exception as exc: log.error('Exception occurred while ' - 'introspecting {0}: {1}'.format(fun, exc)) + 'introspecting {0}: {1}'.format(fun, exc)) data['return'] = 'Exception occurred in wheel {0}: {1}: {2}'.format( fun, exc.__class__.__name__, exc, ) data['success'] = False - self.event.fire_event(data, tagify([jid, 'ret'], 'wheel')) + self.event.fire_event(data, salt.utils.event.tagify([jid, 'ret'], 'wheel')) return {'tag': tag, 'data': data} @@ -1201,11 +1160,12 @@ class LocalFuncs(object): # Retrieve the minions list delimiter = load.get('kwargs', {}).get('delimiter', DEFAULT_TARGET_DELIM) - minions = self.ckminions.check_minions( + _res = self.ckminions.check_minions( load['tgt'], load.get('tgt_type', 'glob'), delimiter ) + minions = _res['minions'] # Check for external auth calls if extra.get('token', False): @@ -1215,12 +1175,7 @@ class LocalFuncs(object): return '' # Get acl from eauth module. - if self.opts['keep_acl_in_token'] and 'auth_list' in token: - auth_list = token['auth_list'] - else: - extra['eauth'] = token['eauth'] - extra['username'] = token['name'] - auth_list = self.loadauth.get_auth_list(extra) + auth_list = self.loadauth.get_auth_list(extra, token) # Authorize the request if not self.ckminions.auth_check( @@ -1325,7 +1280,7 @@ class LocalFuncs(object): # Announce the job on the event bus self.event.fire_event(new_job_load, 'new_job') # old dup event - self.event.fire_event(new_job_load, tagify([load['jid'], 'new'], 'job')) + self.event.fire_event(new_job_load, salt.utils.event.tagify([load['jid'], 'new'], 'job')) # Save the invocation information if self.opts['ext_job_cache']: @@ -1417,3 +1372,18 @@ class LocalFuncs(object): }, 'pub': pub_load } + + def _prep_auth_info(self, load): + key = None + if 'token' in load: + auth_type = 'token' + err_name = 'TokenAuthenticationError' + elif 'eauth' in load: + auth_type = 'eauth' + err_name = 'EauthAuthenticationError' + else: + auth_type = 'user' + err_name = 'UserAuthenticationError' + key = self.key + + return auth_type, err_name, key diff --git a/salt/daemons/salting.py b/salt/daemons/salting.py index fe3ac1e024d..d68857cb48f 100644 --- a/salt/daemons/salting.py +++ b/salt/daemons/salting.py @@ -21,7 +21,7 @@ from raet.keeping import Keep from salt.key import RaetKey -from salt.utils import kinds +import salt.utils.kinds as kinds class SaltKeep(Keep): diff --git a/salt/daemons/test/test_saltkeep.py b/salt/daemons/test/test_saltkeep.py index 8b40e7e3c88..e869454d3ae 100644 --- a/salt/daemons/test/test_saltkeep.py +++ b/salt/daemons/test/test_saltkeep.py @@ -33,7 +33,7 @@ from raet.road import estating, keeping, stacking from salt.key import RaetKey from salt.daemons import salting from salt import daemons -from salt.utils import kinds +import salt.utils.kinds as kinds def setUpModule(): console.reinit(verbosity=console.Wordage.concise) diff --git a/salt/engines/__init__.py b/salt/engines/__init__.py index 06636e70b03..2399710b225 100644 --- a/salt/engines/__init__.py +++ b/salt/engines/__init__.py @@ -11,7 +11,7 @@ import logging # Import salt libs import salt import salt.loader -import salt.utils +import salt.utils.platform from salt.utils.process import SignalHandlingMultiprocessingProcess log = logging.getLogger(__name__) @@ -36,7 +36,7 @@ def start_engines(opts, proc_mgr, proxy=None): # Function references are not picklable. Windows needs to pickle when # spawning processes. On Windows, these will need to be recalculated # in the spawned child process. - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): runners = None utils = None funcs = None @@ -110,7 +110,7 @@ class Engine(SignalHandlingMultiprocessingProcess): Run the master service! ''' self.utils = salt.loader.utils(self.opts, proxy=self.proxy) - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): # Calculate function references since they can't be pickled. if self.opts['__role'] == 'master': self.runners = salt.loader.runner(self.opts, utils=self.utils) diff --git a/salt/engines/docker_events.py b/salt/engines/docker_events.py index b5b65b5a56d..3985213e6fd 100644 --- a/salt/engines/docker_events.py +++ b/salt/engines/docker_events.py @@ -50,8 +50,8 @@ def start(docker_url='unix://var/run/docker.sock', .. code-block:: yaml engines: - docker_events: - docker_url: unix://var/run/docker.sock + - docker_events: + docker_url: unix://var/run/docker.sock The config above sets up engines to listen for events from the Docker daemon and publish diff --git a/salt/engines/hipchat.py b/salt/engines/hipchat.py index a069a4f7482..f28e4900fdd 100644 --- a/salt/engines/hipchat.py +++ b/salt/engines/hipchat.py @@ -14,25 +14,25 @@ keys make the engine interactive. .. code-block:: yaml engines: - - hipchat: - api_url: http://api.hipchat.myteam.com - token: 'XXXXXX' - room: 'salt' - control: True - valid_users: - - SomeUser - valid_commands: - - test.ping - - cmd.run - - list_jobs - - list_commands - aliases: - list_jobs: - cmd: jobs.list_jobs - list_commands: - cmd: pillar.get salt:engines:hipchat:valid_commands target=saltmaster tgt_type=list - max_rooms: 0 - wait_time: 1 + - hipchat: + api_url: http://api.hipchat.myteam.com + token: 'XXXXXX' + room: 'salt' + control: True + valid_users: + - SomeUser + valid_commands: + - test.ping + - cmd.run + - list_jobs + - list_commands + aliases: + list_jobs: + cmd: jobs.list_jobs + list_commands: + cmd: pillar.get salt:engines:hipchat:valid_commands target=saltmaster + max_rooms: 0 + wait_time: 1 ''' from __future__ import absolute_import @@ -48,13 +48,15 @@ try: except ImportError: HAS_HYPCHAT = False -import salt.utils +import salt.utils.args +import salt.utils.event import salt.utils.files +import salt.utils.http import salt.runner import salt.client import salt.loader import salt.output -import salt.ext.six as six +from salt.ext import six def __virtual__(): @@ -93,7 +95,7 @@ def _publish_file(token, room, filepath, message='', outputter=None, api_url=Non headers['Authorization'] = "Bearer " + token msg = json.dumps({'message': message}) - with salt.utils.fopen(filepath, 'rb') as rfh: + with salt.utils.files.fopen(filepath, 'rb') as rfh: payload = """\ --boundary123456 Content-Type: application/json; charset=UTF-8 @@ -339,13 +341,13 @@ def start(token, args = [] kwargs = {} - cmdline = salt.utils.shlex_split(text) + cmdline = salt.utils.args.shlex_split(text) cmd = cmdline[0] # Evaluate aliases if aliases and isinstance(aliases, dict) and cmd in aliases.keys(): cmdline = aliases[cmd].get('cmd') - cmdline = salt.utils.shlex_split(cmdline) + cmdline = salt.utils.args.shlex_split(cmdline) cmd = cmdline[0] # Parse args and kwargs @@ -411,8 +413,8 @@ def start(token, _publish_code_message(token, room, ret, message=message_string, outputter=outputter, api_url=api_url) else: tmp_path_fn = salt.utils.files.mkstemp() - with salt.utils.fopen(tmp_path_fn, 'w+') as fp_: + with salt.utils.files.fopen(tmp_path_fn, 'w+') as fp_: fp_.write(json.dumps(ret, sort_keys=True, indent=4)) _publish_file(token, room, tmp_path_fn, message=message_string, api_url=api_url) - salt.utils.safe_rm(tmp_path_fn) + salt.utils.files.safe_rm(tmp_path_fn) time.sleep(wait_time or _DEFAULT_SLEEP) diff --git a/salt/engines/http_logstash.py b/salt/engines/http_logstash.py index 3e4b89ad789..78edc0d1b9d 100644 --- a/salt/engines/http_logstash.py +++ b/salt/engines/http_logstash.py @@ -12,13 +12,13 @@ them onto a logstash endpoint via HTTP requests. engines: - http_logstash: - url: http://blabla.com/salt-stuff - tags: - - salt/job/*/new - - salt/job/*/ret/* - funs: - - probes.results - - bgp.config + url: http://blabla.com/salt-stuff + tags: + - salt/job/*/new + - salt/job/*/ret/* + funs: + - probes.results + - bgp.config ''' from __future__ import absolute_import diff --git a/salt/engines/ircbot.py b/salt/engines/ircbot.py index 1963ccb8553..933c7dc83f2 100644 --- a/salt/engines/ircbot.py +++ b/salt/engines/ircbot.py @@ -73,6 +73,9 @@ log = logging.getLogger(__name__) # Import salt libraries import salt.utils.event +# Import 3rd-party libs +from salt.ext import six + # Nothing listening here Event = namedtuple("Event", "source code line") PrivEvent = namedtuple("PrivEvent", "source nick user host code channel command line") @@ -212,7 +215,7 @@ class IRCClient(object): logging.info("on_closed") def send_message(self, line): - if isinstance(line, str): + if isinstance(line, six.string_types): line = line.encode('utf-8') log.debug("Sending: %s", line) self._stream.write(line + b'\r\n') diff --git a/salt/engines/junos_syslog.py b/salt/engines/junos_syslog.py index 1f8b57f15e6..65648783dbb 100644 --- a/salt/engines/junos_syslog.py +++ b/salt/engines/junos_syslog.py @@ -102,8 +102,11 @@ except ImportError: class DatagramProtocol(object): pass -from salt.utils import event -from salt.ext.six import moves +import salt.utils.event as event + +# Import 3rd-party libs +from salt.ext import six +from salt.ext.six.moves import range # pylint: disable=redefined-builtin # logging.basicConfig(level=logging.DEBUG) log = logging.getLogger(__name__) @@ -264,7 +267,7 @@ class _SyslogServerFactory(DatagramProtocol): "jnpr/syslog". Using the default topic.') self.title = ['jnpr', 'syslog', 'hostname', 'event'] else: - for i in moves.range(2, len(topics)): + for i in range(2, len(topics)): if topics[i] not in data: log.debug( 'Please check the topic specified. \ @@ -305,7 +308,7 @@ class _SyslogServerFactory(DatagramProtocol): send_this_event = True for key in options: if key in data: - if isinstance(options[key], (str, int)): + if isinstance(options[key], (six.string_types, int)): if str(options[key]) != str(data[key]): send_this_event = False break @@ -328,7 +331,7 @@ class _SyslogServerFactory(DatagramProtocol): if 'event' in data: topic = 'jnpr/syslog' - for i in moves.range(2, len(self.title)): + for i in range(2, len(self.title)): topic += '/' + str(data[self.title[i]]) log.debug( 'Junos Syslog - sending this event on the bus: \ diff --git a/salt/engines/logentries.py b/salt/engines/logentries.py index 0fe422edfe9..9dd0fc37358 100644 --- a/salt/engines/logentries.py +++ b/salt/engines/logentries.py @@ -24,6 +24,9 @@ master config. :configuration: Example configuration + + .. code-block:: yaml + engines: - logentries: endpoint: data.logentries.com diff --git a/salt/engines/logstash.py b/salt/engines/logstash.py index d4ad095c7fd..d56db4d951d 100644 --- a/salt/engines/logstash.py +++ b/salt/engines/logstash.py @@ -8,6 +8,9 @@ them onto a logstash endpoint. :configuration: Example configuration + + .. code-block:: yaml + engines: - logstash: host: log.my_network.com diff --git a/salt/engines/napalm_syslog.py b/salt/engines/napalm_syslog.py index 6932616176f..8c1f94dc69c 100644 --- a/salt/engines/napalm_syslog.py +++ b/salt/engines/napalm_syslog.py @@ -190,8 +190,8 @@ except ImportError: # Import salt libs import salt.utils +import salt.utils.event as event import salt.utils.network -from salt.utils import event # ---------------------------------------------------------------------------------------------------------------------- # module properties diff --git a/salt/engines/reactor.py b/salt/engines/reactor.py index b95a8213e3f..43829f0a276 100644 --- a/salt/engines/reactor.py +++ b/salt/engines/reactor.py @@ -7,10 +7,10 @@ Example Config in Master or Minion config .. code-block:: yaml engines: - reactor: - refresh_interval: 60 - worker_threads: 10 - worker_hwm: 10000 + - reactor: + refresh_interval: 60 + worker_threads: 10 + worker_hwm: 10000 reactor: - 'salt/cloud/*/destroyed': diff --git a/salt/engines/redis_sentinel.py b/salt/engines/redis_sentinel.py index 8f4e8073133..596f610d38f 100644 --- a/salt/engines/redis_sentinel.py +++ b/salt/engines/redis_sentinel.py @@ -8,6 +8,9 @@ events based on the channels they are subscribed to. :configuration: Example configuration + + .. code-block:: yaml + engines: - redis_sentinel: hosts: diff --git a/salt/engines/slack.py b/salt/engines/slack.py index 0a3490a4c80..9f0e1638bc1 100644 --- a/salt/engines/slack.py +++ b/salt/engines/slack.py @@ -18,70 +18,73 @@ the saltmaster's minion pillar. .. versionadded: 2016.3.0 -:configuration: Example configuration using only a "default" group. The default group is not special. In addition, other groups are being loaded from pillars +:configuration: Example configuration using only a 'default' group. The default group is not special. +In addition, other groups are being loaded from pillars. - .. code-block:: yaml +.. code-block:: yaml - engines: - slack: - token: 'xoxb-xxxxxxxxxx-xxxxxxxxxxxxxxxxxxxxxxxx' - control: True - fire_all: False - groups_pillar_name: "slack_engine:groups_pillar" - groups: - default: - users: - - * - commands: - - test.ping - - cmd.run - - list_jobs - - list_commands - aliases: - list_jobs: - cmd: jobs.list_jobs - list_commands: - cmd: pillar.get salt:engines:slack:valid_commands target=saltmaster tgt_type=list - default_target: - target: saltmaster - tgt_type: glob - targets: - test.ping: - target: '*' - tgt_type: glob - cmd.run: - target: saltmaster - tgt_type: list + engines: + - slack: + token: 'xoxb-xxxxxxxxxx-xxxxxxxxxxxxxxxxxxxxxxxx' + control: True + fire_all: False + groups_pillar_name: 'slack_engine:groups_pillar' + groups: + default: + users: + - * + commands: + - test.ping + - cmd.run + - list_jobs + - list_commands + aliases: + list_jobs: + cmd: jobs.list_jobs + list_commands: + cmd: pillar.get salt:engines:slack:valid_commands target=saltmaster tgt_type=list + default_target: + target: saltmaster + tgt_type: glob + targets: + test.ping: + target: '*' + tgt_type: glob + cmd.run: + target: saltmaster + tgt_type: list +:configuration: Example configuration using the 'default' group and a non-default group and a pillar that will be merged in + If the user is '*' (without the quotes) then the group's users or commands will match all users as appropriate - :configuration: Example configuration using the "default" group and a non-default group and a pillar that will be merged in - If the user is '*' (without the quotes) then the group's users or commands will match all users as appropriate - .. versionadded: 2017.7.0 +.. versionadded: 2017.7.0 - engines: - slack: - groups_pillar: slack_engine_pillar - token: 'xoxb-xxxxxxxxxx-xxxxxxxxxxxxxxxxxxxxxxxx' - control: True - fire_all: True - tag: salt/engines/slack - groups_pillar_name: "slack_engine:groups_pillar" - groups: - default: - valid_users: - - * - valid_commands: - - test.ping - aliases: - list_jobs: - cmd: jobs.list_jobs - list_commands: - cmd: pillar.get salt:engines:slack:valid_commands target=saltmaster tgt_type=list - gods: - users: - - garethgreenaway - commands: - - * +.. code-block:: yaml + + engines: + - slack: + groups_pillar: slack_engine_pillar + token: 'xoxb-xxxxxxxxxx-xxxxxxxxxxxxxxxxxxxxxxxx' + control: True + fire_all: True + tag: salt/engines/slack + groups_pillar_name: 'slack_engine:groups_pillar' + groups: + default: + valid_users: + - * + valid_commands: + - test.ping + aliases: + list_jobs: + cmd: jobs.list_jobs + list_commands: + cmd: pillar.get salt:engines:slack:valid_commands target=saltmaster tgt_type=list + gods: + users: + - garethgreenaway + commands: + - * :depends: slackclient @@ -89,6 +92,8 @@ the saltmaster's minion pillar. # Import python libraries from __future__ import absolute_import +import ast +import datetime import json import itertools import logging @@ -111,6 +116,7 @@ import salt.loader import salt.minion import salt.runner import salt.utils +import salt.utils.args import salt.utils.event import salt.utils.http import salt.utils.slack @@ -125,605 +131,628 @@ def __virtual__(): return __virtualname__ -def get_slack_users(token): - ''' - Get all users from Slack - ''' +class SlackClient(object): + def __init__(self, token): + self.master_minion = salt.minion.MasterMinion(__opts__) - ret = salt.utils.slack.query(function='users', - api_key=token, - opts=__opts__) - users = {} - if 'message' in ret: - for item in ret['message']: - if 'is_bot' in item: - if not item['is_bot']: - users[item['name']] = item['id'] - users[item['id']] = item['name'] - return users + self.sc = slackclient.SlackClient(token) + self.slack_connect = self.sc.rtm_connect() + def get_slack_users(self, token): + ''' + Get all users from Slack + ''' -def get_slack_channels(token): - ''' - Get all channel names from Slack - ''' + ret = salt.utils.slack.query(function='users', + api_key=token, + opts=__opts__) + users = {} + if 'message' in ret: + for item in ret['message']: + if 'is_bot' in item: + if not item['is_bot']: + users[item['name']] = item['id'] + users[item['id']] = item['name'] + return users - ret = salt.utils.slack.query( - function='rooms', - api_key=token, - # These won't be honored until https://github.com/saltstack/salt/pull/41187/files is merged - opts={ - 'exclude_archived': True, - 'exclude_members': True - }) - channels = {} - if 'message' in ret: - for item in ret['message']: - channels[item["id"]] = item["name"] - return channels + def get_slack_channels(self, token): + ''' + Get all channel names from Slack + ''' + ret = salt.utils.slack.query( + function='rooms', + api_key=token, + # These won't be honored until https://github.com/saltstack/salt/pull/41187/files is merged + opts={ + 'exclude_archived': True, + 'exclude_members': True + }) + channels = {} + if 'message' in ret: + for item in ret['message']: + channels[item['id']] = item['name'] + return channels -def get_config_groups(groups_conf, groups_pillar_name): - """ - get info from groups in config, and from the named pillar + def get_config_groups(self, groups_conf, groups_pillar_name): + ''' + get info from groups in config, and from the named pillar - todo: add specification for the minion to use to recover pillar - """ - # Get groups - # Default to returning something that'll never match - ret_groups = { - "default": { - "users": set(), - "commands": set(), - "aliases": dict(), - "default_target": dict(), - "targets": dict() + todo: add specification for the minion to use to recover pillar + ''' + # Get groups + # Default to returning something that'll never match + ret_groups = { + 'default': { + 'users': set(), + 'commands': set(), + 'aliases': dict(), + 'default_target': dict(), + 'targets': dict() + } } - } - # allow for empty groups in the config file, and instead let some/all of this come - # from pillar data. - if not groups_conf: - use_groups = {} - else: - use_groups = groups_conf - # First obtain group lists from pillars, then in case there is any overlap, iterate over the groups - # that come from pillars. The configuration in files on disk/from startup - # will override any configs from pillars. They are meant to be complementary not to provide overrides. - try: - groups_gen = itertools.chain(_groups_from_pillar(groups_pillar_name).items(), use_groups.items()) - except AttributeError: - log.warn("Failed to get groups from {}: {}".format(groups_pillar_name, _groups_from_pillar(groups_pillar_name))) - log.warn("or from config: {}".format(use_groups)) - groups_gen = [] - for name, config in groups_gen: - log.info("Trying to get {} and {} to be useful".format(name, config)) - ret_groups.setdefault(name, { - "users": set(), "commands": set(), "aliases": dict(), "default_target": dict(), "targets": dict() - }) - try: - ret_groups[name]['users'].update(set(config.get('users', []))) - ret_groups[name]['commands'].update(set(config.get('commands', []))) - ret_groups[name]['aliases'].update(config.get('aliases', {})) - ret_groups[name]['default_target'].update(config.get('default_target', {})) - ret_groups[name]['targets'].update(config.get('targets', {})) - except IndexError: - log.warn("Couldn't use group {}. Check that targets is a dict and not a list".format(name)) - - log.debug("Got the groups: {}".format(ret_groups)) - return ret_groups - - -def _groups_from_pillar(pillar_name): - """pillar_prefix is the pillar.get syntax for the pillar to be queried. - Group name is gotten via the equivalent of using - ``salt['pillar.get']('{}:{}'.format(pillar_prefix, group_name))`` - in a jinja template. - - returns a dictionary (unless the pillar is mis-formatted) - XXX: instead of using Caller, make the minion to use configurable so there could be some - restrictions placed on what pillars can be used. - """ - caller = salt.client.Caller() - pillar_groups = caller.cmd('pillar.get', pillar_name) - # pillar_groups = __salt__['pillar.get'](pillar_name, {}) - log.info("Got pillar groups {} from pillar {}".format(pillar_groups, pillar_name)) - log.info("pillar groups type is {}".format(type(pillar_groups))) - return pillar_groups - - -def fire(tag, msg): - """ - This replaces a function in main called "fire" - - It fires an event into the salt bus. - """ - if __opts__.get('__role') == 'master': - fire_master = salt.utils.event.get_master_event( - __opts__, - __opts__['sock_dir']).fire_event - else: - fire_master = None - - if fire_master: - fire_master(msg, tag) - else: - __salt__['event.send'](tag, msg) - - -def can_user_run(user, command, groups): - """ - Break out the permissions into the folowing: - - Check whether a user is in any group, including whether a group has the '*' membership - - :type user: str - :param user: The username being checked against - - :type command: str - :param command: The command that is being invoked (e.g. test.ping) - - :type groups: dict - :param groups: the dictionary with groups permissions structure. - - :rtype: tuple - :returns: On a successful permitting match, returns 2-element tuple that contains - the name of the group that successfuly matched, and a dictionary containing - the configuration of the group so it can be referenced. - - On failure it returns an empty tuple - - """ - log.info("{} wants to run {} with groups {}".format(user, command, groups)) - for key, val in groups.items(): - if user not in val['users']: - if '*' not in val['users']: - continue # this doesn't grant permissions, pass - if (command not in val['commands']) and (command not in val.get('aliases', {}).keys()): - if '*' not in val['commands']: - continue # again, pass - log.info("Slack user {} permitted to run {}".format(user, command)) - return (key, val,) # matched this group, return the group - log.info("Slack user {} denied trying to run {}".format(user, command)) - return () - - -def commandline_to_list(cmdline_str, trigger_string): - """ - cmdline_str is the string of the command line - trigger_string is the trigger string, to be removed - """ - cmdline = salt.utils.shlex_split(cmdline_str[len(trigger_string):]) - # Remove slack url parsing - # Translate target= - # to target=host.domain.net - cmdlist = [] - for cmditem in cmdline: - pattern = r'(?P.*)(<.*\|)(?P.*)(>)(?P.*)' - mtch = re.match(pattern, cmditem) - if mtch: - origtext = mtch.group('begin') + mtch.group('url') + mtch.group('remainder') - cmdlist.append(origtext) + # allow for empty groups in the config file, and instead let some/all of this come + # from pillar data. + if not groups_conf: + use_groups = {} else: - cmdlist.append(cmditem) - return cmdlist + use_groups = groups_conf + # First obtain group lists from pillars, then in case there is any overlap, iterate over the groups + # that come from pillars. The configuration in files on disk/from startup + # will override any configs from pillars. They are meant to be complementary not to provide overrides. + log.debug('use_groups {}'.format(use_groups)) + try: + groups_gen = itertools.chain(self._groups_from_pillar(groups_pillar_name).items(), use_groups.items()) + except AttributeError: + log.warn('Failed to get groups from {}: {}'.format(groups_pillar_name, self._groups_from_pillar(groups_pillar_name))) + log.warn('or from config: {}'.format(use_groups)) + groups_gen = [] + for name, config in groups_gen: + log.info('Trying to get {} and {} to be useful'.format(name, config)) + ret_groups.setdefault(name, { + 'users': set(), 'commands': set(), 'aliases': dict(), 'default_target': dict(), 'targets': dict() + }) + try: + ret_groups[name]['users'].update(set(config.get('users', []))) + ret_groups[name]['commands'].update(set(config.get('commands', []))) + ret_groups[name]['aliases'].update(config.get('aliases', {})) + ret_groups[name]['default_target'].update(config.get('default_target', {})) + ret_groups[name]['targets'].update(config.get('targets', {})) + except IndexError: + log.warn("Couldn't use group {}. Check that targets is a dict and not a list".format(name)) + log.debug('Got the groups: {}'.format(ret_groups)) + return ret_groups + + def _groups_from_pillar(self, pillar_name): + ''' + pillar_prefix is the pillar.get syntax for the pillar to be queried. + Group name is gotten via the equivalent of using + ``salt['pillar.get']('{}:{}'.format(pillar_prefix, group_name))`` + in a jinja template. + + returns a dictionary (unless the pillar is mis-formatted) + XXX: instead of using Caller, make the minion to use configurable so there could be some + restrictions placed on what pillars can be used. + ''' + caller = salt.client.Caller() + pillar_groups = caller.cmd('pillar.get', pillar_name) + # pillar_groups = __salt__['pillar.get'](pillar_name, {}) + log.debug('Got pillar groups %s from pillar %s', pillar_groups, pillar_name) + log.debug('pillar groups is %s', pillar_groups) + log.debug('pillar groups type is %s', type(pillar_groups)) + if pillar_groups: + return pillar_groups + else: + return {} + + def fire(self, tag, msg): + ''' + This replaces a function in main called 'fire' + + It fires an event into the salt bus. + ''' + if __opts__.get('__role') == 'master': + fire_master = salt.utils.event.get_master_event( + __opts__, + __opts__['sock_dir']).fire_master + else: + fire_master = None + + if fire_master: + fire_master(msg, tag) + else: + __salt__['event.send'](tag, msg) + + def can_user_run(self, user, command, groups): + ''' + Break out the permissions into the folowing: + + Check whether a user is in any group, including whether a group has the '*' membership + + :type user: str + :param user: The username being checked against + + :type command: str + :param command: The command that is being invoked (e.g. test.ping) + + :type groups: dict + :param groups: the dictionary with groups permissions structure. + + :rtype: tuple + :returns: On a successful permitting match, returns 2-element tuple that contains + the name of the group that successfuly matched, and a dictionary containing + the configuration of the group so it can be referenced. + + On failure it returns an empty tuple + + ''' + log.info('{} wants to run {} with groups {}'.format(user, command, groups)) + for key, val in groups.items(): + if user not in val['users']: + if '*' not in val['users']: + continue # this doesn't grant permissions, pass + if (command not in val['commands']) and (command not in val.get('aliases', {}).keys()): + if '*' not in val['commands']: + continue # again, pass + log.info('Slack user {} permitted to run {}'.format(user, command)) + return (key, val,) # matched this group, return the group + log.info('Slack user {} denied trying to run {}'.format(user, command)) + return () + + def commandline_to_list(self, cmdline_str, trigger_string): + ''' + cmdline_str is the string of the command line + trigger_string is the trigger string, to be removed + ''' + cmdline = salt.utils.args.shlex_split(cmdline_str[len(trigger_string):]) + # Remove slack url parsing + # Translate target= + # to target=host.domain.net + cmdlist = [] + for cmditem in cmdline: + pattern = r'(?P.*)(<.*\|)(?P.*)(>)(?P.*)' + mtch = re.match(pattern, cmditem) + if mtch: + origtext = mtch.group('begin') + mtch.group('url') + mtch.group('remainder') + cmdlist.append(origtext) + else: + cmdlist.append(cmditem) + return cmdlist # m_data -> m_data, _text -> test, all_slack_users -> all_slack_users, -def control_message_target(slack_user_name, text, loaded_groups, trigger_string): - """Returns a tuple of (target, cmdline,) for the response + def control_message_target(self, slack_user_name, text, loaded_groups, trigger_string): + '''Returns a tuple of (target, cmdline,) for the response - Raises IndexError if a user can't be looked up from all_slack_users + Raises IndexError if a user can't be looked up from all_slack_users - Returns (False, False) if the user doesn't have permission + Returns (False, False) if the user doesn't have permission - These are returned together because the commandline and the targeting - interact with the group config (specifically aliases and targeting configuration) - so taking care of them together works out. + These are returned together because the commandline and the targeting + interact with the group config (specifically aliases and targeting configuration) + so taking care of them together works out. - The cmdline that is returned is the actual list that should be - processed by salt, and not the alias. + The cmdline that is returned is the actual list that should be + processed by salt, and not the alias. - """ + ''' - # Trim the trigger string from the front - # cmdline = _text[1:].split(' ', 1) - cmdline = commandline_to_list(text, trigger_string) - permitted_group = can_user_run(slack_user_name, cmdline[0], loaded_groups) - log.debug("slack_user_name is {} and the permitted group is {}".format(slack_user_name, permitted_group)) - if not permitted_group: - return (False, False) - if not slack_user_name: - return (False, False) + # Trim the trigger string from the front + # cmdline = _text[1:].split(' ', 1) + cmdline = self.commandline_to_list(text, trigger_string) + permitted_group = self.can_user_run(slack_user_name, cmdline[0], loaded_groups) + log.debug('slack_user_name is {} and the permitted group is {}'.format(slack_user_name, permitted_group)) - # maybe there are aliases, so check on that - if cmdline[0] in permitted_group[1].get('aliases', {}).keys(): - use_cmdline = commandline_to_list(permitted_group[1]['aliases'][cmdline[0]], "") - else: - use_cmdline = cmdline - target = get_target(permitted_group, cmdline, use_cmdline) - return (target, use_cmdline,) + if not permitted_group: + return (False, None, cmdline[0]) + if not slack_user_name: + return (False, None, cmdline[0]) - -def message_text(m_data): - """ - Raises ValueError if a value doesn't work out, and TypeError if - this isn't a message type - """ - if m_data.get('type') != 'message': - raise TypeError("This isn't a message") - # Edited messages have text in message - _text = m_data.get('text', None) or m_data.get('message', {}).get('text', None) - try: - log.info("Message is {}".format(_text)) # this can violate the ascii codec - except UnicodeEncodeError as uee: - log.warn("Got a message that I couldn't log. The reason is: {}".format(uee)) - - # Convert UTF to string - _text = json.dumps(_text) - _text = yaml.safe_load(_text) - - if not _text: - raise ValueError("_text has no value") - return _text - - -def generate_triggered_messages(token, trigger_string, groups, groups_pillar_name): - """slack_token = string - trigger_string = string - input_valid_users = set - input_valid_commands = set - - When the trigger_string prefixes the message text, yields a dictionary of { - "message_data": m_data, - "cmdline": cmdline_list, # this is a list - "channel": channel, - "user": m_data['user'], - "slack_client": sc - } - - else yields {"message_data": m_data} and the caller can handle that - - When encountering an error (e.g. invalid message), yields {}, the caller can proceed to the next message - - When the websocket being read from has given up all its messages, yields {"done": True} to - indicate that the caller has read all of the relevent data for now, and should continue - its own processing and check back for more data later. - - This relies on the caller sleeping between checks, otherwise this could flood - """ - sc = slackclient.SlackClient(token) - slack_connect = sc.rtm_connect() - all_slack_users = get_slack_users(token) # re-checks this if we have an negative lookup result - all_slack_channels = get_slack_channels(token) # re-checks this if we have an negative lookup result - - def just_data(m_data): - """Always try to return the user and channel anyway""" - user_id = m_data.get('user') - channel_id = m_data.get('channel') - if channel_id.startswith('D'): # private chate with bot user - channel_name = "private chat" + # maybe there are aliases, so check on that + if cmdline[0] in permitted_group[1].get('aliases', {}).keys(): + use_cmdline = self.commandline_to_list(permitted_group[1]['aliases'][cmdline[0]], '') else: - channel_name = all_slack_channels.get(channel_id) - data = { - "message_data": m_data, - "user_name": all_slack_users.get(user_id), - "channel_name": channel_name + use_cmdline = cmdline + target = self.get_target(permitted_group, cmdline, use_cmdline) + + return (True, target, use_cmdline) + + def message_text(self, m_data): + ''' + Raises ValueError if a value doesn't work out, and TypeError if + this isn't a message type + ''' + if m_data.get('type') != 'message': + raise TypeError('This is not a message') + # Edited messages have text in message + _text = m_data.get('text', None) or m_data.get('message', {}).get('text', None) + try: + log.info('Message is {}'.format(_text)) # this can violate the ascii codec + except UnicodeEncodeError as uee: + log.warn('Got a message that I could not log. The reason is: {}'.format(uee)) + + # Convert UTF to string + _text = json.dumps(_text) + _text = yaml.safe_load(_text) + + if not _text: + raise ValueError('_text has no value') + return _text + + def generate_triggered_messages(self, token, trigger_string, groups, groups_pillar_name): + ''' + slack_token = string + trigger_string = string + input_valid_users = set + input_valid_commands = set + + When the trigger_string prefixes the message text, yields a dictionary of { + 'message_data': m_data, + 'cmdline': cmdline_list, # this is a list + 'channel': channel, + 'user': m_data['user'], + 'slack_client': sc } - if not data["user_name"]: - all_slack_users.clear() - all_slack_users.update(get_slack_users(token)) - data["user_name"] = all_slack_users.get(user_id) - if not data["channel_name"]: - all_slack_channels.clear() - all_slack_channels.update(get_slack_channels(token)) - data["channel_name"] = all_slack_channels.get(channel_id) - return data - for sleeps in (5, 10, 30, 60): - if slack_connect: - break + else yields {'message_data': m_data} and the caller can handle that + + When encountering an error (e.g. invalid message), yields {}, the caller can proceed to the next message + + When the websocket being read from has given up all its messages, yields {'done': True} to + indicate that the caller has read all of the relevent data for now, and should continue + its own processing and check back for more data later. + + This relies on the caller sleeping between checks, otherwise this could flood + ''' + all_slack_users = self.get_slack_users(token) # re-checks this if we have an negative lookup result + all_slack_channels = self.get_slack_channels(token) # re-checks this if we have an negative lookup result + + def just_data(m_data): + '''Always try to return the user and channel anyway''' + if 'user' not in m_data: + if 'message' in m_data and 'user' in m_data['message']: + log.debug('Message was edited, ' + 'so we look for user in ' + 'the original message.') + user_id = m_data['message']['user'] + else: + user_id = m_data.get('user') + channel_id = m_data.get('channel') + if channel_id.startswith('D'): # private chate with bot user + channel_name = 'private chat' + else: + channel_name = all_slack_channels.get(channel_id) + data = { + 'message_data': m_data, + 'user_id': user_id, + 'user_name': all_slack_users.get(user_id), + 'channel_name': channel_name + } + if not data['user_name']: + all_slack_users.clear() + all_slack_users.update(self.get_slack_users(token)) + data['user_name'] = all_slack_users.get(user_id) + if not data['channel_name']: + all_slack_channels.clear() + all_slack_channels.update(self.get_slack_channels(token)) + data['channel_name'] = all_slack_channels.get(channel_id) + return data + + for sleeps in (5, 10, 30, 60): + if self.slack_connect: + break + else: + # see https://api.slack.com/docs/rate-limits + log.warning('Slack connection is invalid. Server: {}, sleeping {}'.format(self.sc.server, sleeps)) + time.sleep(sleeps) # respawning too fast makes the slack API unhappy about the next reconnection else: - # see https://api.slack.com/docs/rate-limits - log.warning("Slack connection is invalid. Server: {}, sleeping {}".format(sc.server, sleeps)) - time.sleep(sleeps) # respawning too fast makes the slack API unhappy about the next reconnection - else: - raise UserWarning("Connection to slack is still invalid, giving up: {}".format(slack_connect)) # Boom! - while True: - msg = sc.rtm_read() - for m_data in msg: - try: - msg_text = message_text(m_data) - except (ValueError, TypeError) as msg_err: - log.debug("Got an error from trying to get the message text {}".format(msg_err)) - yield {"message_data": m_data} # Not a message type from the API? - continue + raise UserWarning('Connection to slack is still invalid, giving up: {}'.format(self.slack_connect)) # Boom! + while True: + msg = self.sc.rtm_read() + for m_data in msg: + try: + msg_text = self.message_text(m_data) + except (ValueError, TypeError) as msg_err: + log.debug('Got an error from trying to get the message text {}'.format(msg_err)) + yield {'message_data': m_data} # Not a message type from the API? + continue - # Find the channel object from the channel name - channel = sc.server.channels.find(m_data['channel']) - data = just_data(m_data) - if msg_text.startswith(trigger_string): - loaded_groups = get_config_groups(groups, groups_pillar_name) - user_id = m_data.get('user') # slack user ID, e.g. 'U11011' - if not data.get('user_name'): - log.error("The user {} can't be looked up via slack. What has happened here?".format( - m_data.get('user'))) - channel.send_message("The user {} can't be looked up via slack. Not running {}".format( - user_id, msg_text)) - yield {"message_data": m_data} - continue - (target, cmdline) = control_message_target( - data['user_name'], msg_text, loaded_groups, trigger_string) - log.debug("Got target: {}, cmdline: {}".format(target, cmdline)) - if target and cmdline: - yield { - "message_data": m_data, - "slack_client": sc, - "channel": channel, - "user": user_id, - "user_name": all_slack_users[user_id], - "cmdline": cmdline, - "target": target - } - continue + # Find the channel object from the channel name + channel = self.sc.server.channels.find(m_data['channel']) + data = just_data(m_data) + if msg_text.startswith(trigger_string): + loaded_groups = self.get_config_groups(groups, groups_pillar_name) + if not data.get('user_name'): + log.error('The user {} can not be looked up via slack. What has happened here?'.format( + m_data.get('user'))) + channel.send_message('The user {} can not be looked up via slack. Not running {}'.format( + data['user_id'], msg_text)) + yield {'message_data': m_data} + continue + (allowed, target, cmdline) = self.control_message_target( + data['user_name'], msg_text, loaded_groups, trigger_string) + log.debug('Got target: {}, cmdline: {}'.format(target, cmdline)) + if allowed: + yield { + 'message_data': m_data, + 'channel': m_data['channel'], + 'user': data['user_id'], + 'user_name': data['user_name'], + 'cmdline': cmdline, + 'target': target + } + continue + else: + channel.send_message('{0} is not allowed to use command {1}.'.format( + data['user_name'], cmdline)) + yield data + continue else: - channel.send_message('{}, {} is not allowed to use command {}.'.format( - user_id, all_slack_users[user_id], cmdline)) yield data continue - else: - yield data - continue - yield {"done": True} + yield {'done': True} + def get_target(self, permitted_group, cmdline, alias_cmdline): + ''' + When we are permitted to run a command on a target, look to see + what the default targeting is for that group, and for that specific + command (if provided). -def get_target(permitted_group, cmdline, alias_cmdline): - """When we are permitted to run a command on a target, look to see - what the default targeting is for that group, and for that specific - command (if provided). + It's possible for None or False to be the result of either, which means + that it's expected that the caller provide a specific target. - It's possible for None or False to be the result of either, which means - that it's expected that the caller provide a specific target. + If no configured target is provided, the command line will be parsed + for target=foo and tgt_type=bar - If no configured target is provided, the command line will be parsed - for target=foo and tgt_type=bar + Test for this: + h = {'aliases': {}, 'commands': {'cmd.run', 'pillar.get'}, + 'default_target': {'target': '*', 'tgt_type': 'glob'}, + 'targets': {'pillar.get': {'target': 'you_momma', 'tgt_type': 'list'}}, + 'users': {'dmangot', 'jmickle', 'pcn'}} + f = {'aliases': {}, 'commands': {'cmd.run', 'pillar.get'}, + 'default_target': {}, 'targets': {},'users': {'dmangot', 'jmickle', 'pcn'}} - Test for this: - h = {'aliases': {}, 'commands': {'cmd.run', 'pillar.get'}, - 'default_target': {'target': '*', 'tgt_type': 'glob'}, - 'targets': {'pillar.get': {'target': 'you_momma', 'tgt_type': 'list'}}, - 'users': {'dmangot', 'jmickle', 'pcn'}} - f = {'aliases': {}, 'commands': {'cmd.run', 'pillar.get'}, - 'default_target': {}, 'targets': {},'users': {'dmangot', 'jmickle', 'pcn'}} + g = {'aliases': {}, 'commands': {'cmd.run', 'pillar.get'}, + 'default_target': {'target': '*', 'tgt_type': 'glob'}, + 'targets': {}, 'users': {'dmangot', 'jmickle', 'pcn'}} - g = {'aliases': {}, 'commands': {'cmd.run', 'pillar.get'}, - 'default_target': {'target': '*', 'tgt_type': 'glob'}, - 'targets': {}, 'users': {'dmangot', 'jmickle', 'pcn'}} + Run each of them through ``get_configured_target(('foo', f), 'pillar.get')`` and confirm a valid target - Run each of them through ``get_configured_target(("foo", f), "pillar.get")`` and confirm a valid target + ''' + # Default to targetting all minions with a type of glob + null_target = {'target': '*', 'tgt_type': 'glob'} - """ - null_target = {"target": None, "tgt_type": None} + def check_cmd_against_group(cmd): + ''' + Validate cmd against the group to return the target, or a null target + ''' + name, group_config = permitted_group + target = group_config.get('default_target') + if not target: # Empty, None, or False + target = null_target + if group_config.get('targets'): + if group_config['targets'].get(cmd): + target = group_config['targets'][cmd] + if not target.get('target'): + log.debug('Group {} is not configured to have a target for cmd {}.'.format(name, cmd)) + return target - def check_cmd_against_group(cmd): - """Validate cmd against the group to return the target, or a null target""" - name, group_config = permitted_group - target = group_config.get('default_target') - if not target: # Empty, None, or False - target = null_target - if group_config.get('targets'): - if group_config['targets'].get(cmd): - target = group_config['targets'][cmd] - if not target.get("target"): - log.debug("Group {} is not configured to have a target for cmd {}.".format(name, cmd)) - return target + for this_cl in cmdline, alias_cmdline: + _, kwargs = self.parse_args_and_kwargs(this_cl) + if 'target' in kwargs: + log.debug('target is in kwargs {}.'.format(kwargs)) + if 'tgt_type' in kwargs: + log.debug('tgt_type is in kwargs {}.'.format(kwargs)) + return {'target': kwargs['target'], 'tgt_type': kwargs['tgt_type']} + return {'target': kwargs['target'], 'tgt_type': 'glob'} - for this_cl in cmdline, alias_cmdline: - _, kwargs = parse_args_and_kwargs(this_cl) - if 'target' in kwargs: - log.debug("target is in kwargs {}.".format(kwargs)) - if 'tgt_type' in kwargs: - log.debug("tgt_type is in kwargs {}.".format(kwargs)) - return {"target": kwargs['target'], "tgt_type": kwargs['tgt_type']} - return {"target": kwargs['target'], "tgt_type": 'glob'} - - for this_cl in cmdline, alias_cmdline: - checked = check_cmd_against_group(this_cl[0]) - log.debug("this cmdline has target {}.".format(this_cl)) - if checked.get("target"): - return checked - return null_target + for this_cl in cmdline, alias_cmdline: + checked = check_cmd_against_group(this_cl[0]) + log.debug('this cmdline has target {}.'.format(this_cl)) + if checked.get('target'): + return checked + return null_target # emulate the yaml_out output formatter. It relies on a global __opts__ object which we can't # obviously pass in -def format_return_text(data, **kwargs): # pylint: disable=unused-argument - ''' - Print out YAML using the block mode - ''' - params = dict(Dumper=OrderedDumper) - if 'output_indent' not in __opts__: - # default indentation - params.update(default_flow_style=False) - elif __opts__['output_indent'] >= 0: - # custom indent - params.update(default_flow_style=False, - indent=__opts__['output_indent']) - else: # no indentation - params.update(default_flow_style=True, - indent=0) - try: - return yaml.dump(data, **params).replace("\n\n", "\n") - # pylint: disable=broad-except - except Exception as exc: - import pprint - log.exception('Exception {0} encountered when trying to serialize {1}'.format( - exc, pprint.pformat(data))) - return "Got an error trying to serialze/clean up the response" + def format_return_text(self, data, **kwargs): # pylint: disable=unused-argument + ''' + Print out YAML using the block mode + ''' + params = dict(Dumper=OrderedDumper) + if 'output_indent' not in __opts__: + # default indentation + params.update(default_flow_style=False) + elif __opts__['output_indent'] >= 0: + # custom indent + params.update(default_flow_style=False, + indent=__opts__['output_indent']) + else: # no indentation + params.update(default_flow_style=True, + indent=0) + try: + #return yaml.dump(data, **params).replace("\n\n", "\n") + return json.dumps(data, sort_keys=True, indent=1) + # pylint: disable=broad-except + except Exception as exc: + import pprint + log.exception('Exception {0} encountered when trying to serialize {1}'.format( + exc, pprint.pformat(data))) + return 'Got an error trying to serialze/clean up the response' + def parse_args_and_kwargs(self, cmdline): + ''' + cmdline: list -def parse_args_and_kwargs(cmdline): - """ - cmdline: list + returns tuple of: args (list), kwargs (dict) + ''' + # Parse args and kwargs + args = [] + kwargs = {} - returns tuple of: args (list), kwargs (dict) - """ - # Parse args and kwargs - args = [] - kwargs = {} + if len(cmdline) > 1: + for item in cmdline[1:]: + if '=' in item: + (key, value) = item.split('=', 1) + kwargs[key] = value + else: + args.append(item) + return (args, kwargs) - if len(cmdline) > 1: - for item in cmdline[1:]: - if '=' in item: - (key, value) = item.split('=', 1) - kwargs[key] = value - else: - args.append(item) - return (args, kwargs) + def get_jobs_from_runner(self, outstanding_jids): + ''' + Given a list of job_ids, return a dictionary of those job_ids that have completed and their results. + Query the salt event bus via the jobs runner. jobs.list_job will show a job in progress, + jobs.lookup_jid will return a job that has completed. -def get_jobs_from_runner(outstanding_jids): - """ - Given a list of job_ids, return a dictionary of those job_ids that have completed and their results. + returns a dictionary of job id: result + ''' + # Can't use the runner because of https://github.com/saltstack/salt/issues/40671 + runner = salt.runner.RunnerClient(__opts__) + # log.debug("Getting job IDs {} will run via runner jobs.lookup_jid".format(outstanding_jids)) + #mm = salt.minion.MasterMinion(__opts__) + source = __opts__.get('ext_job_cache') + if not source: + source = __opts__.get('master_job_cache') - Query the salt event bus via the jobs runner. jobs.list_job will show a job in progress, - jobs.lookup_jid will return a job that has completed. + results = dict() + for jid in outstanding_jids: + # results[jid] = runner.cmd('jobs.lookup_jid', [jid]) + if self.master_minion.returners['{}.get_jid'.format(source)](jid): + jid_result = runner.cmd('jobs.list_job', [jid]).get('Result', {}) + # emulate lookup_jid's return, which is just minion:return + # pylint is tripping + # pylint: disable=missing-whitespace-after-comma + job_data = json.dumps({key:val['return'] for key, val in jid_result.items()}) + results[jid] = yaml.load(job_data) - returns a dictionary of job id: result - """ - # Can't use the runner because of https://github.com/saltstack/salt/issues/40671 - runner = salt.runner.RunnerClient(__opts__) - # log.debug("Getting job IDs {} will run via runner jobs.lookup_jid".format(outstanding_jids)) - mm = salt.minion.MasterMinion(__opts__) - source = __opts__.get('ext_job_cache') - if not source: - source = __opts__.get('master_job_cache') + return results - results = dict() - for jid in outstanding_jids: - # results[jid] = runner.cmd('jobs.lookup_jid', [jid]) - if mm.returners['{}.get_jid'.format(source)](jid): - jid_result = runner.cmd('jobs.list_job', [jid]).get('Result', {}) - # emulate lookup_jid's return, which is just minion:return + def run_commands_from_slack_async(self, message_generator, fire_all, tag, control, interval=1): + ''' + Pull any pending messages from the message_generator, sending each + one to either the event bus, the command_async or both, depending on + the values of fire_all and command + ''' + + outstanding = dict() # set of job_id that we need to check for + + while True: + log.trace('Sleeping for interval of {}'.format(interval)) + time.sleep(interval) + # Drain the slack messages, up to 10 messages at a clip + count = 0 + for msg in message_generator: + # The message_generator yields dicts. Leave this loop + # on a dict that looks like {'done': True} or when we've done it + # 10 times without taking a break. + log.trace('Got a message from the generator: {}'.format(msg.keys())) + if count > 10: + log.warn('Breaking in getting messages because count is exceeded') + break + if len(msg) == 0: + count += 1 + log.warn('len(msg) is zero') + continue # This one is a dud, get the next message + if msg.get('done'): + log.trace('msg is done') + break + if fire_all: + log.debug('Firing message to the bus with tag: {}'.format(tag)) + log.debug('{} {}'.format(tag, msg)) + self.fire('{0}/{1}'.format(tag, msg['message_data'].get('type')), msg) + if control and (len(msg) > 1) and msg.get('cmdline'): + channel = self.sc.server.channels.find(msg['channel']) + jid = self.run_command_async(msg) + log.debug('Submitted a job and got jid: {}'.format(jid)) + outstanding[jid] = msg # record so we can return messages to the caller + channel.send_message("@{}'s job is submitted as salt jid {}".format(msg['user_name'], jid)) + count += 1 + start_time = time.time() + job_status = self.get_jobs_from_runner(outstanding.keys()) # dict of job_ids:results are returned + log.trace('Getting {} jobs status took {} seconds'.format(len(job_status), time.time() - start_time)) + for jid, result in job_status.items(): + if result: + log.debug('ret to send back is {}'.format(result)) + # formatting function? + this_job = outstanding[jid] + channel = self.sc.server.channels.find(this_job['channel']) + return_text = self.format_return_text(result) + return_prefix = "@{}'s job `{}` (id: {}) (target: {}) returned".format( + this_job['user_name'], this_job['cmdline'], jid, this_job['target']) + channel.send_message(return_prefix) + ts = time.time() + st = datetime.datetime.fromtimestamp(ts).strftime('%Y%m%d%H%M%S%f') + filename = 'salt-results-{0}.yaml'.format(st) + r = self.sc.api_call( + 'files.upload', channels=channel.id, filename=filename, + content=return_text) + # Handle unicode return + log.debug('Got back {} via the slack client'.format(r)) + resp = yaml.safe_load(json.dumps(r)) + if 'ok' in resp and resp['ok'] is False: + this_job['channel'].send_message('Error: {0}'.format(resp['error'])) + del outstanding[jid] + + def run_command_async(self, msg): + + ''' + :type message_generator: generator of dict + :param message_generator: Generates messages from slack that should be run + + :type fire_all: bool + :param fire_all: Whether to also fire messages to the event bus + + :type tag: str + :param tag: The tag to send to use to send to the event bus + + :type interval: int + :param interval: time to wait between ending a loop and beginning the next + + ''' + log.debug('Going to run a command async') + runner_functions = sorted(salt.runner.Runner(__opts__).functions) + # Parse args and kwargs + cmd = msg['cmdline'][0] + + args, kwargs = self.parse_args_and_kwargs(msg['cmdline']) + + # Check for pillar string representation of dict and convert it to dict + if 'pillar' in kwargs: + kwargs.update(pillar=ast.literal_eval(kwargs['pillar'])) + + # Check for target. Otherwise assume None + target = msg['target']['target'] + # Check for tgt_type. Otherwise assume glob + tgt_type = msg['target']['tgt_type'] + log.debug('target_type is: {}'.format(tgt_type)) + + if cmd in runner_functions: + runner = salt.runner.RunnerClient(__opts__) + log.debug('Command {} will run via runner_functions'.format(cmd)) # pylint is tripping # pylint: disable=missing-whitespace-after-comma - job_data = json.dumps({key:val['return'] for key, val in jid_result.items()}) - results[jid] = yaml.load(job_data) + job_id_dict = runner.async(cmd, {'args': args, 'kwargs': kwargs}) + job_id = job_id_dict['jid'] - return results - - -def run_commands_from_slack_async(message_generator, fire_all, tag, control, interval=1): - """Pull any pending messages from the message_generator, sending each - one to either the event bus, the command_async or both, depending on - the values of fire_all and command - """ - - outstanding = dict() # set of job_id that we need to check for - - while True: - log.debug("Sleeping for interval of {}".format(interval)) - time.sleep(interval) - # Drain the slack messages, up to 10 messages at a clip - count = 0 - for msg in message_generator: - # The message_generator yields dicts. Leave this loop - # on a dict that looks like {"done": True} or when we've done it - # 10 times without taking a break. - log.debug("Got a message from the generator: {}".format(msg.keys())) - if count > 10: - log.warn("Breaking in getting messages because count is exceeded") - break - if len(msg) == 0: - count += 1 - log.warn("len(msg) is zero") - continue # This one is a dud, get the next message - if msg.get("done"): - log.debug("msg is done") - break - if fire_all: - log.debug("Firing message to the bus with tag: {}".format(tag)) - fire('{0}/{1}'.format(tag, msg['message_data'].get('type')), msg) - if control and (len(msg) > 1) and msg.get('cmdline'): - jid = run_command_async(msg) - log.debug("Submitted a job and got jid: {}".format(jid)) - outstanding[jid] = msg # record so we can return messages to the caller - msg['channel'].send_message("@{}'s job is submitted as salt jid {}".format(msg['user_name'], jid)) - count += 1 - start_time = time.time() - job_status = get_jobs_from_runner(outstanding.keys()) # dict of job_ids:results are returned - log.debug("Getting {} jobs status took {} seconds".format(len(job_status), time.time() - start_time)) - for jid, result in job_status.items(): - if result: - log.debug("ret to send back is {}".format(result)) - # formatting function? - this_job = outstanding[jid] - return_text = format_return_text(result) - return_prefix = "@{}'s job `{}` (id: {}) (target: {}) returned".format( - this_job["user_name"], this_job["cmdline"], jid, this_job["target"]) - this_job['channel'].send_message(return_prefix) - r = this_job["slack_client"].api_call( - "files.upload", channels=this_job['channel'].id, files=None, - content=return_text) - # Handle unicode return - log.debug("Got back {} via the slack client".format(r)) - resp = yaml.safe_load(json.dumps(r)) - if 'ok' in resp and resp['ok'] is False: - this_job['channel'].send_message('Error: {0}'.format(resp['error'])) - del outstanding[jid] - - -def run_command_async(msg): - - """ - :type message_generator: generator of dict - :param message_generator: Generates messages from slack that should be run - - :type fire_all: bool - :param fire_all: Whether to also fire messages to the event bus - - :type tag: str - :param tag: The tag to send to use to send to the event bus - - :type interval: int - :param interval: time to wait between ending a loop and beginning the next - - """ - log.debug("Going to run a command async") - runner_functions = sorted(salt.runner.Runner(__opts__).functions) - # Parse args and kwargs - cmd = msg['cmdline'][0] - - args, kwargs = parse_args_and_kwargs(msg['cmdline']) - # Check for target. Otherwise assume None - target = msg["target"]["target"] - # Check for tgt_type. Otherwise assume glob - tgt_type = msg["target"]['tgt_type'] - log.debug("target_type is: {}".format(tgt_type)) - - if cmd in runner_functions: - runner = salt.runner.RunnerClient(__opts__) - log.debug("Command {} will run via runner_functions".format(cmd)) - # pylint is tripping - # pylint: disable=missing-whitespace-after-comma - job_id_dict = runner.async(cmd, {"args":args, "kwargs":kwargs}) - job_id = job_id_dict['jid'] - - # Default to trying to run as a client module. - else: - local = salt.client.LocalClient() - log.debug("Command {} will run via local.cmd_async, targeting {}".format(cmd, target)) - log.debug("Running {}, {}, {}, {}, {}".format(str(target), cmd, args, kwargs, str(tgt_type))) - # according to https://github.com/saltstack/salt-api/issues/164, tgt_type has changed to expr_form - job_id = local.cmd_async(str(target), cmd, arg=args, kwargs=kwargs, expr_form=str(tgt_type)) - log.info("ret from local.cmd_async is {}".format(job_id)) - return job_id + # Default to trying to run as a client module. + else: + local = salt.client.LocalClient() + log.debug('Command {} will run via local.cmd_async, targeting {}'.format(cmd, target)) + log.debug('Running {}, {}, {}, {}, {}'.format(str(target), cmd, args, kwargs, str(tgt_type))) + # according to https://github.com/saltstack/salt-api/issues/164, tgt_type has changed to expr_form + job_id = local.cmd_async(str(target), cmd, arg=args, kwarg=kwargs, tgt_type=str(tgt_type)) + log.info('ret from local.cmd_async is {}'.format(job_id)) + return job_id def start(token, control=False, - trigger="!", + trigger='!', groups=None, groups_pillar_name=None, fire_all=False, @@ -734,11 +763,12 @@ def start(token, if (not token) or (not token.startswith('xoxb')): time.sleep(2) # don't respawn too quickly - log.error("Slack bot token not found, bailing...") + log.error('Slack bot token not found, bailing...') raise UserWarning('Slack Engine bot token not configured') try: - message_generator = generate_triggered_messages(token, trigger, groups, groups_pillar_name) - run_commands_from_slack_async(message_generator, fire_all, tag, control) + client = SlackClient(token=token) + message_generator = client.generate_triggered_messages(token, trigger, groups, groups_pillar_name) + client.run_commands_from_slack_async(message_generator, fire_all, tag, control) except Exception: - raise Exception("{}".format(traceback.format_exc())) + raise Exception('{}'.format(traceback.format_exc())) diff --git a/salt/engines/stalekey.py b/salt/engines/stalekey.py index 0804206dd86..1887be9fd06 100644 --- a/salt/engines/stalekey.py +++ b/salt/engines/stalekey.py @@ -24,14 +24,14 @@ import time import logging # Import salt libs -import salt.utils.minions import salt.config import salt.key +import salt.utils.files +import salt.utils.minions import salt.wheel -import salt.utils # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six import msgpack log = logging.getLogger(__name__) @@ -59,7 +59,7 @@ def start(interval=3600, expire=604800): minions = {} if os.path.exists(presence_file): try: - with salt.utils.fopen(presence_file, 'r') as f: + with salt.utils.files.fopen(presence_file, 'r') as f: minions = msgpack.load(f) except IOError as e: log.error('Could not open presence file {0}: {1}'.format(presence_file, e)) @@ -94,7 +94,7 @@ def start(interval=3600, expire=604800): del minions[k] try: - with salt.utils.fopen(presence_file, 'w') as f: + with salt.utils.files.fopen(presence_file, 'w') as f: msgpack.dump(minions, f) except IOError as e: log.error('Could not write to presence file {0}: {1}'.format(presence_file, e)) diff --git a/salt/exceptions.py b/salt/exceptions.py index 256537dd77f..6283ddf729d 100644 --- a/salt/exceptions.py +++ b/salt/exceptions.py @@ -11,7 +11,7 @@ import time # Import Salt libs import salt.defaults.exitcodes -import salt.ext.six as six +from salt.ext import six log = logging.getLogger(__name__) @@ -31,14 +31,14 @@ def get_error_message(error): ''' Get human readable message from Python Exception ''' - return error.args[0] if error.args else '' + return error.args[0] if error.args else u'' class SaltException(Exception): ''' Base exception class; all Salt-specific exceptions should subclass this ''' - def __init__(self, message=''): + def __init__(self, message=u''): super(SaltException, self).__init__(message) self.strerror = message @@ -48,7 +48,8 @@ class SaltException(Exception): transport via msgpack ''' if six.PY3: - return {'message': str(self), 'args': self.args} + # The message should be a str type, not a unicode + return {u'message': str(self), u'args': self.args} return dict(message=self.__unicode__(), args=self.args) @@ -99,16 +100,16 @@ class CommandExecutionError(SaltException): Used when a module runs a command which returns an error and wants to show the user the output gracefully instead of dying ''' - def __init__(self, message='', info=None): + def __init__(self, message=u'', info=None): self.error = exc_str_prefix = message self.info = info if self.info: if exc_str_prefix: - if exc_str_prefix[-1] not in '.?!': - exc_str_prefix += '.' - exc_str_prefix += ' ' + if exc_str_prefix[-1] not in u'.?!': + exc_str_prefix += u'.' + exc_str_prefix += u' ' - exc_str_prefix += 'Additional info follows:\n\n' + exc_str_prefix += u'Additional info follows:\n\n' # NOTE: exc_str will be passed to the parent class' constructor and # become self.strerror. exc_str = exc_str_prefix + _nested_output(self.info) @@ -119,7 +120,7 @@ class CommandExecutionError(SaltException): # this information would be redundant). if isinstance(self.info, dict): info_without_changes = copy.deepcopy(self.info) - info_without_changes.pop('changes', None) + info_without_changes.pop(u'changes', None) if info_without_changes: self.strerror_without_changes = \ exc_str_prefix + _nested_output(info_without_changes) @@ -168,9 +169,9 @@ class FileLockError(SaltException): super(FileLockError, self).__init__(msg, *args, **kwargs) if time_start is None: log.warning( - 'time_start should be provided when raising a FileLockError. ' - 'Defaulting to current time as a fallback, but this may ' - 'result in an inaccurate timeout.' + u'time_start should be provided when raising a FileLockError. ' + u'Defaulting to current time as a fallback, but this may ' + u'result in an inaccurate timeout.' ) self.time_start = time.time() else: @@ -223,29 +224,29 @@ class SaltRenderError(SaltException): def __init__(self, message, line_num=None, - buf='', - marker=' <======================', + buf=u'', + marker=u' <======================', trace=None): self.error = message exc_str = copy.deepcopy(message) self.line_num = line_num self.buffer = buf - self.context = '' + self.context = u'' if trace: - exc_str += '\n{0}\n'.format(trace) + exc_str += u'\n{0}\n'.format(trace) if self.line_num and self.buffer: - import salt.utils + import salt.utils.stringutils self.context = salt.utils.get_context( self.buffer, self.line_num, marker=marker ) - exc_str += '; line {0}\n\n{1}'.format( + exc_str += '; line {0}\n\n{1}'.format( # future lint: disable=non-unicode-string self.line_num, - self.context + salt.utils.stringutils.to_str(self.context), ) - SaltException.__init__(self, exc_str) + super(SaltRenderError, self).__init__(exc_str) class SaltClientTimeout(SaltException): @@ -383,6 +384,25 @@ class NotImplemented(SaltException): ''' +class TemplateError(SaltException): + ''' + Used when a custom error is triggered in a template + ''' + + +# Validation related exceptions +class InvalidConfigError(CommandExecutionError): + ''' + Used when the config is invalid + ''' + + +class ArgumentValueError(CommandExecutionError): + ''' + Used when an invalid argument was passed to a command execution + ''' + + # VMware related exceptions class VMwareSaltError(CommandExecutionError): ''' diff --git a/salt/executors/sudo.py b/salt/executors/sudo.py index cbdb4f4d013..61ada996656 100644 --- a/salt/executors/sudo.py +++ b/salt/executors/sudo.py @@ -11,14 +11,14 @@ except ImportError: from pipes import quote as _cmd_quote # Import salt libs -import salt.utils +import salt.utils.path import salt.syspaths __virtualname__ = 'sudo' def __virtual__(): - if salt.utils.which('sudo') and __opts__.get('sudo_user'): + if salt.utils.path.which('sudo') and __opts__.get('sudo_user'): return __virtualname__ return False @@ -60,7 +60,7 @@ def execute(opts, data, func, args, kwargs): '-c', salt.syspaths.CONFIG_DIR, '--', data.get('fun')] - if data['fun'] == 'state.sls': + if data['fun'] in ('state.sls', 'state.highstate', 'state.apply'): kwargs['concurrent'] = True for arg in args: cmd.append(_cmd_quote(str(arg))) diff --git a/salt/ext/ipaddress.py b/salt/ext/ipaddress.py index d2dfc5ee509..75011f76066 100644 --- a/salt/ext/ipaddress.py +++ b/salt/ext/ipaddress.py @@ -28,9 +28,6 @@ bytes = bytearray # Python 2 does not support exception chaining. # s/ from None$// -# Python 2 ranges need to fit in a C long -# 'fix' hosts() for IPv6Network - # When checking for instances of int, also allow Python 2's long. _builtin_isinstance = isinstance @@ -2259,7 +2256,7 @@ class IPv6Network(_BaseV6, _BaseNetwork): """ network = int(self.network_address) broadcast = int(self.broadcast_address) - for x in range(1, broadcast - network + 1): + for x in long_range(1, broadcast - network + 1): yield self._address_class(network + x) @property diff --git a/salt/ext/vsan/__init__.py b/salt/ext/vsan/__init__.py new file mode 100644 index 00000000000..cf17f9d75ee --- /dev/null +++ b/salt/ext/vsan/__init__.py @@ -0,0 +1,7 @@ +# coding: utf-8 -*- +''' +This directory contains the object model and utils for the vsan VMware SDK +extension. + +They are governed under their respective licenses. +''' diff --git a/salt/ext/vsan/vsanapiutils.py b/salt/ext/vsan/vsanapiutils.py new file mode 100644 index 00000000000..6b9b1b826c4 --- /dev/null +++ b/salt/ext/vsan/vsanapiutils.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" +Copyright 2016 VMware, Inc. All rights reserved. + +This module defines basic helper functions used in the sampe codes +""" + +# pylint: skip-file +__author__ = 'VMware, Inc' + +from pyVmomi import vim, vmodl, SoapStubAdapter +#import the VSAN API python bindings +import vsanmgmtObjects + +VSAN_API_VC_SERVICE_ENDPOINT = '/vsanHealth' +VSAN_API_ESXI_SERVICE_ENDPOINT = '/vsan' + +#Constuct a stub for VSAN API access using VC or ESXi sessions from existing +#stubs. Correspoding VC or ESXi service endpoint is required. VC service +#endpoint is used as default +def _GetVsanStub( + stub, endpoint=VSAN_API_VC_SERVICE_ENDPOINT, + context=None, version='vim.version.version10' + ): + + hostname = stub.host.split(':')[0] + vsanStub = SoapStubAdapter( + host=hostname, + path=endpoint, + version=version, + sslContext=context + ) + vsanStub.cookie = stub.cookie + return vsanStub + +#Construct a stub for access VC side VSAN APIs +def GetVsanVcStub(stub, context=None): + return _GetVsanStub(stub, endpoint=VSAN_API_VC_SERVICE_ENDPOINT, + context=context) + +#Construct a stub for access ESXi side VSAN APIs +def GetVsanEsxStub(stub, context=None): + return _GetVsanStub(stub, endpoint=VSAN_API_ESXI_SERVICE_ENDPOINT, + context=context) + +#Construct a stub for access ESXi side VSAN APIs +def GetVsanVcMos(vcStub, context=None): + vsanStub = GetVsanVcStub(vcStub, context) + vcMos = { + 'vsan-disk-management-system' : vim.cluster.VsanVcDiskManagementSystem( + 'vsan-disk-management-system', + vsanStub + ), + 'vsan-stretched-cluster-system' : vim.cluster.VsanVcStretchedClusterSystem( + 'vsan-stretched-cluster-system', + vsanStub + ), + 'vsan-cluster-config-system' : vim.cluster.VsanVcClusterConfigSystem( + 'vsan-cluster-config-system', + vsanStub + ), + 'vsan-performance-manager' : vim.cluster.VsanPerformanceManager( + 'vsan-performance-manager', + vsanStub + ), + 'vsan-cluster-health-system' : vim.cluster.VsanVcClusterHealthSystem( + 'vsan-cluster-health-system', + vsanStub + ), + 'vsan-upgrade-systemex' : vim.VsanUpgradeSystemEx( + 'vsan-upgrade-systemex', + vsanStub + ), + 'vsan-cluster-space-report-system' : vim.cluster.VsanSpaceReportSystem( + 'vsan-cluster-space-report-system', + vsanStub + ), + + 'vsan-cluster-object-system' : vim.cluster.VsanObjectSystem( + 'vsan-cluster-object-system', + vsanStub + ), + } + + return vcMos + +#Construct a stub for access ESXi side VSAN APIs +def GetVsanEsxMos(esxStub, context=None): + vsanStub = GetVsanEsxStub(esxStub, context) + esxMos = { + 'vsan-performance-manager' : vim.cluster.VsanPerformanceManager( + 'vsan-performance-manager', + vsanStub + ), + 'ha-vsan-health-system' : vim.host.VsanHealthSystem( + 'ha-vsan-health-system', + vsanStub + ), + 'vsan-object-system' : vim.cluster.VsanObjectSystem( + 'vsan-object-system', + vsanStub + ), + } + + return esxMos + +#Convert a VSAN Task to a Task MO binding to VC service +#@param vsanTask the VSAN Task MO +#@param stub the stub for the VC API +def ConvertVsanTaskToVcTask(vsanTask, vcStub): + vcTask = vim.Task(vsanTask._moId, vcStub) + return vcTask + +def WaitForTasks(tasks, si): + """ + Given the service instance si and tasks, it returns after all the + tasks are complete + """ + + pc = si.content.propertyCollector + + taskList = [str(task) for task in tasks] + + # Create filter + objSpecs = [vmodl.query.PropertyCollector.ObjectSpec(obj=task) + for task in tasks] + propSpec = vmodl.query.PropertyCollector.PropertySpec(type=vim.Task, + pathSet=[], all=True) + filterSpec = vmodl.query.PropertyCollector.FilterSpec() + filterSpec.objectSet = objSpecs + filterSpec.propSet = [propSpec] + filter = pc.CreateFilter(filterSpec, True) + + try: + version, state = None, None + + # Loop looking for updates till the state moves to a completed state. + while len(taskList): + update = pc.WaitForUpdates(version) + for filterSet in update.filterSet: + for objSet in filterSet.objectSet: + task = objSet.obj + for change in objSet.changeSet: + if change.name == 'info': + state = change.val.state + elif change.name == 'info.state': + state = change.val + else: + continue + + if not str(task) in taskList: + continue + + if state == vim.TaskInfo.State.success: + # Remove task from taskList + taskList.remove(str(task)) + elif state == vim.TaskInfo.State.error: + raise task.info.error + # Move to next version + version = update.version + finally: + if filter: + filter.Destroy() + diff --git a/salt/ext/win_inet_pton.py b/salt/ext/win_inet_pton.py index 68d5b1b3753..bd9cfcb0b4d 100644 --- a/salt/ext/win_inet_pton.py +++ b/salt/ext/win_inet_pton.py @@ -9,6 +9,7 @@ from __future__ import absolute_import import socket import ctypes import os +import ipaddress class sockaddr(ctypes.Structure): @@ -31,6 +32,24 @@ else: def inet_pton(address_family, ip_string): + # Verify IP Address + # This will catch IP Addresses such as 10.1.2 + if address_family == socket.AF_INET: + try: + ipaddress.ip_address(ip_string.decode()) + except ValueError: + raise socket.error('illegal IP address string passed to inet_pton') + return socket.inet_aton(ip_string) + + # Verify IP Address + # The `WSAStringToAddressA` function handles notations used by Berkeley + # software which includes 3 part IP Addresses such as `10.1.2`. That's why + # the above check is needed to enforce more strict IP Address validation as + # used by the `inet_pton` function in Unix. + # See the following: + # https://stackoverflow.com/a/29286098 + # Docs for the `inet_addr` function on MSDN + # https://msdn.microsoft.com/en-us/library/windows/desktop/ms738563.aspx addr = sockaddr() addr.sa_family = address_family addr_size = ctypes.c_int(ctypes.sizeof(addr)) diff --git a/salt/fileclient.py b/salt/fileclient.py index 33a16b56941..cb3b210a037 100644 --- a/salt/fileclient.py +++ b/salt/fileclient.py @@ -26,15 +26,19 @@ import salt.transport import salt.fileserver import salt.utils import salt.utils.files -import salt.utils.templates -import salt.utils.url import salt.utils.gzip_util import salt.utils.http -import salt.ext.six as six +import salt.utils.path +import salt.utils.platform +import salt.utils.stringutils +import salt.utils.templates +import salt.utils.url +import salt.utils.versions from salt.utils.locales import sdecode from salt.utils.openstack.swift import SaltSwift # pylint: disable=no-name-in-module,import-error +from salt.ext import six import salt.ext.six.moves.BaseHTTPServer as BaseHTTPServer from salt.ext.six.moves.urllib.error import HTTPError, URLError from salt.ext.six.moves.urllib.parse import urlparse, urlunparse @@ -48,13 +52,13 @@ def get_file_client(opts, pillar=False): Read in the ``file_client`` option and return the correct type of file server ''' - client = opts.get('file_client', 'remote') - if pillar and client == 'local': - client = 'pillar' + client = opts.get(u'file_client', u'remote') + if pillar and client == u'local': + client = u'pillar' return { - 'remote': RemoteClient, - 'local': FSClient, - 'pillar': LocalClient, + u'remote': RemoteClient, + u'local': FSClient, + u'pillar': LocalClient, }.get(client, RemoteClient)(opts) @@ -95,16 +99,16 @@ class Client(object): def __setstate__(self, state): # This will polymorphically call __init__ # in the derived class. - self.__init__(state['opts']) + self.__init__(state[u'opts']) def __getstate__(self): - return {'opts': self.opts} + return {u'opts': self.opts} def _check_proto(self, path): ''' Make sure that this path is intended for the salt master and trim it ''' - if not path.startswith('salt://'): + if not path.startswith(u'salt://'): raise MinionError(u'Unsupported path: {0}'.format(path)) file_path, saltenv = salt.utils.url.parse(path) return file_path @@ -128,13 +132,13 @@ class Client(object): return filelist @contextlib.contextmanager - def _cache_loc(self, path, saltenv='base', cachedir=None): + def _cache_loc(self, path, saltenv=u'base', cachedir=None): ''' Return the local location to cache the file, cache dirs will be made ''' cachedir = self.get_cachedir(cachedir) - dest = salt.utils.path_join(cachedir, - 'files', + dest = salt.utils.path.join(cachedir, + u'files', saltenv, path) destdir = os.path.dirname(dest) @@ -157,16 +161,16 @@ class Client(object): def get_cachedir(self, cachedir=None): if cachedir is None: - cachedir = self.opts['cachedir'] + cachedir = self.opts[u'cachedir'] elif not os.path.isabs(cachedir): - cachedir = os.path.join(self.opts['cachedir'], cachedir) + cachedir = os.path.join(self.opts[u'cachedir'], cachedir) return cachedir def get_file(self, path, - dest='', + dest=u'', makedirs=False, - saltenv='base', + saltenv=u'base', gzip=None, cachedir=None): ''' @@ -175,32 +179,32 @@ class Client(object): ''' raise NotImplementedError - def file_list_emptydirs(self, saltenv='base', prefix=''): + def file_list_emptydirs(self, saltenv=u'base', prefix=u''): ''' List the empty dirs ''' raise NotImplementedError - def cache_file(self, path, saltenv='base', cachedir=None): + def cache_file(self, path, saltenv=u'base', cachedir=None): ''' Pull a file down from the file server and store it in the minion file cache ''' - return self.get_url(path, '', True, saltenv, cachedir=cachedir) + return self.get_url(path, u'', True, saltenv, cachedir=cachedir) - def cache_files(self, paths, saltenv='base', cachedir=None): + def cache_files(self, paths, saltenv=u'base', cachedir=None): ''' Download a list of files stored on the master and put them in the minion file cache ''' ret = [] - if isinstance(paths, str): - paths = paths.split(',') + if isinstance(paths, six.string_types): + paths = paths.split(u',') for path in paths: ret.append(self.cache_file(path, saltenv, cachedir=cachedir)) return ret - def cache_master(self, saltenv='base', cachedir=None): + def cache_master(self, saltenv=u'base', cachedir=None): ''' Download and cache all files on a master in a specified environment ''' @@ -212,7 +216,7 @@ class Client(object): ) return ret - def cache_dir(self, path, saltenv='base', include_empty=False, + def cache_dir(self, path, saltenv=u'base', include_empty=False, include_pat=None, exclude_pat=None, cachedir=None): ''' Download all of the files in a subdir of the master @@ -223,13 +227,11 @@ class Client(object): # We want to make sure files start with this *directory*, use # '/' explicitly because the master (that's generating the # list of files) only runs on POSIX - if not path.endswith('/'): - path = path + '/' + if not path.endswith(u'/'): + path = path + u'/' log.info( - 'Caching directory \'{0}\' for environment \'{1}\''.format( - path, saltenv - ) + u'Caching directory \'%s\' for environment \'%s\'', path, saltenv ) # go through the list of all files finding ones that are in # the target directory and caching them @@ -255,11 +257,11 @@ class Client(object): # prefix = separated[0] cachedir = self.get_cachedir(cachedir) - dest = salt.utils.path_join(cachedir, 'files', saltenv) + dest = salt.utils.path.join(cachedir, u'files', saltenv) for fn_ in self.file_list_emptydirs(saltenv): fn_ = sdecode(fn_) if fn_.startswith(path): - minion_dir = '{0}/{1}'.format(dest, fn_) + minion_dir = u'{0}/{1}'.format(dest, fn_) if not os.path.isdir(minion_dir): os.makedirs(minion_dir) ret.append(minion_dir) @@ -269,8 +271,8 @@ class Client(object): ''' Cache a local file on the minion in the localfiles cache ''' - dest = os.path.join(self.opts['cachedir'], 'localfiles', - path.lstrip('/')) + dest = os.path.join(self.opts[u'cachedir'], u'localfiles', + path.lstrip(u'/')) destdir = os.path.dirname(dest) if not os.path.isdir(destdir): @@ -279,41 +281,41 @@ class Client(object): shutil.copyfile(path, dest) return dest - def file_local_list(self, saltenv='base'): + def file_local_list(self, saltenv=u'base'): ''' List files in the local minion files and localfiles caches ''' - filesdest = os.path.join(self.opts['cachedir'], 'files', saltenv) - localfilesdest = os.path.join(self.opts['cachedir'], 'localfiles') + filesdest = os.path.join(self.opts[u'cachedir'], u'files', saltenv) + localfilesdest = os.path.join(self.opts[u'cachedir'], u'localfiles') fdest = self._file_local_list(filesdest) ldest = self._file_local_list(localfilesdest) return sorted(fdest.union(ldest)) - def file_list(self, saltenv='base', prefix=''): + def file_list(self, saltenv=u'base', prefix=u''): ''' This function must be overwritten ''' return [] - def dir_list(self, saltenv='base', prefix=''): + def dir_list(self, saltenv=u'base', prefix=u''): ''' This function must be overwritten ''' return [] - def symlink_list(self, saltenv='base', prefix=''): + def symlink_list(self, saltenv=u'base', prefix=u''): ''' This function must be overwritten ''' return {} - def is_cached(self, path, saltenv='base', cachedir=None): + def is_cached(self, path, saltenv=u'base', cachedir=None): ''' Returns the full path to a file if it is cached locally on the minion otherwise returns a blank string ''' - if path.startswith('salt://'): + if path.startswith(u'salt://'): path, senv = salt.utils.url.parse(path) if senv: saltenv = senv @@ -322,9 +324,9 @@ class Client(object): # also strip escape character '|' localsfilesdest = os.path.join( - self.opts['cachedir'], 'localfiles', path.lstrip('|/')) + self.opts[u'cachedir'], u'localfiles', path.lstrip(u'|/')) filesdest = os.path.join( - self.opts['cachedir'], 'files', saltenv, path.lstrip('|/')) + self.opts[u'cachedir'], u'files', saltenv, path.lstrip(u'|/')) extrndest = self._extrn_path(path, saltenv, cachedir=cachedir) if os.path.exists(filesdest): @@ -336,7 +338,7 @@ class Client(object): elif os.path.exists(extrndest): return extrndest - return '' + return u'' def list_states(self, saltenv): ''' @@ -344,51 +346,53 @@ class Client(object): environment ''' - limit_traversal = self.opts.get('fileserver_limit_traversal', False) + limit_traversal = self.opts.get(u'fileserver_limit_traversal', False) states = [] if limit_traversal: - if saltenv not in self.opts['file_roots']: + if saltenv not in self.opts[u'file_roots']: log.warning( - 'During an attempt to list states for saltenv \'{0}\', ' - 'the environment could not be found in the configured ' - 'file roots'.format(saltenv) + u'During an attempt to list states for saltenv \'%s\', ' + u'the environment could not be found in the configured ' + u'file roots', saltenv ) return states - for path in self.opts['file_roots'][saltenv]: + for path in self.opts[u'file_roots'][saltenv]: for root, dirs, files in os.walk(path, topdown=True): - log.debug('Searching for states in dirs {0} and files ' - '{1}'.format(dirs, files)) - if not [filename.endswith('.sls') for filename in files]: + log.debug( + u'Searching for states in dirs %s and files %s', + dirs, files + ) + if not [filename.endswith(u'.sls') for filename in files]: # Use shallow copy so we don't disturb the memory used by os.walk. Otherwise this breaks! del dirs[:] else: for found_file in files: stripped_root = os.path.relpath(root, path) - if salt.utils.is_windows(): - stripped_root = stripped_root.replace('\\', '/') - stripped_root = stripped_root.replace('/', '.') - if found_file.endswith(('.sls')): - if found_file.endswith('init.sls'): - if stripped_root.endswith('.'): - stripped_root = stripped_root.rstrip('.') + if salt.utils.platform.is_windows(): + stripped_root = stripped_root.replace(u'\\', u'/') + stripped_root = stripped_root.replace(u'/', u'.') + if found_file.endswith((u'.sls')): + if found_file.endswith(u'init.sls'): + if stripped_root.endswith(u'.'): + stripped_root = stripped_root.rstrip(u'.') states.append(stripped_root) else: - if not stripped_root.endswith('.'): - stripped_root += '.' - if stripped_root.startswith('.'): - stripped_root = stripped_root.lstrip('.') + if not stripped_root.endswith(u'.'): + stripped_root += u'.' + if stripped_root.startswith(u'.'): + stripped_root = stripped_root.lstrip(u'.') states.append(stripped_root + found_file[:-4]) else: for path in self.file_list(saltenv): - if salt.utils.is_windows(): - path = path.replace('\\', '/') - if path.endswith('.sls'): + if salt.utils.platform.is_windows(): + path = path.replace(u'\\', u'/') + if path.endswith(u'.sls'): # is an sls module! - if path.endswith('{0}init.sls'.format('/')): - states.append(path.replace('/', '.')[:-9]) + if path.endswith(u'/init.sls'): + states.append(path.replace(u'/', u'.')[:-9]) else: - states.append(path.replace('/', '.')[:-4]) + states.append(path.replace(u'/', u'.')[:-4]) return states def get_state(self, sls, saltenv, cachedir=None): @@ -396,31 +400,31 @@ class Client(object): Get a state file from the master and store it in the local minion cache; return the location of the file ''' - if '.' in sls: - sls = sls.replace('.', '/') - sls_url = salt.utils.url.create(sls + '.sls') - init_url = salt.utils.url.create(sls + '/init.sls') + if u'.' in sls: + sls = sls.replace(u'.', u'/') + sls_url = salt.utils.url.create(sls + u'.sls') + init_url = salt.utils.url.create(sls + u'/init.sls') for path in [sls_url, init_url]: dest = self.cache_file(path, saltenv, cachedir=cachedir) if dest: - return {'source': path, 'dest': dest} + return {u'source': path, u'dest': dest} return {} - def get_dir(self, path, dest='', saltenv='base', gzip=None, + def get_dir(self, path, dest=u'', saltenv=u'base', gzip=None, cachedir=None): ''' Get a directory recursively from the salt-master ''' ret = [] # Strip trailing slash - path = self._check_proto(path).rstrip('/') + path = self._check_proto(path).rstrip(u'/') # Break up the path into a list containing the bottom-level directory # (the one being recursively copied) and the directories preceding it - separated = path.rsplit('/', 1) + separated = path.rsplit(u'/', 1) if len(separated) != 2: # No slashes in path. (This means all files in saltenv will be # copied) - prefix = '' + prefix = u'' else: prefix = separated[0] @@ -429,17 +433,17 @@ class Client(object): # Prevent files in "salt://foobar/" (or salt://foo.sh) from # matching a path of "salt://foo" try: - if fn_[len(path)] != '/': + if fn_[len(path)] != u'/': continue except IndexError: continue # Remove the leading directories from path to derive # the relative path on the minion. - minion_relpath = fn_[len(prefix):].lstrip('/') + minion_relpath = fn_[len(prefix):].lstrip(u'/') ret.append( self.get_file( salt.utils.url.create(fn_), - '{0}/{1}'.format(dest, minion_relpath), + u'{0}/{1}'.format(dest, minion_relpath), True, saltenv, gzip ) ) @@ -449,14 +453,14 @@ class Client(object): # Prevent an empty dir "salt://foobar/" from matching a path of # "salt://foo" try: - if fn_[len(path)] != '/': + if fn_[len(path)] != u'/': continue except IndexError: continue # Remove the leading directories from path to derive # the relative path on the minion. - minion_relpath = fn_[len(prefix):].lstrip('/') - minion_mkdir = '{0}/{1}'.format(dest, minion_relpath) + minion_relpath = fn_[len(prefix):].lstrip(u'/') + minion_mkdir = u'{0}/{1}'.format(dest, minion_relpath) if not os.path.isdir(minion_mkdir): os.makedirs(minion_mkdir) ret.append(minion_mkdir) @@ -465,7 +469,7 @@ class Client(object): ret.sort() return ret - def get_url(self, url, dest, makedirs=False, saltenv='base', + def get_url(self, url, dest, makedirs=False, saltenv=u'base', no_cache=False, cachedir=None): ''' Get a single file from a URL. @@ -475,26 +479,39 @@ class Client(object): url_path = os.path.join( url_data.netloc, url_data.path).rstrip(os.sep) - if url_scheme and url_scheme.lower() in string.ascii_lowercase: - url_path = ':'.join((url_scheme, url_path)) - url_scheme = 'file' + # If dest is a directory, rewrite dest with filename + if dest is not None \ + and (os.path.isdir(dest) or dest.endswith((u'/', u'\\'))): + if url_data.query or len(url_data.path) > 1 and not url_data.path.endswith(u'/'): + strpath = url.split(u'/')[-1] + else: + strpath = u'index.html' - if url_scheme in ('file', ''): + if salt.utils.platform.is_windows(): + strpath = salt.utils.sanitize_win_path_string(strpath) + + dest = os.path.join(dest, strpath) + + if url_scheme and url_scheme.lower() in string.ascii_lowercase: + url_path = u':'.join((url_scheme, url_path)) + url_scheme = u'file' + + if url_scheme in (u'file', u''): # Local filesystem if not os.path.isabs(url_path): raise CommandExecutionError( - 'Path \'{0}\' is not absolute'.format(url_path) + u'Path \'{0}\' is not absolute'.format(url_path) ) if dest is None: - with salt.utils.fopen(url_path, 'r') as fp_: + with salt.utils.files.fopen(url_path, u'r') as fp_: data = fp_.read() return data return url_path - if url_scheme == 'salt': + if url_scheme == u'salt': result = self.get_file(url, dest, makedirs, saltenv, cachedir=cachedir) if result and dest is None: - with salt.utils.fopen(result, 'r') as fp_: + with salt.utils.files.fopen(result, u'r') as fp_: data = fp_.read() return data return result @@ -505,87 +522,89 @@ class Client(object): if makedirs: os.makedirs(destdir) else: - return '' + return u'' elif not no_cache: dest = self._extrn_path(url, saltenv, cachedir=cachedir) destdir = os.path.dirname(dest) if not os.path.isdir(destdir): os.makedirs(destdir) - if url_data.scheme == 's3': + if url_data.scheme == u's3': try: def s3_opt(key, default=None): - '''Get value of s3. from Minion config or from Pillar''' - if 's3.' + key in self.opts: - return self.opts['s3.' + key] + u'''Get value of s3. from Minion config or from Pillar''' + if u's3.' + key in self.opts: + return self.opts[u's3.' + key] try: - return self.opts['pillar']['s3'][key] + return self.opts[u'pillar'][u's3'][key] except (KeyError, TypeError): return default - self.utils['s3.query'](method='GET', + self.utils[u's3.query'](method=u'GET', bucket=url_data.netloc, path=url_data.path[1:], return_bin=False, local_file=dest, action=None, - key=s3_opt('key'), - keyid=s3_opt('keyid'), - service_url=s3_opt('service_url'), - verify_ssl=s3_opt('verify_ssl', True), - location=s3_opt('location'), - path_style=s3_opt('path_style', False), - https_enable=s3_opt('https_enable', True)) + key=s3_opt(u'key'), + keyid=s3_opt(u'keyid'), + service_url=s3_opt(u'service_url'), + verify_ssl=s3_opt(u'verify_ssl', True), + location=s3_opt(u'location'), + path_style=s3_opt(u'path_style', False), + https_enable=s3_opt(u'https_enable', True)) return dest except Exception as exc: raise MinionError( - 'Could not fetch from {0}. Exception: {1}'.format(url, exc) + u'Could not fetch from {0}. Exception: {1}'.format(url, exc) ) - if url_data.scheme == 'ftp': + if url_data.scheme == u'ftp': try: ftp = ftplib.FTP() ftp.connect(url_data.hostname, url_data.port) ftp.login(url_data.username, url_data.password) - with salt.utils.fopen(dest, 'wb') as fp_: - ftp.retrbinary('RETR {0}'.format(url_data.path), fp_.write) + with salt.utils.files.fopen(dest, u'wb') as fp_: + ftp.retrbinary(u'RETR {0}'.format(url_data.path), fp_.write) ftp.quit() return dest except Exception as exc: - raise MinionError('Could not retrieve {0} from FTP server. Exception: {1}'.format(url, exc)) + raise MinionError(u'Could not retrieve {0} from FTP server. Exception: {1}'.format(url, exc)) - if url_data.scheme == 'swift': + if url_data.scheme == u'swift': try: def swift_opt(key, default): - '''Get value of from Minion config or from Pillar''' + ''' + Get value of from Minion config or from Pillar + ''' if key in self.opts: return self.opts[key] try: - return self.opts['pillar'][key] + return self.opts[u'pillar'][key] except (KeyError, TypeError): return default - swift_conn = SaltSwift(swift_opt('keystone.user', None), - swift_opt('keystone.tenant', None), - swift_opt('keystone.auth_url', None), - swift_opt('keystone.password', None)) + swift_conn = SaltSwift(swift_opt(u'keystone.user', None), + swift_opt(u'keystone.tenant', None), + swift_opt(u'keystone.auth_url', None), + swift_opt(u'keystone.password', None)) swift_conn.get_object(url_data.netloc, url_data.path[1:], dest) return dest except Exception: - raise MinionError('Could not fetch from {0}'.format(url)) + raise MinionError(u'Could not fetch from {0}'.format(url)) get_kwargs = {} if url_data.username is not None \ - and url_data.scheme in ('http', 'https'): + and url_data.scheme in (u'http', u'https'): netloc = url_data.netloc - at_sign_pos = netloc.rfind('@') + at_sign_pos = netloc.rfind(u'@') if at_sign_pos != -1: netloc = netloc[at_sign_pos + 1:] fixed_url = urlunparse( (url_data.scheme, netloc, url_data.path, url_data.params, url_data.query, url_data.fragment)) - get_kwargs['auth'] = (url_data.username, url_data.password) + get_kwargs[u'auth'] = (url_data.username, url_data.password) else: fixed_url = url @@ -621,19 +640,26 @@ class Client(object): def on_header(hdr): if write_body[1] is not False and write_body[2] is None: + if not hdr.strip() and 'Content-Type' not in write_body[1]: + # We've reached the end of the headers and not yet + # found the Content-Type. Reset the values we're + # tracking so that we properly follow the redirect. + write_body[0] = None + write_body[1] = False + return # Try to find out what content type encoding is used if # this is a text file write_body[1].parse_line(hdr) # pylint: disable=no-member - if 'Content-Type' in write_body[1]: - content_type = write_body[1].get('Content-Type') # pylint: disable=no-member - if not content_type.startswith('text'): + if u'Content-Type' in write_body[1]: + content_type = write_body[1].get(u'Content-Type') # pylint: disable=no-member + if not content_type.startswith(u'text'): write_body[1] = write_body[2] = False else: - encoding = 'utf-8' - fields = content_type.split(';') + encoding = u'utf-8' + fields = content_type.split(u';') for field in fields: - if 'encoding' in field: - encoding = field.split('encoding=')[-1] + if u'encoding' in field: + encoding = field.split(u'encoding=')[-1] write_body[2] = encoding # We have found our encoding. Stop processing headers. write_body[1] = False @@ -664,10 +690,10 @@ class Client(object): chunk = chunk.decode(write_body[2]) result.append(chunk) else: - dest_tmp = "{0}.part".format(dest) + dest_tmp = u"{0}.part".format(dest) # We need an open filehandle to use in the on_chunk callback, # that's why we're not using a with clause here. - destfp = salt.utils.fopen(dest_tmp, 'wb') # pylint: disable=resource-leakage + destfp = salt.utils.files.fopen(dest_tmp, u'wb') # pylint: disable=resource-leakage def on_chunk(chunk): if write_body[0]: @@ -683,24 +709,24 @@ class Client(object): opts=self.opts, **get_kwargs ) - if 'handle' not in query: - raise MinionError('Error: {0} reading {1}'.format(query['error'], url)) + if u'handle' not in query: + raise MinionError(u'Error: {0} reading {1}'.format(query[u'error'], url)) if no_cache: if write_body[2]: - return six.u('').join(result) - return six.b('').join(result) + return u''.join(result) + return six.b(u'').join(result) else: destfp.close() destfp = None salt.utils.files.rename(dest_tmp, dest) return dest except HTTPError as exc: - raise MinionError('HTTP error {0} reading {1}: {3}'.format( + raise MinionError(u'HTTP error {0} reading {1}: {3}'.format( exc.code, url, *BaseHTTPServer.BaseHTTPRequestHandler.responses[exc.code])) except URLError as exc: - raise MinionError('Error reading {0}: {1}'.format(url, exc.reason)) + raise MinionError(u'Error reading {0}: {1}'.format(url, exc.reason)) finally: if destfp is not None: destfp.close() @@ -709,45 +735,38 @@ class Client(object): self, url, dest, - template='jinja', + template=u'jinja', makedirs=False, - saltenv='base', + saltenv=u'base', cachedir=None, **kwargs): ''' Cache a file then process it as a template ''' - if 'env' in kwargs: - salt.utils.warn_until( - 'Oxygen', - 'Parameter \'env\' has been detected in the argument list. This ' - 'parameter is no longer used and has been replaced by \'saltenv\' ' - 'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.' - ) - kwargs.pop('env') + if u'env' in kwargs: + # "env" is not supported; Use "saltenv". + kwargs.pop(u'env') - kwargs['saltenv'] = saltenv + kwargs[u'saltenv'] = saltenv url_data = urlparse(url) sfn = self.cache_file(url, saltenv, cachedir=cachedir) if not os.path.exists(sfn): - return '' + return u'' if template in salt.utils.templates.TEMPLATE_REGISTRY: data = salt.utils.templates.TEMPLATE_REGISTRY[template]( sfn, **kwargs ) else: - log.error('Attempted to render template with unavailable engine ' - '{0}'.format(template)) - return '' - if not data['result']: - # Failed to render the template log.error( - 'Failed to render template with error: {0}'.format( - data['data'] - ) + u'Attempted to render template with unavailable engine %s', + template ) - return '' + return u'' + if not data[u'result']: + # Failed to render the template + log.error(u'Failed to render template with error: %s', data[u'data']) + return u'' if not dest: # No destination passed, set the dest as an extrn_files cache dest = self._extrn_path(url, saltenv, cachedir=cachedir) @@ -759,9 +778,9 @@ class Client(object): if makedirs: os.makedirs(destdir) else: - salt.utils.safe_rm(data['data']) - return '' - shutil.move(data['data'], dest) + salt.utils.files.safe_rm(data[u'data']) + return u'' + shutil.move(data[u'data'], dest) return dest def _extrn_path(self, url, saltenv, cachedir=None): @@ -769,27 +788,27 @@ class Client(object): Return the extn_filepath for a given url ''' url_data = urlparse(url) - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): netloc = salt.utils.sanitize_win_path_string(url_data.netloc) else: netloc = url_data.netloc # Strip user:pass from URLs - netloc = netloc.split('@')[-1] + netloc = netloc.split(u'@')[-1] if cachedir is None: - cachedir = self.opts['cachedir'] + cachedir = self.opts[u'cachedir'] elif not os.path.isabs(cachedir): - cachedir = os.path.join(self.opts['cachedir'], cachedir) + cachedir = os.path.join(self.opts[u'cachedir'], cachedir) if url_data.query: - file_name = '-'.join([url_data.path, url_data.query]) + file_name = u'-'.join([url_data.path, url_data.query]) else: file_name = url_data.path - return salt.utils.path_join( + return salt.utils.path.join( cachedir, - 'extrn_files', + u'extrn_files', saltenv, netloc, file_name @@ -803,31 +822,31 @@ class LocalClient(Client): def __init__(self, opts): Client.__init__(self, opts) - def _find_file(self, path, saltenv='base'): + def _find_file(self, path, saltenv=u'base'): ''' Locate the file path ''' - fnd = {'path': '', - 'rel': ''} + fnd = {u'path': u'', + u'rel': u''} - if saltenv not in self.opts['file_roots']: + if saltenv not in self.opts[u'file_roots']: return fnd if salt.utils.url.is_escaped(path): # The path arguments are escaped path = salt.utils.url.unescape(path) - for root in self.opts['file_roots'][saltenv]: + for root in self.opts[u'file_roots'][saltenv]: full = os.path.join(root, path) if os.path.isfile(full): - fnd['path'] = full - fnd['rel'] = path + fnd[u'path'] = full + fnd[u'rel'] = path return fnd return fnd def get_file(self, path, - dest='', + dest=u'', makedirs=False, - saltenv='base', + saltenv=u'base', gzip=None, cachedir=None): ''' @@ -836,22 +855,22 @@ class LocalClient(Client): ''' path = self._check_proto(path) fnd = self._find_file(path, saltenv) - fnd_path = fnd.get('path') + fnd_path = fnd.get(u'path') if not fnd_path: - return '' + return u'' return fnd_path - def file_list(self, saltenv='base', prefix=''): + def file_list(self, saltenv=u'base', prefix=u''): ''' Return a list of files in the given environment with optional relative prefix path to limit directory traversal ''' ret = [] - if saltenv not in self.opts['file_roots']: + if saltenv not in self.opts[u'file_roots']: return ret - prefix = prefix.strip('/') - for path in self.opts['file_roots'][saltenv]: + prefix = prefix.strip(u'/') + for path in self.opts[u'file_roots'][saltenv]: for root, dirs, files in os.walk( os.path.join(path, prefix), followlinks=True ): @@ -862,16 +881,16 @@ class LocalClient(Client): ret.append(sdecode(relpath)) return ret - def file_list_emptydirs(self, saltenv='base', prefix=''): + def file_list_emptydirs(self, saltenv=u'base', prefix=u''): ''' List the empty dirs in the file_roots with optional relative prefix path to limit directory traversal ''' ret = [] - prefix = prefix.strip('/') - if saltenv not in self.opts['file_roots']: + prefix = prefix.strip(u'/') + if saltenv not in self.opts[u'file_roots']: return ret - for path in self.opts['file_roots'][saltenv]: + for path in self.opts[u'file_roots'][saltenv]: for root, dirs, files in os.walk( os.path.join(path, prefix), followlinks=True ): @@ -881,23 +900,23 @@ class LocalClient(Client): ret.append(sdecode(os.path.relpath(root, path))) return ret - def dir_list(self, saltenv='base', prefix=''): + def dir_list(self, saltenv=u'base', prefix=u''): ''' List the dirs in the file_roots with optional relative prefix path to limit directory traversal ''' ret = [] - if saltenv not in self.opts['file_roots']: + if saltenv not in self.opts[u'file_roots']: return ret - prefix = prefix.strip('/') - for path in self.opts['file_roots'][saltenv]: + prefix = prefix.strip(u'/') + for path in self.opts[u'file_roots'][saltenv]: for root, dirs, files in os.walk( os.path.join(path, prefix), followlinks=True ): ret.append(sdecode(os.path.relpath(root, path))) return ret - def __get_file_path(self, path, saltenv='base'): + def __get_file_path(self, path, saltenv=u'base'): ''' Return either a file path or the result of a remote find_file call. ''' @@ -906,14 +925,16 @@ class LocalClient(Client): except MinionError as err: # Local file path if not os.path.isfile(path): - msg = 'specified file {0} is not present to generate hash: {1}' - log.warning(msg.format(path, err)) + log.warning( + u'specified file %s is not present to generate hash: %s', + path, err + ) return None else: return path return self._find_file(path, saltenv) - def hash_file(self, path, saltenv='base'): + def hash_file(self, path, saltenv=u'base'): ''' Return the hash of a file, to get the hash of a file in the file_roots prepend the path with salt:// otherwise, prepend the @@ -926,17 +947,17 @@ class LocalClient(Client): try: # Remote file path (self._find_file() invoked) - fnd_path = fnd['path'] + fnd_path = fnd[u'path'] except TypeError: # Local file path fnd_path = fnd - hash_type = self.opts.get('hash_type', 'md5') - ret['hsum'] = salt.utils.get_hash(fnd_path, form=hash_type) - ret['hash_type'] = hash_type + hash_type = self.opts.get(u'hash_type', u'md5') + ret[u'hsum'] = salt.utils.get_hash(fnd_path, form=hash_type) + ret[u'hash_type'] = hash_type return ret - def hash_and_stat_file(self, path, saltenv='base'): + def hash_and_stat_file(self, path, saltenv=u'base'): ''' Return the hash of a file, to get the hash of a file in the file_roots prepend the path with salt:// otherwise, prepend the @@ -952,8 +973,8 @@ class LocalClient(Client): try: # Remote file path (self._find_file() invoked) - fnd_path = fnd['path'] - fnd_stat = fnd.get('stat') + fnd_path = fnd[u'path'] + fnd_stat = fnd.get(u'stat') except TypeError: # Local file path fnd_path = fnd @@ -962,12 +983,12 @@ class LocalClient(Client): except Exception: fnd_stat = None - hash_type = self.opts.get('hash_type', 'md5') - ret['hsum'] = salt.utils.get_hash(fnd_path, form=hash_type) - ret['hash_type'] = hash_type + hash_type = self.opts.get(u'hash_type', u'md5') + ret[u'hsum'] = salt.utils.get_hash(fnd_path, form=hash_type) + ret[u'hash_type'] = hash_type return ret, fnd_stat - def list_env(self, saltenv='base'): + def list_env(self, saltenv=u'base'): ''' Return a list of the files in the file server's specified environment ''' @@ -984,7 +1005,7 @@ class LocalClient(Client): Return the available environments ''' ret = [] - for saltenv in self.opts['file_roots']: + for saltenv in self.opts[u'file_roots']: ret.append(saltenv) return ret @@ -1011,10 +1032,10 @@ class RemoteClient(Client): def __init__(self, opts): Client.__init__(self, opts) self.channel = salt.transport.Channel.factory(self.opts) - if hasattr(self.channel, 'auth'): + if hasattr(self.channel, u'auth'): self.auth = self.channel.auth else: - self.auth = '' + self.auth = u'' def _refresh_channel(self): ''' @@ -1025,9 +1046,9 @@ class RemoteClient(Client): def get_file(self, path, - dest='', + dest=u'', makedirs=False, - saltenv='base', + saltenv=u'base', gzip=None, cachedir=None): ''' @@ -1040,7 +1061,7 @@ class RemoteClient(Client): if senv: saltenv = senv - if not salt.utils.is_windows(): + if not salt.utils.platform.is_windows(): hash_server, stat_server = self.hash_and_stat_file(path, saltenv) try: mode_server = stat_server[0] @@ -1052,13 +1073,22 @@ class RemoteClient(Client): # Check if file exists on server, before creating files and # directories - if hash_server == '': + if hash_server == u'': log.debug( - 'Could not find file \'%s\' in saltenv \'%s\'', + u'Could not find file \'%s\' in saltenv \'%s\'', path, saltenv ) return False + # If dest is a directory, rewrite dest with filename + if dest is not None \ + and (os.path.isdir(dest) or dest.endswith((u'/', u'\\'))): + dest = os.path.join(dest, os.path.basename(path)) + log.debug( + u'In saltenv \'%s\', \'%s\' is a directory. Changing dest to ' + u'\'%s\'', saltenv, os.path.dirname(dest), dest + ) + # Hash compare local copy with master and skip download # if no difference found. dest2check = dest @@ -1066,20 +1096,20 @@ class RemoteClient(Client): rel_path = self._check_proto(path) log.debug( - 'In saltenv \'%s\', looking at rel_path \'%s\' to resolve ' - '\'%s\'', saltenv, rel_path, path + u'In saltenv \'%s\', looking at rel_path \'%s\' to resolve ' + u'\'%s\'', saltenv, rel_path, path ) with self._cache_loc( rel_path, saltenv, cachedir=cachedir) as cache_dest: dest2check = cache_dest log.debug( - 'In saltenv \'%s\', ** considering ** path \'%s\' to resolve ' - '\'%s\'', saltenv, dest2check, path + u'In saltenv \'%s\', ** considering ** path \'%s\' to resolve ' + u'\'%s\'', saltenv, dest2check, path ) if dest2check and os.path.isfile(dest2check): - if not salt.utils.is_windows(): + if not salt.utils.platform.is_windows(): hash_local, stat_local = \ self.hash_and_stat_file(dest2check, saltenv) try: @@ -1094,18 +1124,18 @@ class RemoteClient(Client): return dest2check log.debug( - 'Fetching file from saltenv \'%s\', ** attempting ** \'%s\'', + u'Fetching file from saltenv \'%s\', ** attempting ** \'%s\'', saltenv, path ) d_tries = 0 transport_tries = 0 path = self._check_proto(path) - load = {'path': path, - 'saltenv': saltenv, - 'cmd': '_serve_file'} + load = {u'path': path, + u'saltenv': saltenv, + u'cmd': u'_serve_file'} if gzip: gzip = int(gzip) - load['gzip'] = gzip + load[u'gzip'] = gzip fn_ = None if dest: @@ -1117,15 +1147,15 @@ class RemoteClient(Client): return False # We need an open filehandle here, that's why we're not using a # with clause: - fn_ = salt.utils.fopen(dest, 'wb+') # pylint: disable=resource-leakage + fn_ = salt.utils.files.fopen(dest, u'wb+') # pylint: disable=resource-leakage else: - log.debug('No dest file found') + log.debug(u'No dest file found') while True: if not fn_: - load['loc'] = 0 + load[u'loc'] = 0 else: - load['loc'] = fn_.tell() + load[u'loc'] = fn_.tell() data = self.channel.send(load, raw=True) if six.PY3: # Sometimes the source is local (eg when using @@ -1135,43 +1165,43 @@ class RemoteClient(Client): # strings for the top-level keys to simplify things. data = decode_dict_keys_to_str(data) try: - if not data['data']: - if not fn_ and data['dest']: + if not data[u'data']: + if not fn_ and data[u'dest']: # This is a 0 byte file on the master with self._cache_loc( - data['dest'], + data[u'dest'], saltenv, cachedir=cachedir) as cache_dest: dest = cache_dest - with salt.utils.fopen(cache_dest, 'wb+') as ofile: - ofile.write(data['data']) - if 'hsum' in data and d_tries < 3: + with salt.utils.files.fopen(cache_dest, u'wb+') as ofile: + ofile.write(data[u'data']) + if u'hsum' in data and d_tries < 3: # Master has prompted a file verification, if the # verification fails, re-download the file. Try 3 times d_tries += 1 - hsum = salt.utils.get_hash(dest, salt.utils.to_str(data.get('hash_type', b'md5'))) - if hsum != data['hsum']: + hsum = salt.utils.get_hash(dest, salt.utils.stringutils.to_str(data.get(u'hash_type', b'md5'))) # future lint: disable=non-unicode-string + if hsum != data[u'hsum']: log.warning( - 'Bad download of file %s, attempt %d of 3', + u'Bad download of file %s, attempt %d of 3', path, d_tries ) continue break if not fn_: with self._cache_loc( - data['dest'], + data[u'dest'], saltenv, cachedir=cachedir) as cache_dest: dest = cache_dest # If a directory was formerly cached at this path, then # remove it to avoid a traceback trying to write the file if os.path.isdir(dest): - salt.utils.rm_rf(dest) - fn_ = salt.utils.fopen(dest, 'wb+') - if data.get('gzip', None): - data = salt.utils.gzip_util.uncompress(data['data']) + salt.utils.files.rm_rf(dest) + fn_ = salt.utils.files.fopen(dest, u'wb+') + if data.get(u'gzip', None): + data = salt.utils.gzip_util.uncompress(data[u'data']) else: - data = data['data'] + data = data[u'data'] if six.PY3 and isinstance(data, str): data = data.encode() fn_.write(data) @@ -1183,15 +1213,15 @@ class RemoteClient(Client): data_type = str(type(data)) transport_tries += 1 log.warning( - 'Data transport is broken, got: %s, type: %s, ' - 'exception: %s, attempt %d of 3', + u'Data transport is broken, got: %s, type: %s, ' + u'exception: %s, attempt %d of 3', data, data_type, exc, transport_tries ) self._refresh_channel() if transport_tries > 3: log.error( - 'Data transport is broken, got: %s, type: %s, ' - 'exception: %s, retry attempts exhausted', + u'Data transport is broken, got: %s, type: %s, ' + u'exception: %s, retry attempts exhausted', data, data_type, exc ) break @@ -1199,55 +1229,55 @@ class RemoteClient(Client): if fn_: fn_.close() log.info( - 'Fetching file from saltenv \'%s\', ** done ** \'%s\'', + u'Fetching file from saltenv \'%s\', ** done ** \'%s\'', saltenv, path ) else: log.debug( - 'In saltenv \'%s\', we are ** missing ** the file \'%s\'', + u'In saltenv \'%s\', we are ** missing ** the file \'%s\'', saltenv, path ) return dest - def file_list(self, saltenv='base', prefix=''): + def file_list(self, saltenv=u'base', prefix=u''): ''' List the files on the master ''' - load = {'saltenv': saltenv, - 'prefix': prefix, - 'cmd': '_file_list'} + load = {u'saltenv': saltenv, + u'prefix': prefix, + u'cmd': u'_file_list'} return [sdecode(fn_) for fn_ in self.channel.send(load)] - def file_list_emptydirs(self, saltenv='base', prefix=''): + def file_list_emptydirs(self, saltenv=u'base', prefix=u''): ''' List the empty dirs on the master ''' - load = {'saltenv': saltenv, - 'prefix': prefix, - 'cmd': '_file_list_emptydirs'} + load = {u'saltenv': saltenv, + u'prefix': prefix, + u'cmd': u'_file_list_emptydirs'} self.channel.send(load) - def dir_list(self, saltenv='base', prefix=''): + def dir_list(self, saltenv=u'base', prefix=u''): ''' List the dirs on the master ''' - load = {'saltenv': saltenv, - 'prefix': prefix, - 'cmd': '_dir_list'} + load = {u'saltenv': saltenv, + u'prefix': prefix, + u'cmd': u'_dir_list'} return self.channel.send(load) - def symlink_list(self, saltenv='base', prefix=''): + def symlink_list(self, saltenv=u'base', prefix=u''): ''' List symlinked files and dirs on the master ''' - load = {'saltenv': saltenv, - 'prefix': prefix, - 'cmd': '_symlink_list'} + load = {u'saltenv': saltenv, + u'prefix': prefix, + u'cmd': u'_symlink_list'} return self.channel.send(load) - def __hash_and_stat_file(self, path, saltenv='base'): + def __hash_and_stat_file(self, path, saltenv=u'base'): ''' Common code for hashing and stating files ''' @@ -1255,66 +1285,87 @@ class RemoteClient(Client): path = self._check_proto(path) except MinionError as err: if not os.path.isfile(path): - msg = 'specified file {0} is not present to generate hash: {1}' - log.warning(msg.format(path, err)) - return {} + log.warning( + u'specified file %s is not present to generate hash: %s', + path, err + ) + return {}, None else: ret = {} - hash_type = self.opts.get('hash_type', 'md5') - ret['hsum'] = salt.utils.get_hash(path, form=hash_type) - ret['hash_type'] = hash_type - return ret, list(os.stat(path)) - load = {'path': path, - 'saltenv': saltenv, - 'cmd': '_file_hash_and_stat'} + hash_type = self.opts.get(u'hash_type', u'md5') + ret[u'hsum'] = salt.utils.get_hash(path, form=hash_type) + ret[u'hash_type'] = hash_type + return ret + load = {u'path': path, + u'saltenv': saltenv, + u'cmd': u'_file_hash'} return self.channel.send(load) - def hash_file(self, path, saltenv='base'): + def hash_file(self, path, saltenv=u'base'): ''' Return the hash of a file, to get the hash of a file on the salt master file server prepend the path with salt:// otherwise, prepend the file with / for a local file. ''' - return self.__hash_and_stat_file(path, saltenv)[0] + return self.__hash_and_stat_file(path, saltenv) - def hash_and_stat_file(self, path, saltenv='base'): + def hash_and_stat_file(self, path, saltenv=u'base'): ''' The same as hash_file, but also return the file's mode, or None if no mode data is present. ''' - return self.__hash_and_stat_file(path, saltenv) + hash_result = self.hash_file(path, saltenv) + try: + path = self._check_proto(path) + except MinionError as err: + if not os.path.isfile(path): + return hash_result, None + else: + try: + return hash_result, list(os.stat(path)) + except Exception: + return hash_result, None + load = {'path': path, + 'saltenv': saltenv, + 'cmd': '_file_find'} + fnd = self.channel.send(load) + try: + stat_result = fnd.get('stat') + except AttributeError: + stat_result = None + return hash_result, stat_result - def list_env(self, saltenv='base'): + def list_env(self, saltenv=u'base'): ''' Return a list of the files in the file server's specified environment ''' - load = {'saltenv': saltenv, - 'cmd': '_file_list'} + load = {u'saltenv': saltenv, + u'cmd': u'_file_list'} return self.channel.send(load) def envs(self): ''' Return a list of available environments ''' - load = {'cmd': '_file_envs'} + load = {u'cmd': u'_file_envs'} return self.channel.send(load) def master_opts(self): ''' Return the master opts data ''' - load = {'cmd': '_master_opts'} + load = {u'cmd': u'_master_opts'} return self.channel.send(load) def master_tops(self): ''' Return the metadata derived from the master_tops system ''' - load = {'cmd': '_master_tops', - 'id': self.opts['id'], - 'opts': self.opts} + load = {u'cmd': u'_master_tops', + u'id': self.opts[u'id'], + u'opts': self.opts} if self.auth: - load['tok'] = self.auth.gen_token('salt') + load[u'tok'] = self.auth.gen_token(u'salt') return self.channel.send(load) diff --git a/salt/fileserver/__init__.py b/salt/fileserver/__init__.py index e0b3dac091d..9ba52e6e153 100644 --- a/salt/fileserver/__init__.py +++ b/salt/fileserver/__init__.py @@ -15,12 +15,14 @@ import time # Import salt libs import salt.loader -import salt.utils +import salt.utils.files import salt.utils.locales +import salt.utils.url +import salt.utils.versions from salt.utils.args import get_function_argspec as _argspec # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six log = logging.getLogger(__name__) @@ -126,7 +128,7 @@ def check_file_list_cache(opts, form, list_cache, w_lock): age = opts.get('fileserver_list_cache_time', 20) + 1 if age < opts.get('fileserver_list_cache_time', 20): # Young enough! Load this sucker up! - with salt.utils.fopen(list_cache, 'rb') as fp_: + with salt.utils.files.fopen(list_cache, 'rb') as fp_: log.trace('Returning file_lists cache data from ' '{0}'.format(list_cache)) return serial.load(fp_).get(form, []), False, False @@ -151,7 +153,7 @@ def write_file_list_cache(opts, data, list_cache, w_lock): backend to determine if the cache needs to be refreshed/written). ''' serial = salt.payload.Serial(opts) - with salt.utils.fopen(list_cache, 'w+b') as fp_: + with salt.utils.files.fopen(list_cache, 'w+b') as fp_: fp_.write(serial.dumps(data)) _unlock_cache(w_lock) log.trace('Lockfile {0} removed'.format(w_lock)) @@ -164,7 +166,7 @@ def check_env_cache(opts, env_cache): if not os.path.isfile(env_cache): return None try: - with salt.utils.fopen(env_cache, 'rb') as fp_: + with salt.utils.files.fopen(env_cache, 'rb') as fp_: log.trace('Returning env cache data from {0}'.format(env_cache)) serial = salt.payload.Serial(opts) return serial.load(fp_) @@ -551,12 +553,7 @@ class Fileserver(object): kwargs[args[0]] = args[1] if 'env' in kwargs: - salt.utils.warn_until( - 'Oxygen', - 'Parameter \'env\' has been detected in the argument list. This ' - 'parameter is no longer used and has been replaced by \'saltenv\' ' - 'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.' - ) + # "env" is not supported; Use "saltenv". kwargs.pop('env') if 'saltenv' in kwargs: saltenv = kwargs.pop('saltenv') @@ -581,12 +578,7 @@ class Fileserver(object): 'dest': ''} if 'env' in load: - salt.utils.warn_until( - 'Oxygen', - 'Parameter \'env\' has been detected in the argument list. This ' - 'parameter is no longer used and has been replaced by \'saltenv\' ' - 'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.' - ) + # "env" is not supported; Use "saltenv". load.pop('env') if 'path' not in load or 'loc' not in load or 'saltenv' not in load: @@ -607,13 +599,7 @@ class Fileserver(object): Common code for hashing and stating files ''' if 'env' in load: - salt.utils.warn_until( - 'Oxygen', - 'Parameter \'env\' has been detected in the argument list. ' - 'This parameter is no longer used and has been replaced by ' - '\'saltenv\' as of Salt 2016.11.0. This warning will be removed ' - 'in Salt Oxygen.' - ) + # "env" is not supported; Use "saltenv". load.pop('env') if 'path' not in load or 'saltenv' not in load: @@ -654,12 +640,7 @@ class Fileserver(object): Deletes the file_lists cache files ''' if 'env' in load: - salt.utils.warn_until( - 'Oxygen', - 'Parameter \'env\' has been detected in the argument list. This ' - 'parameter is no longer used and has been replaced by \'saltenv\' ' - 'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.' - ) + # "env" is not supported; Use "saltenv". load.pop('env') saltenv = load.get('saltenv', []) @@ -736,12 +717,7 @@ class Fileserver(object): Return a list of files from the dominant environment ''' if 'env' in load: - salt.utils.warn_until( - 'Oxygen', - 'Parameter \'env\' has been detected in the argument list. This ' - 'parameter is no longer used and has been replaced by \'saltenv\' ' - 'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.' - ) + # "env" is not supported; Use "saltenv". load.pop('env') ret = set() @@ -767,12 +743,7 @@ class Fileserver(object): List all emptydirs in the given environment ''' if 'env' in load: - salt.utils.warn_until( - 'Oxygen', - 'Parameter \'env\' has been detected in the argument list. This ' - 'parameter is no longer used and has been replaced by \'saltenv\' ' - 'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.' - ) + # "env" is not supported; Use "saltenv". load.pop('env') ret = set() @@ -798,12 +769,7 @@ class Fileserver(object): List all directories in the given environment ''' if 'env' in load: - salt.utils.warn_until( - 'Oxygen', - 'Parameter \'env\' has been detected in the argument list. This ' - 'parameter is no longer used and has been replaced by \'saltenv\' ' - 'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.' - ) + # "env" is not supported; Use "saltenv". load.pop('env') ret = set() @@ -829,12 +795,7 @@ class Fileserver(object): Return a list of symlinked files and dirs ''' if 'env' in load: - salt.utils.warn_until( - 'Oxygen', - 'Parameter \'env\' has been detected in the argument list. This ' - 'parameter is no longer used and has been replaced by \'saltenv\' ' - 'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.' - ) + # "env" is not supported; Use "saltenv". load.pop('env') ret = {} diff --git a/salt/fileserver/azurefs.py b/salt/fileserver/azurefs.py index 22ec6a09a83..d1cdf1c33b3 100644 --- a/salt/fileserver/azurefs.py +++ b/salt/fileserver/azurefs.py @@ -46,7 +46,6 @@ permissions. # Import python libs from __future__ import absolute_import -from salt.utils.versions import LooseVersion import base64 import json import logging @@ -57,6 +56,10 @@ import shutil # Import salt libs import salt.fileserver import salt.utils +import salt.utils.files +import salt.utils.gzip_util +import salt.utils.path +from salt.utils.versions import LooseVersion try: import azure.storage @@ -67,7 +70,7 @@ except ImportError: HAS_AZURE = False # Import third party libs -import salt.ext.six as six +from salt.ext import six __virtualname__ = 'azurefs' @@ -160,7 +163,7 @@ def serve_file(load, fnd): ret['dest'] = fnd['rel'] gzip = load.get('gzip', None) fpath = os.path.normpath(fnd['path']) - with salt.utils.fopen(fpath, 'rb') as fp_: + with salt.utils.files.fopen(fpath, 'rb') as fp_: fp_.seek(load['loc']) data = fp_.read(__opts__['file_buffer_size']) if data and six.PY3 and not salt.utils.is_bin_file(fpath): @@ -235,7 +238,7 @@ def update(): # Lock writes lk_fn = fname + '.lk' salt.fileserver.wait_lock(lk_fn, fname) - with salt.utils.fopen(lk_fn, 'w+') as fp_: + with salt.utils.files.fopen(lk_fn, 'w+') as fp_: fp_.write('') try: @@ -254,9 +257,9 @@ def update(): container_list = path + '.list' lk_fn = container_list + '.lk' salt.fileserver.wait_lock(lk_fn, container_list) - with salt.utils.fopen(lk_fn, 'w+') as fp_: + with salt.utils.files.fopen(lk_fn, 'w+') as fp_: fp_.write('') - with salt.utils.fopen(container_list, 'w') as fp_: + with salt.utils.files.fopen(container_list, 'w') as fp_: fp_.write(json.dumps(blob_names)) try: os.unlink(lk_fn) @@ -274,7 +277,7 @@ def file_hash(load, fnd): relpath = fnd['rel'] path = fnd['path'] hash_cachedir = os.path.join(__opts__['cachedir'], 'azurefs', 'hashes') - hashdest = salt.utils.path_join(hash_cachedir, + hashdest = salt.utils.path.join(hash_cachedir, load['saltenv'], '{0}.hash.{1}'.format(relpath, __opts__['hash_type'])) @@ -282,11 +285,11 @@ def file_hash(load, fnd): if not os.path.exists(os.path.dirname(hashdest)): os.makedirs(os.path.dirname(hashdest)) ret['hsum'] = salt.utils.get_hash(path, __opts__['hash_type']) - with salt.utils.fopen(hashdest, 'w+') as fp_: + with salt.utils.files.fopen(hashdest, 'w+') as fp_: fp_.write(ret['hsum']) return ret else: - with salt.utils.fopen(hashdest, 'rb') as fp_: + with salt.utils.files.fopen(hashdest, 'rb') as fp_: ret['hsum'] = fp_.read() return ret @@ -305,7 +308,7 @@ def file_list(load): salt.fileserver.wait_lock(lk, container_list, 5) if not os.path.exists(container_list): continue - with salt.utils.fopen(container_list, 'r') as fp_: + with salt.utils.files.fopen(container_list, 'r') as fp_: ret.update(set(json.load(fp_))) except Exception as exc: log.error('azurefs: an error ocurred retrieving file lists. ' diff --git a/salt/fileserver/gitfs.py b/salt/fileserver/gitfs.py index fec489857d8..1182ec69bea 100644 --- a/salt/fileserver/gitfs.py +++ b/salt/fileserver/gitfs.py @@ -48,9 +48,12 @@ Walkthrough `. from __future__ import absolute_import import logging -PER_REMOTE_OVERRIDES = ('base', 'mountpoint', 'root', 'ssl_verify', - 'saltenv_whitelist', 'saltenv_blacklist', - 'env_whitelist', 'env_blacklist', 'refspecs') +PER_REMOTE_OVERRIDES = ( + 'base', 'mountpoint', 'root', 'ssl_verify', + 'saltenv_whitelist', 'saltenv_blacklist', + 'env_whitelist', 'env_blacklist', 'refspecs', + 'disable_saltenv_mapping', 'ref_types' +) PER_REMOTE_ONLY = ('all_saltenvs', 'name', 'saltenv') # Auth support (auth params can be global or per-remote, too) diff --git a/salt/fileserver/hgfs.py b/salt/fileserver/hgfs.py index b8d1561e809..e386a9e6e8c 100644 --- a/salt/fileserver/hgfs.py +++ b/salt/fileserver/hgfs.py @@ -49,7 +49,7 @@ VALID_BRANCH_METHODS = ('branches', 'bookmarks', 'mixed') PER_REMOTE_OVERRIDES = ('base', 'branch_method', 'mountpoint', 'root') # Import third party libs -import salt.ext.six as six +from salt.ext import six # pylint: disable=import-error try: import hglib @@ -60,7 +60,10 @@ except ImportError: # Import salt libs import salt.utils +import salt.utils.files +import salt.utils.gzip_util import salt.utils.url +import salt.utils.versions import salt.fileserver from salt.utils.event import tagify @@ -295,7 +298,7 @@ def init(): if not refs: # Write an hgrc defining the remote URL hgconfpath = os.path.join(rp_, '.hg', 'hgrc') - with salt.utils.fopen(hgconfpath, 'w+') as hgconfig: + with salt.utils.files.fopen(hgconfpath, 'w+') as hgconfig: hgconfig.write('[paths]\n') hgconfig.write('default = {0}\n'.format(repo_url)) @@ -314,7 +317,7 @@ def init(): if new_remote: remote_map = os.path.join(__opts__['cachedir'], 'hgfs/remote_map.txt') try: - with salt.utils.fopen(remote_map, 'w+') as fp_: + with salt.utils.files.fopen(remote_map, 'w+') as fp_: timestamp = datetime.now().strftime('%d %b %Y %H:%M:%S.%f') fp_.write('# hgfs_remote map as of {0}\n'.format(timestamp)) for repo in repos: @@ -453,7 +456,7 @@ def lock(remote=None): failed = [] if not os.path.exists(repo['lockfile']): try: - with salt.utils.fopen(repo['lockfile'], 'w+') as fp_: + with salt.utils.files.fopen(repo['lockfile'], 'w+') as fp_: fp_.write('') except (IOError, OSError) as exc: msg = ('Unable to set update lock for {0} ({1}): {2} ' @@ -538,7 +541,7 @@ def update(): os.makedirs(env_cachedir) new_envs = envs(ignore_cache=True) serial = salt.payload.Serial(__opts__) - with salt.utils.fopen(env_cache, 'wb+') as fp_: + with salt.utils.files.fopen(env_cache, 'wb+') as fp_: fp_.write(serial.dumps(new_envs)) log.trace('Wrote env cache data to {0}'.format(env_cache)) @@ -567,7 +570,7 @@ def _env_is_exposed(env): blacklist. ''' if __opts__['hgfs_env_whitelist']: - salt.utils.warn_until( + salt.utils.versions.warn_until( 'Neon', 'The hgfs_env_whitelist config option has been renamed to ' 'hgfs_saltenv_whitelist. Please update your configuration.' @@ -577,7 +580,7 @@ def _env_is_exposed(env): whitelist = __opts__['hgfs_saltenv_whitelist'] if __opts__['hgfs_env_blacklist']: - salt.utils.warn_until( + salt.utils.versions.warn_until( 'Neon', 'The hgfs_env_blacklist config option has been renamed to ' 'hgfs_saltenv_blacklist. Please update your configuration.' @@ -678,7 +681,7 @@ def find_file(path, tgt_env='base', **kwargs): # pylint: disable=W0613 continue salt.fileserver.wait_lock(lk_fn, dest) if os.path.isfile(blobshadest) and os.path.isfile(dest): - with salt.utils.fopen(blobshadest, 'r') as fp_: + with salt.utils.files.fopen(blobshadest, 'r') as fp_: sha = fp_.read() if sha == ref[2]: fnd['rel'] = path @@ -692,14 +695,14 @@ def find_file(path, tgt_env='base', **kwargs): # pylint: disable=W0613 except hglib.error.CommandError: repo['repo'].close() continue - with salt.utils.fopen(lk_fn, 'w+') as fp_: + with salt.utils.files.fopen(lk_fn, 'w+') as fp_: fp_.write('') for filename in glob.glob(hashes_glob): try: os.remove(filename) except Exception: pass - with salt.utils.fopen(blobshadest, 'w+') as fp_: + with salt.utils.files.fopen(blobshadest, 'w+') as fp_: fp_.write(ref[2]) try: os.remove(lk_fn) @@ -733,12 +736,7 @@ def serve_file(load, fnd): Return a chunk from a file based on the data received ''' if 'env' in load: - salt.utils.warn_until( - 'Oxygen', - 'Parameter \'env\' has been detected in the argument list. This ' - 'parameter is no longer used and has been replaced by \'saltenv\' ' - 'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.' - ) + # "env" is not supported; Use "saltenv". load.pop('env') ret = {'data': '', @@ -750,7 +748,7 @@ def serve_file(load, fnd): ret['dest'] = fnd['rel'] gzip = load.get('gzip', None) fpath = os.path.normpath(fnd['path']) - with salt.utils.fopen(fpath, 'rb') as fp_: + with salt.utils.files.fopen(fpath, 'rb') as fp_: fp_.seek(load['loc']) data = fp_.read(__opts__['file_buffer_size']) if data and six.PY3 and not salt.utils.is_bin_file(fpath): @@ -767,12 +765,7 @@ def file_hash(load, fnd): Return a file hash, the hash type is set in the master config file ''' if 'env' in load: - salt.utils.warn_until( - 'Oxygen', - 'Parameter \'env\' has been detected in the argument list. This ' - 'parameter is no longer used and has been replaced by \'saltenv\' ' - 'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.' - ) + # "env" is not supported; Use "saltenv". load.pop('env') if not all(x in load for x in ('path', 'saltenv')): @@ -787,11 +780,11 @@ def file_hash(load, fnd): __opts__['hash_type'])) if not os.path.isfile(hashdest): ret['hsum'] = salt.utils.get_hash(path, __opts__['hash_type']) - with salt.utils.fopen(hashdest, 'w+') as fp_: + with salt.utils.files.fopen(hashdest, 'w+') as fp_: fp_.write(ret['hsum']) return ret else: - with salt.utils.fopen(hashdest, 'rb') as fp_: + with salt.utils.files.fopen(hashdest, 'rb') as fp_: ret['hsum'] = fp_.read() return ret @@ -801,12 +794,7 @@ def _file_lists(load, form): Return a dict containing the file lists for files and dirs ''' if 'env' in load: - salt.utils.warn_until( - 'Oxygen', - 'Parameter \'env\' has been detected in the argument list. This ' - 'parameter is no longer used and has been replaced by \'saltenv\' ' - 'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.' - ) + # "env" is not supported; Use "saltenv". load.pop('env') list_cachedir = os.path.join(__opts__['cachedir'], 'file_lists/hgfs') @@ -849,12 +837,7 @@ def _get_file_list(load): Get a list of all files on the file server in a specified environment ''' if 'env' in load: - salt.utils.warn_until( - 'Oxygen', - 'Parameter \'env\' has been detected in the argument list. This ' - 'parameter is no longer used and has been replaced by \'saltenv\' ' - 'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.' - ) + # "env" is not supported; Use "saltenv". load.pop('env') if 'saltenv' not in load or load['saltenv'] not in envs(): @@ -894,12 +877,7 @@ def _get_dir_list(load): Get a list of all directories on the master ''' if 'env' in load: - salt.utils.warn_until( - 'Oxygen', - 'Parameter \'env\' has been detected in the argument list. This ' - 'parameter is no longer used and has been replaced by \'saltenv\' ' - 'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.' - ) + # "env" is not supported; Use "saltenv". load.pop('env') if 'saltenv' not in load or load['saltenv'] not in envs(): diff --git a/salt/fileserver/minionfs.py b/salt/fileserver/minionfs.py index d59c07f18c5..292bf0f85ed 100644 --- a/salt/fileserver/minionfs.py +++ b/salt/fileserver/minionfs.py @@ -32,10 +32,13 @@ import logging # Import salt libs import salt.fileserver import salt.utils +import salt.utils.files +import salt.utils.gzip_util import salt.utils.url +import salt.utils.versions # Import third party libs -import salt.ext.six as six +from salt.ext import six log = logging.getLogger(__name__) @@ -129,7 +132,7 @@ def serve_file(load, fnd): # AP # May I sleep here to slow down serving of big files? # How many threads are serving files? - with salt.utils.fopen(fpath, 'rb') as fp_: + with salt.utils.files.fopen(fpath, 'rb') as fp_: fp_.seek(load['loc']) data = fp_.read(__opts__['file_buffer_size']) if data and six.PY3 and not salt.utils.is_bin_file(fpath): @@ -162,12 +165,7 @@ def file_hash(load, fnd): ret = {} if 'env' in load: - salt.utils.warn_until( - 'Oxygen', - 'Parameter \'env\' has been detected in the argument list. This ' - 'parameter is no longer used and has been replaced by \'saltenv\' ' - 'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.' - ) + # "env" is not supported; Use "saltenv". load.pop('env') if load['saltenv'] not in envs(): @@ -191,7 +189,7 @@ def file_hash(load, fnd): # if we have a cache, serve that if the mtime hasn't changed if os.path.exists(cache_path): try: - with salt.utils.fopen(cache_path, 'rb') as fp_: + with salt.utils.files.fopen(cache_path, 'rb') as fp_: try: hsum, mtime = fp_.read().split(':') except ValueError: @@ -222,7 +220,7 @@ def file_hash(load, fnd): os.makedirs(cache_dir) # save the cache object "hash:mtime" cache_object = '{0}:{1}'.format(ret['hsum'], os.path.getmtime(path)) - with salt.utils.flopen(cache_path, 'w') as fp_: + with salt.utils.files.flopen(cache_path, 'w') as fp_: fp_.write(cache_object) return ret @@ -232,12 +230,7 @@ def file_list(load): Return a list of all files on the file server in a specified environment ''' if 'env' in load: - salt.utils.warn_until( - 'Oxygen', - 'Parameter \'env\' has been detected in the argument list. This ' - 'parameter is no longer used and has been replaced by \'saltenv\' ' - 'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.' - ) + # "env" is not supported; Use "saltenv". load.pop('env') if load['saltenv'] not in envs(): @@ -316,12 +309,7 @@ def dir_list(load): - source-minion/absolute/path ''' if 'env' in load: - salt.utils.warn_until( - 'Oxygen', - 'Parameter \'env\' has been detected in the argument list. This ' - 'parameter is no longer used and has been replaced by \'saltenv\' ' - 'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.' - ) + # "env" is not supported; Use "saltenv". load.pop('env') if load['saltenv'] not in envs(): diff --git a/salt/fileserver/roots.py b/salt/fileserver/roots.py index 62563006c2b..c795163b9d4 100644 --- a/salt/fileserver/roots.py +++ b/salt/fileserver/roots.py @@ -24,10 +24,13 @@ import logging # Import salt libs import salt.fileserver -import salt.utils +import salt.utils # Can be removed once is_bin_file and get_hash are moved +import salt.utils.event +import salt.utils.files +import salt.utils.gzip_util import salt.utils.path -from salt.utils.event import tagify -import salt.ext.six as six +import salt.utils.versions +from salt.ext import six log = logging.getLogger(__name__) @@ -37,12 +40,7 @@ def find_file(path, saltenv='base', **kwargs): Search the environment for the relative path. ''' if 'env' in kwargs: - salt.utils.warn_until( - 'Oxygen', - 'Parameter \'env\' has been detected in the argument list. This ' - 'parameter is no longer used and has been replaced by \'saltenv\' ' - 'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.' - ) + # "env" is not supported; Use "saltenv". kwargs.pop('env') path = os.path.normpath(path) @@ -114,12 +112,7 @@ def serve_file(load, fnd): Return a chunk from a file based on the data received ''' if 'env' in load: - salt.utils.warn_until( - 'Oxygen', - 'Parameter \'env\' has been detected in the argument list. This ' - 'parameter is no longer used and has been replaced by \'saltenv\' ' - 'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.' - ) + # "env" is not supported; Use "saltenv". load.pop('env') ret = {'data': '', @@ -131,7 +124,7 @@ def serve_file(load, fnd): ret['dest'] = fnd['rel'] gzip = load.get('gzip', None) fpath = os.path.normpath(fnd['path']) - with salt.utils.fopen(fpath, 'rb') as fp_: + with salt.utils.files.fopen(fpath, 'rb') as fp_: fp_.seek(load['loc']) data = fp_.read(__opts__['file_buffer_size']) if data and six.PY3 and not salt.utils.is_bin_file(fpath): @@ -168,7 +161,7 @@ def update(): old_mtime_map = {} # if you have an old map, load that if os.path.exists(mtime_map_path): - with salt.utils.fopen(mtime_map_path, 'r') as fp_: + with salt.utils.files.fopen(mtime_map_path, 'r') as fp_: for line in fp_: try: file_path, mtime = line.replace('\n', '').split(':', 1) @@ -193,7 +186,7 @@ def update(): mtime_map_path_dir = os.path.dirname(mtime_map_path) if not os.path.exists(mtime_map_path_dir): os.makedirs(mtime_map_path_dir) - with salt.utils.fopen(mtime_map_path, 'w') as fp_: + with salt.utils.files.fopen(mtime_map_path, 'w') as fp_: for file_path, mtime in six.iteritems(new_mtime_map): fp_.write('{file_path}:{mtime}\n'.format(file_path=file_path, mtime=mtime)) @@ -206,7 +199,8 @@ def update(): __opts__['transport'], opts=__opts__, listen=False) - event.fire_event(data, tagify(['roots', 'update'], prefix='fileserver')) + event.fire_event(data, + salt.utils.event.tagify(['roots', 'update'], prefix='fileserver')) def file_hash(load, fnd): @@ -214,12 +208,7 @@ def file_hash(load, fnd): Return a file hash, the hash type is set in the master config file ''' if 'env' in load: - salt.utils.warn_until( - 'Oxygen', - 'Parameter \'env\' has been detected in the argument list. This ' - 'parameter is no longer used and has been replaced by \'saltenv\' ' - 'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.' - ) + # "env" is not supported; Use "saltenv". load.pop('env') if 'path' not in load or 'saltenv' not in load: @@ -244,7 +233,7 @@ def file_hash(load, fnd): # if we have a cache, serve that if the mtime hasn't changed if os.path.exists(cache_path): try: - with salt.utils.fopen(cache_path, 'r') as fp_: + with salt.utils.files.fopen(cache_path, 'r') as fp_: try: hsum, mtime = fp_.read().split(':') except ValueError: @@ -284,7 +273,7 @@ def file_hash(load, fnd): raise # save the cache object "hash:mtime" cache_object = '{0}:{1}'.format(ret['hsum'], os.path.getmtime(path)) - with salt.utils.flopen(cache_path, 'w') as fp_: + with salt.utils.files.flopen(cache_path, 'w') as fp_: fp_.write(cache_object) return ret @@ -294,12 +283,7 @@ def _file_lists(load, form): Return a dict containing the file lists for files, dirs, emtydirs and symlinks ''' if 'env' in load: - salt.utils.warn_until( - 'Oxygen', - 'Parameter \'env\' has been detected in the argument list. This ' - 'parameter is no longer used and has been replaced by \'saltenv\' ' - 'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.' - ) + # "env" is not supported; Use "saltenv". load.pop('env') if load['saltenv'] not in __opts__['file_roots']: @@ -440,12 +424,7 @@ def symlink_list(load): Return a dict of all symlinks based on a given path on the Master ''' if 'env' in load: - salt.utils.warn_until( - 'Oxygen', - 'Parameter \'env\' has been detected in the argument list. This ' - 'parameter is no longer used and has been replaced by \'saltenv\' ' - 'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.' - ) + # "env" is not supported; Use "saltenv". load.pop('env') ret = {} diff --git a/salt/fileserver/s3fs.py b/salt/fileserver/s3fs.py index 5f6fca0b61a..04f0b5e51cd 100644 --- a/salt/fileserver/s3fs.py +++ b/salt/fileserver/s3fs.py @@ -69,10 +69,13 @@ import logging import salt.fileserver as fs import salt.modules import salt.utils +import salt.utils.files +import salt.utils.gzip_util +import salt.utils.versions # Import 3rd-party libs # pylint: disable=import-error,no-name-in-module,redefined-builtin -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import filter from salt.ext.six.moves.urllib.parse import quote as _quote # pylint: enable=import-error,no-name-in-module,redefined-builtin @@ -123,12 +126,7 @@ def find_file(path, saltenv='base', **kwargs): is missing, or if the MD5 does not match. ''' if 'env' in kwargs: - salt.utils.warn_until( - 'Oxygen', - 'Parameter \'env\' has been detected in the argument list. This ' - 'parameter is no longer used and has been replaced by \'saltenv\' ' - 'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.' - ) + # "env" is not supported; Use "saltenv". kwargs.pop('env') fnd = {'bucket': None, @@ -165,12 +163,7 @@ def file_hash(load, fnd): Return an MD5 file hash ''' if 'env' in load: - salt.utils.warn_until( - 'Oxygen', - 'Parameter \'env\' has been detected in the argument list. This ' - 'parameter is no longer used and has been replaced by \'saltenv\' ' - 'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.' - ) + # "env" is not supported; Use "saltenv". load.pop('env') ret = {} @@ -198,12 +191,7 @@ def serve_file(load, fnd): Return a chunk from a file based on the data received ''' if 'env' in load: - salt.utils.warn_until( - 'Oxygen', - 'Parameter \'env\' has been detected in the argument list. This ' - 'parameter is no longer used and has been replaced by \'saltenv\' ' - 'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.' - ) + # "env" is not supported; Use "saltenv". load.pop('env') ret = {'data': '', @@ -225,7 +213,7 @@ def serve_file(load, fnd): ret['dest'] = _trim_env_off_path([fnd['path']], load['saltenv'])[0] - with salt.utils.fopen(cached_file_path, 'rb') as fp_: + with salt.utils.files.fopen(cached_file_path, 'rb') as fp_: fp_.seek(load['loc']) data = fp_.read(__opts__['file_buffer_size']) if data and six.PY3 and not salt.utils.is_bin_file(cached_file_path): @@ -242,12 +230,7 @@ def file_list(load): Return a list of all files on the file server in a specified environment ''' if 'env' in load: - salt.utils.warn_until( - 'Oxygen', - 'Parameter \'env\' has been detected in the argument list. This ' - 'parameter is no longer used and has been replaced by \'saltenv\' ' - 'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.' - ) + # "env" is not supported; Use "saltenv". load.pop('env') ret = [] @@ -283,12 +266,7 @@ def dir_list(load): Return a list of all directories on the master ''' if 'env' in load: - salt.utils.warn_until( - 'Oxygen', - 'Parameter \'env\' has been detected in the argument list. This ' - 'parameter is no longer used and has been replaced by \'saltenv\' ' - 'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.' - ) + # "env" is not supported; Use "saltenv". load.pop('env') ret = [] @@ -535,7 +513,7 @@ def _refresh_buckets_cache_file(cache_file): log.debug('Writing buckets cache file') - with salt.utils.fopen(cache_file, 'w') as fp_: + with salt.utils.files.fopen(cache_file, 'w') as fp_: pickle.dump(metadata, fp_) return metadata @@ -548,7 +526,7 @@ def _read_buckets_cache_file(cache_file): log.debug('Reading buckets cache file') - with salt.utils.fopen(cache_file, 'rb') as fp_: + with salt.utils.files.fopen(cache_file, 'rb') as fp_: try: data = pickle.load(fp_) except (pickle.UnpicklingError, AttributeError, EOFError, ImportError, diff --git a/salt/fileserver/svnfs.py b/salt/fileserver/svnfs.py index eb2044c8ad1..2ba3cc227e1 100644 --- a/salt/fileserver/svnfs.py +++ b/salt/fileserver/svnfs.py @@ -2,7 +2,7 @@ ''' Subversion Fileserver Backend -After enabling this backend, branches, and tags in a remote subversion +After enabling this backend, branches and tags in a remote subversion repository are exposed to salt as different environments. To enable this backend, add ``svn`` to the :conf_master:`fileserver_backend` option in the Master config file. @@ -42,7 +42,7 @@ from salt.exceptions import FileserverConfigError PER_REMOTE_OVERRIDES = ('mountpoint', 'root', 'trunk', 'branches', 'tags') # Import third party libs -import salt.ext.six as six +from salt.ext import six # pylint: disable=import-error HAS_SVN = False try: @@ -55,7 +55,10 @@ except ImportError: # Import salt libs import salt.utils +import salt.utils.files +import salt.utils.gzip_util import salt.utils.url +import salt.utils.versions import salt.fileserver from salt.utils.event import tagify @@ -224,7 +227,7 @@ def init(): if new_remote: remote_map = os.path.join(__opts__['cachedir'], 'svnfs/remote_map.txt') try: - with salt.utils.fopen(remote_map, 'w+') as fp_: + with salt.utils.files.fopen(remote_map, 'w+') as fp_: timestamp = datetime.now().strftime('%d %b %Y %H:%M:%S.%f') fp_.write('# svnfs_remote map as of {0}\n'.format(timestamp)) for repo_conf in repos: @@ -367,7 +370,7 @@ def lock(remote=None): failed = [] if not os.path.exists(repo['lockfile']): try: - with salt.utils.fopen(repo['lockfile'], 'w+') as fp_: + with salt.utils.files.fopen(repo['lockfile'], 'w+') as fp_: fp_.write('') except (IOError, OSError) as exc: msg = ('Unable to set update lock for {0} ({1}): {2} ' @@ -453,7 +456,7 @@ def update(): os.makedirs(env_cachedir) new_envs = envs(ignore_cache=True) serial = salt.payload.Serial(__opts__) - with salt.utils.fopen(env_cache, 'wb+') as fp_: + with salt.utils.files.fopen(env_cache, 'wb+') as fp_: fp_.write(serial.dumps(new_envs)) log.trace('Wrote env cache data to {0}'.format(env_cache)) @@ -482,7 +485,7 @@ def _env_is_exposed(env): blacklist. ''' if __opts__['svnfs_env_whitelist']: - salt.utils.warn_until( + salt.utils.versions.warn_until( 'Neon', 'The svnfs_env_whitelist config option has been renamed to ' 'svnfs_saltenv_whitelist. Please update your configuration.' @@ -492,7 +495,7 @@ def _env_is_exposed(env): whitelist = __opts__['svnfs_saltenv_whitelist'] if __opts__['svnfs_env_blacklist']: - salt.utils.warn_until( + salt.utils.versions.warn_until( 'Neon', 'The svnfs_env_blacklist config option has been renamed to ' 'svnfs_saltenv_blacklist. Please update your configuration.' @@ -628,12 +631,7 @@ def serve_file(load, fnd): Return a chunk from a file based on the data received ''' if 'env' in load: - salt.utils.warn_until( - 'Oxygen', - 'Parameter \'env\' has been detected in the argument list. This ' - 'parameter is no longer used and has been replaced by \'saltenv\' ' - 'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.' - ) + # "env" is not supported; Use "saltenv". load.pop('env') ret = {'data': '', @@ -645,7 +643,7 @@ def serve_file(load, fnd): ret['dest'] = fnd['rel'] gzip = load.get('gzip', None) fpath = os.path.normpath(fnd['path']) - with salt.utils.fopen(fpath, 'rb') as fp_: + with salt.utils.files.fopen(fpath, 'rb') as fp_: fp_.seek(load['loc']) data = fp_.read(__opts__['file_buffer_size']) if data and six.PY3 and not salt.utils.is_bin_file(fpath): @@ -662,12 +660,7 @@ def file_hash(load, fnd): Return a file hash, the hash type is set in the master config file ''' if 'env' in load: - salt.utils.warn_until( - 'Oxygen', - 'Parameter \'env\' has been detected in the argument list. This ' - 'parameter is no longer used and has been replaced by \'saltenv\' ' - 'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.' - ) + # "env" is not supported; Use "saltenv". load.pop('env') if not all(x in load for x in ('path', 'saltenv')): @@ -695,7 +688,7 @@ def file_hash(load, fnd): __opts__['hash_type'])) # If we have a cache, serve that if the mtime hasn't changed if os.path.exists(cache_path): - with salt.utils.fopen(cache_path, 'rb') as fp_: + with salt.utils.files.fopen(cache_path, 'rb') as fp_: hsum, mtime = fp_.read().split(':') if os.path.getmtime(path) == mtime: # check if mtime changed @@ -709,7 +702,7 @@ def file_hash(load, fnd): if not os.path.exists(cache_dir): os.makedirs(cache_dir) # save the cache object "hash:mtime" - with salt.utils.fopen(cache_path, 'w') as fp_: + with salt.utils.files.fopen(cache_path, 'w') as fp_: fp_.write('{0}:{1}'.format(ret['hsum'], os.path.getmtime(path))) return ret @@ -717,15 +710,10 @@ def file_hash(load, fnd): def _file_lists(load, form): ''' - Return a dict containing the file lists for files, dirs, emtydirs and symlinks + Return a dict containing the file lists for files, dirs, emptydirs and symlinks ''' if 'env' in load: - salt.utils.warn_until( - 'Oxygen', - 'Parameter \'env\' has been detected in the argument list. This ' - 'parameter is no longer used and has been replaced by \'saltenv\' ' - 'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.' - ) + # "env" is not supported; Use "saltenv". load.pop('env') if 'saltenv' not in load or load['saltenv'] not in envs(): diff --git a/salt/grains/chronos.py b/salt/grains/chronos.py index 24a11eec7d8..e5a46e2ada6 100644 --- a/salt/grains/chronos.py +++ b/salt/grains/chronos.py @@ -8,14 +8,14 @@ Generate chronos proxy minion grains. from __future__ import absolute_import -import salt.utils import salt.utils.http +import salt.utils.platform __proxyenabled__ = ['chronos'] __virtualname__ = 'chronos' def __virtual__(): - if not salt.utils.is_proxy() or 'proxy' not in __opts__: + if not salt.utils.platform.is_proxy() or 'proxy' not in __opts__: return False else: return __virtualname__ diff --git a/salt/grains/cimc.py b/salt/grains/cimc.py new file mode 100644 index 00000000000..e1fba649470 --- /dev/null +++ b/salt/grains/cimc.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- +''' +Generate baseline proxy minion grains for cimc hosts. + +''' + +# Import Python Libs +from __future__ import absolute_import +import logging + +# Import Salt Libs +import salt.utils.platform +import salt.proxy.cimc + +__proxyenabled__ = ['cimc'] +__virtualname__ = 'cimc' + +log = logging.getLogger(__file__) + +GRAINS_CACHE = {'os_family': 'Cisco UCS'} + + +def __virtual__(): + try: + if salt.utils.platform.is_proxy() and __opts__['proxy']['proxytype'] == 'cimc': + return __virtualname__ + except KeyError: + pass + + return False + + +def cimc(proxy=None): + if not proxy: + return {} + if proxy['cimc.initialized']() is False: + return {} + return {'cimc': proxy['cimc.grains']()} diff --git a/salt/grains/core.py b/salt/grains/core.py index 19d489d00dc..bc3eb5ff0ff 100644 --- a/salt/grains/core.py +++ b/salt/grains/core.py @@ -22,6 +22,7 @@ import logging import locale import uuid from errno import EACCES, EPERM +import datetime __proxyenabled__ = ['*'] __FQDN__ = None @@ -42,12 +43,15 @@ except ImportError: import salt.exceptions import salt.log import salt.utils -import salt.utils.network import salt.utils.dns -import salt.ext.six as six +import salt.utils.files +import salt.utils.network +import salt.utils.path +import salt.utils.platform +from salt.ext import six from salt.ext.six.moves import range -if salt.utils.is_windows(): +if salt.utils.platform.is_windows(): import salt.utils.win_osinfo # Solve the Chicken and egg problem where grains need to run before any @@ -65,7 +69,7 @@ __salt__ = { log = logging.getLogger(__name__) HAS_WMI = False -if salt.utils.is_windows(): +if salt.utils.platform.is_windows(): # attempt to import the python wmi module # the Windows minion uses WMI for some of its grains try: @@ -118,7 +122,7 @@ def _linux_cpudata(): cpuinfo = '/proc/cpuinfo' # Parse over the cpuinfo file if os.path.isfile(cpuinfo): - with salt.utils.fopen(cpuinfo, 'r') as _fp: + with salt.utils.files.fopen(cpuinfo, 'r') as _fp: for line in _fp: comps = line.split(':') if not len(comps) > 1: @@ -172,7 +176,7 @@ def _linux_gpu_data(): if __opts__.get('enable_gpu_grains', True) is False: return {} - lspci = salt.utils.which('lspci') + lspci = salt.utils.path.which('lspci') if not lspci: log.debug( 'The `lspci` binary is not available on the system. GPU grains ' @@ -303,8 +307,8 @@ def _bsd_cpudata(osdata): # num_cpus # cpu_model # cpu_flags - sysctl = salt.utils.which('sysctl') - arch = salt.utils.which('arch') + sysctl = salt.utils.path.which('sysctl') + arch = salt.utils.path.which('arch') cmds = {} if sysctl: @@ -337,7 +341,7 @@ def _bsd_cpudata(osdata): if osdata['kernel'] == 'FreeBSD' and os.path.isfile('/var/run/dmesg.boot'): grains['cpu_flags'] = [] # TODO: at least it needs to be tested for BSD other then FreeBSD - with salt.utils.fopen('/var/run/dmesg.boot', 'r') as _fp: + with salt.utils.files.fopen('/var/run/dmesg.boot', 'r') as _fp: cpu_here = False for line in _fp: if line.startswith('CPU: '): @@ -401,7 +405,7 @@ def _memdata(osdata): meminfo = '/proc/meminfo' if os.path.isfile(meminfo): - with salt.utils.fopen(meminfo, 'r') as ifile: + with salt.utils.files.fopen(meminfo, 'r') as ifile: for line in ifile: comps = line.rstrip('\n').split(':') if not len(comps) > 1: @@ -410,7 +414,7 @@ def _memdata(osdata): # Use floor division to force output to be an integer grains['mem_total'] = int(comps[1].split()[0]) // 1024 elif osdata['kernel'] in ('FreeBSD', 'OpenBSD', 'NetBSD', 'Darwin'): - sysctl = salt.utils.which('sysctl') + sysctl = salt.utils.path.which('sysctl') if sysctl: if osdata['kernel'] == 'Darwin': mem = __salt__['cmd.run']('{0} -n hw.memsize'.format(sysctl)) @@ -508,8 +512,8 @@ def _virtual(osdata): _cmds = ['systemd-detect-virt', 'virt-what', 'dmidecode'] # test first for virt-what, which covers most of the desired functionality # on most platforms - if not salt.utils.is_windows() and osdata['kernel'] not in skip_cmds: - if salt.utils.which('virt-what'): + if not salt.utils.platform.is_windows() and osdata['kernel'] not in skip_cmds: + if salt.utils.path.which('virt-what'): _cmds = ['virt-what'] else: log.debug( @@ -539,10 +543,22 @@ def _virtual(osdata): command = 'system_profiler' args = ['SPDisplaysDataType'] elif osdata['kernel'] == 'SunOS': - command = 'prtdiag' - args = [] + virtinfo = salt.utils.path.which('virtinfo') + if virtinfo: + try: + ret = __salt__['cmd.run_all']('{0} -a'.format(virtinfo)) + except salt.exceptions.CommandExecutionError: + if salt.log.is_logging_configured(): + failed_commands.add(virtinfo) + else: + if ret['stdout'].endswith('not supported'): + command = 'prtdiag' + else: + command = 'virtinfo' + else: + command = 'prtdiag' - cmd = salt.utils.which(command) + cmd = salt.utils.path.which(command) if not cmd: continue @@ -557,13 +573,13 @@ def _virtual(osdata): # systemd-detect-virt always returns > 0 on non-virtualized # systems # prtdiag only works in the global zone, skip if it fails - if salt.utils.is_windows() or 'systemd-detect-virt' in cmd or 'prtdiag' in cmd: + if salt.utils.platform.is_windows() or 'systemd-detect-virt' in cmd or 'prtdiag' in cmd: continue failed_commands.add(command) continue except salt.exceptions.CommandExecutionError: if salt.log.is_logging_configured(): - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): continue failed_commands.add(command) continue @@ -687,6 +703,9 @@ def _virtual(osdata): elif 'joyent smartdc hvm' in model: grains['virtual'] = 'kvm' break + elif command == 'virtinfo': + grains['virtual'] = 'LDOM' + break else: if osdata['kernel'] not in skip_cmds: log.debug( @@ -698,7 +717,7 @@ def _virtual(osdata): choices = ('Linux', 'HP-UX') isdir = os.path.isdir - sysctl = salt.utils.which('sysctl') + sysctl = salt.utils.path.which('sysctl') if osdata['kernel'] in choices: if os.path.isdir('/proc'): try: @@ -710,10 +729,10 @@ def _virtual(osdata): pass if os.path.isfile('/proc/1/cgroup'): try: - with salt.utils.fopen('/proc/1/cgroup', 'r') as fhr: + with salt.utils.files.fopen('/proc/1/cgroup', 'r') as fhr: if ':/lxc/' in fhr.read(): grains['virtual_subtype'] = 'LXC' - with salt.utils.fopen('/proc/1/cgroup', 'r') as fhr: + with salt.utils.files.fopen('/proc/1/cgroup', 'r') as fhr: fhr_contents = fhr.read() if ':/docker/' in fhr_contents or ':/system.slice/docker' in fhr_contents: grains['virtual_subtype'] = 'Docker' @@ -729,7 +748,7 @@ def _virtual(osdata): failed_commands.discard('dmidecode') # Provide additional detection for OpenVZ if os.path.isfile('/proc/self/status'): - with salt.utils.fopen('/proc/self/status') as status_file: + with salt.utils.files.fopen('/proc/self/status') as status_file: vz_re = re.compile(r'^envID:\s+(\d+)$') for line in status_file: vz_match = vz_re.match(line.rstrip('\n')) @@ -749,7 +768,7 @@ def _virtual(osdata): grains['virtual_subtype'] = 'Xen HVM DomU' elif os.path.isfile('/proc/xen/capabilities') and \ os.access('/proc/xen/capabilities', os.R_OK): - with salt.utils.fopen('/proc/xen/capabilities') as fhr: + with salt.utils.files.fopen('/proc/xen/capabilities') as fhr: if 'control_d' not in fhr.read(): # Tested on CentOS 5.5 / 2.6.18-194.3.1.el5xen grains['virtual_subtype'] = 'Xen PV DomU' @@ -769,12 +788,12 @@ def _virtual(osdata): if 'dom' in grains.get('virtual_subtype', '').lower(): grains['virtual'] = 'xen' if os.path.isfile('/proc/cpuinfo'): - with salt.utils.fopen('/proc/cpuinfo', 'r') as fhr: + with salt.utils.files.fopen('/proc/cpuinfo', 'r') as fhr: if 'QEMU Virtual CPU' in fhr.read(): grains['virtual'] = 'kvm' if os.path.isfile('/sys/devices/virtual/dmi/id/product_name'): try: - with salt.utils.fopen('/sys/devices/virtual/dmi/id/product_name', 'r') as fhr: + with salt.utils.files.fopen('/sys/devices/virtual/dmi/id/product_name', 'r') as fhr: output = fhr.read() if 'VirtualBox' in output: grains['virtual'] = 'VirtualBox' @@ -786,10 +805,12 @@ def _virtual(osdata): grains['virtual_subtype'] = 'ovirt' elif 'Google' in output: grains['virtual'] = 'gce' + elif 'BHYVE' in output: + grains['virtual'] = 'bhyve' except IOError: pass elif osdata['kernel'] == 'FreeBSD': - kenv = salt.utils.which('kenv') + kenv = salt.utils.path.which('kenv') if kenv: product = __salt__['cmd.run']( '{0} smbios.system.product'.format(kenv) @@ -823,20 +844,30 @@ def _virtual(osdata): if 'QEMU Virtual CPU' in model: grains['virtual'] = 'kvm' elif osdata['kernel'] == 'OpenBSD': - if osdata['manufacturer'] == 'QEMU': + if osdata['manufacturer'] in ['QEMU', 'Red Hat']: grains['virtual'] = 'kvm' elif osdata['kernel'] == 'SunOS': - # Check if it's a "regular" zone. (i.e. Solaris 10/11 zone) - zonename = salt.utils.which('zonename') - if zonename: - zone = __salt__['cmd.run']('{0}'.format(zonename)) - if zone != 'global': + if grains['virtual'] == 'LDOM': + roles = [] + for role in ('control', 'io', 'root', 'service'): + subtype_cmd = '{0} -c current get -H -o value {1}-role'.format(cmd, role) + ret = __salt__['cmd.run_all']('{0}'.format(subtype_cmd)) + if ret['stdout'] == 'true': + roles.append(role) + if roles: + grains['virtual_subtype'] = roles + else: + # Check if it's a "regular" zone. (i.e. Solaris 10/11 zone) + zonename = salt.utils.path.which('zonename') + if zonename: + zone = __salt__['cmd.run']('{0}'.format(zonename)) + if zone != 'global': + grains['virtual'] = 'zone' + if salt.utils.platform.is_smartos_zone(): + grains.update(_smartos_zone_data()) + # Check if it's a branded zone (i.e. Solaris 8/9 zone) + if isdir('/.SUNWnative'): grains['virtual'] = 'zone' - if salt.utils.is_smartos_zone(): - grains.update(_smartos_zone_data()) - # Check if it's a branded zone (i.e. Solaris 8/9 zone) - if isdir('/.SUNWnative'): - grains['virtual'] = 'zone' elif osdata['kernel'] == 'NetBSD': if sysctl: if 'QEMU Virtual CPU' in __salt__['cmd.run']( @@ -983,28 +1014,20 @@ def _windows_platform_data(): os_release = platform.release() kernel_version = platform.version() info = salt.utils.win_osinfo.get_os_version_info() + server = {'Vista': '2008Server', + '7': '2008ServerR2', + '8': '2012Server', + '8.1': '2012ServerR2', + '10': '2016Server'} # Starting with Python 2.7.12 and 3.5.2 the `platform.uname()` function # started reporting the Desktop version instead of the Server version on # Server versions of Windows, so we need to look those up - # Check for Python >=2.7.12 or >=3.5.2 - ver = pythonversion()['pythonversion'] - if ((six.PY2 and - salt.utils.compare_versions(ver, '>=', [2, 7, 12, 'final', 0])) - or - (six.PY3 and - salt.utils.compare_versions(ver, '>=', [3, 5, 2, 'final', 0]))): - # (Product Type 1 is Desktop, Everything else is Server) - if info['ProductType'] > 1: - server = {'Vista': '2008Server', - '7': '2008ServerR2', - '8': '2012Server', - '8.1': '2012ServerR2', - '10': '2016Server'} - os_release = server.get(os_release, - 'Grain not found. Update lookup table ' - 'in the `_windows_platform_data` ' - 'function in `grains\\core.py`') + # So, if you find a Server Platform that's a key in the server + # dictionary, then lookup the actual Server Release. + # (Product Type 1 is Desktop, Everything else is Server) + if info['ProductType'] > 1 and os_release in server: + os_release = server[os_release] service_pack = None if info['ServicePackMajor'] > 0: @@ -1117,10 +1140,11 @@ _OS_NAME_MAP = { 'nilrt': 'NILinuxRT', 'nilrt-xfce': 'NILinuxRT-XFCE', 'manjaro': 'Manjaro', + 'manjarolin': 'Manjaro', 'antergos': 'Antergos', 'sles': 'SUSE', - 'slesexpand': 'RES', 'void': 'Void', + 'slesexpand': 'RES', 'linuxmint': 'Mint', 'neon': 'KDE neon', } @@ -1185,6 +1209,7 @@ _OS_FAMILY_MAP = { 'NILinuxRT-XFCE': 'NILinuxRT', 'KDE neon': 'Debian', 'Void': 'Void', + 'IDMS': 'Debian', } @@ -1232,7 +1257,7 @@ def _parse_os_release(): filename = '/usr/lib/os-release' data = dict() - with salt.utils.fopen(filename) as ifile: + with salt.utils.files.fopen(filename) as ifile: regex = re.compile('^([\\w]+)=(?:\'|")?(.*?)(?:\'|")?$') for line in ifile: match = regex.match(line.strip()) @@ -1265,7 +1290,7 @@ def os_data(): grains['kernelrelease'], grains['kernelversion'], grains['cpuarch'], _) = platform.uname() # pylint: enable=unpacking-non-sequence - if salt.utils.is_proxy(): + if salt.utils.platform.is_proxy(): grains['kernel'] = 'proxy' grains['kernelrelease'] = 'proxy' grains['kernelversion'] = 'proxy' @@ -1273,7 +1298,7 @@ def os_data(): grains['os'] = 'proxy' grains['os_family'] = 'proxy' grains['osfullname'] = 'proxy' - elif salt.utils.is_windows(): + elif salt.utils.platform.is_windows(): grains['os'] = 'Windows' grains['os_family'] = 'Windows' grains.update(_memdata(grains)) @@ -1301,7 +1326,7 @@ def os_data(): grains['init'] = 'Windows' return grains - elif salt.utils.is_linux(): + elif salt.utils.platform.is_linux(): # Add SELinux grain, if you have it if _linux_bin_exists('selinuxenabled'): grains['selinux'] = {} @@ -1329,10 +1354,10 @@ def os_data(): grains['init'] = 'systemd' except (OSError, IOError): if os.path.exists('/proc/1/cmdline'): - with salt.utils.fopen('/proc/1/cmdline') as fhr: + with salt.utils.files.fopen('/proc/1/cmdline') as fhr: init_cmdline = fhr.read().replace('\x00', ' ').split() try: - init_bin = salt.utils.which(init_cmdline[0]) + init_bin = salt.utils.path.which(init_cmdline[0]) except IndexError: # Emtpy init_cmdline init_bin = None @@ -1348,7 +1373,7 @@ def os_data(): # Default to the value of file_buffer_size for the minion buf_size = 262144 try: - with salt.utils.fopen(init_bin, 'rb') as fp_: + with salt.utils.files.fopen(init_bin, 'rb') as fp_: buf = True edge = six.b('') buf = fp_.read(buf_size).lower() @@ -1368,7 +1393,7 @@ def os_data(): 'Unable to read from init_bin ({0}): {1}' .format(init_bin, exc) ) - elif salt.utils.which('supervisord') in init_cmdline: + elif salt.utils.path.which('supervisord') in init_cmdline: grains['init'] = 'supervisord' elif init_cmdline == ['runit']: grains['init'] = 'runit' @@ -1405,7 +1430,7 @@ def os_data(): '^(DISTRIB_(?:ID|RELEASE|CODENAME|DESCRIPTION))=(?:\'|")?' '([\\w\\s\\.\\-_]+)(?:\'|")?' )) - with salt.utils.fopen('/etc/lsb-release') as ifile: + with salt.utils.files.fopen('/etc/lsb-release') as ifile: for line in ifile: match = regex.match(line.rstrip('\n')) if match: @@ -1440,7 +1465,7 @@ def os_data(): grains['lsb_distrib_id'] = 'SUSE' version = '' patch = '' - with salt.utils.fopen('/etc/SuSE-release') as fhr: + with salt.utils.files.fopen('/etc/SuSE-release') as fhr: for line in fhr: if 'enterprise' in line.lower(): grains['lsb_distrib_id'] = 'SLES' @@ -1460,7 +1485,7 @@ def os_data(): elif os.path.isfile('/etc/altlinux-release'): # ALT Linux grains['lsb_distrib_id'] = 'altlinux' - with salt.utils.fopen('/etc/altlinux-release') as ifile: + with salt.utils.files.fopen('/etc/altlinux-release') as ifile: # This file is symlinked to from: # /etc/fedora-release # /etc/redhat-release @@ -1475,7 +1500,7 @@ def os_data(): elif os.path.isfile('/etc/centos-release'): # CentOS Linux grains['lsb_distrib_id'] = 'CentOS' - with salt.utils.fopen('/etc/centos-release') as ifile: + with salt.utils.files.fopen('/etc/centos-release') as ifile: for line in ifile: # Need to pull out the version and codename # in the case of custom content in /etc/centos-release @@ -1490,7 +1515,7 @@ def os_data(): elif os.path.isfile('/etc.defaults/VERSION') \ and os.path.isfile('/etc.defaults/synoinfo.conf'): grains['osfullname'] = 'Synology' - with salt.utils.fopen('/etc.defaults/VERSION', 'r') as fp_: + with salt.utils.files.fopen('/etc.defaults/VERSION', 'r') as fp_: synoinfo = {} for line in fp_: try: @@ -1547,7 +1572,7 @@ def os_data(): grains.update(_linux_cpudata()) grains.update(_linux_gpu_data()) elif grains['kernel'] == 'SunOS': - if salt.utils.is_smartos(): + if salt.utils.platform.is_smartos(): # See https://github.com/joyent/smartos-live/issues/224 uname_v = os.uname()[3] # format: joyent_20161101T004406Z uname_v = uname_v[uname_v.index('_')+1:] @@ -1560,10 +1585,10 @@ def os_data(): ]) # store a untouched copy of the timestamp in osrelease_stamp grains['osrelease_stamp'] = uname_v - if salt.utils.is_smartos_globalzone(): + if salt.utils.platform.is_smartos_globalzone(): grains.update(_smartos_computenode_data()) elif os.path.isfile('/etc/release'): - with salt.utils.fopen('/etc/release', 'r') as fp_: + with salt.utils.files.fopen('/etc/release', 'r') as fp_: rel_data = fp_.read() try: release_re = re.compile( @@ -1705,7 +1730,7 @@ def locale_info(): grains = {} grains['locale_info'] = {} - if salt.utils.is_proxy(): + if salt.utils.platform.is_proxy(): return grains try: @@ -1735,7 +1760,7 @@ def hostname(): global __FQDN__ grains = {} - if salt.utils.is_proxy(): + if salt.utils.platform.is_proxy(): return grains grains['localhost'] = socket.gethostname() @@ -1763,7 +1788,7 @@ def append_domain(): grain = {} - if salt.utils.is_proxy(): + if salt.utils.platform.is_proxy(): return grain if 'append_domain' in __opts__: @@ -1775,7 +1800,7 @@ def ip_fqdn(): ''' Return ip address and FQDN grains ''' - if salt.utils.is_proxy(): + if salt.utils.platform.is_proxy(): return {} ret = {} @@ -1789,12 +1814,14 @@ def ip_fqdn(): ret[key] = [] else: try: + start_time = datetime.datetime.utcnow() info = socket.getaddrinfo(_fqdn, None, socket_type) ret[key] = list(set(item[4][0] for item in info)) except socket.error: - if __opts__['__role'] == 'master': - log.warning('Unable to find IPv{0} record for "{1}" causing a 10 second timeout when rendering grains. ' - 'Set the dns or /etc/hosts for IPv{0} to clear this.'.format(ipv_num, _fqdn)) + timediff = datetime.datetime.utcnow() - start_time + if timediff.seconds > 5 and __opts__['__role'] == 'master': + log.warning('Unable to find IPv{0} record for "{1}" causing a {2} second timeout when rendering grains. ' + 'Set the dns or /etc/hosts for IPv{0} to clear this.'.format(ipv_num, _fqdn, timediff)) ret[key] = [] return ret @@ -1808,7 +1835,7 @@ def ip_interfaces(): # Provides: # ip_interfaces - if salt.utils.is_proxy(): + if salt.utils.platform.is_proxy(): return {} ret = {} @@ -1836,7 +1863,7 @@ def ip4_interfaces(): # Provides: # ip_interfaces - if salt.utils.is_proxy(): + if salt.utils.platform.is_proxy(): return {} ret = {} @@ -1861,7 +1888,7 @@ def ip6_interfaces(): # Provides: # ip_interfaces - if salt.utils.is_proxy(): + if salt.utils.platform.is_proxy(): return {} ret = {} @@ -1901,7 +1928,7 @@ def dns(): ''' # Provides: # dns - if salt.utils.is_windows() or 'proxyminion' in __opts__: + if salt.utils.platform.is_windows() or 'proxyminion' in __opts__: return {} resolv = salt.utils.dns.parse_resolv() @@ -1924,7 +1951,7 @@ def get_machine_id(): if not existing_locations: return {} else: - with salt.utils.fopen(existing_locations[0]) as machineid: + with salt.utils.files.fopen(existing_locations[0]) as machineid: return {'machine_id': machineid.read().strip()} @@ -2024,7 +2051,7 @@ def _hw_data(osdata): .. versionadded:: 0.9.5 ''' - if salt.utils.is_proxy(): + if salt.utils.platform.is_proxy(): return {} grains = {} @@ -2043,7 +2070,7 @@ def _hw_data(osdata): contents_file = os.path.join('/sys/class/dmi/id', fw_file) if os.path.exists(contents_file): try: - with salt.utils.fopen(contents_file, 'r') as ifile: + with salt.utils.files.fopen(contents_file, 'r') as ifile: grains[key] = ifile.read() if key == 'uuid': grains['uuid'] = grains['uuid'].lower() @@ -2053,8 +2080,8 @@ def _hw_data(osdata): if err.errno == EACCES or err.errno == EPERM: # Skip the grain if non-root user has no access to the file. pass - elif salt.utils.which_bin(['dmidecode', 'smbios']) is not None and not ( - salt.utils.is_smartos() or + elif salt.utils.path.which_bin(['dmidecode', 'smbios']) is not None and not ( + salt.utils.platform.is_smartos() or ( # SunOS on SPARC - 'smbios: failed to load SMBIOS: System does not export an SMBIOS table' osdata['kernel'] == 'SunOS' and osdata['cpuarch'].startswith('sparc') @@ -2077,7 +2104,7 @@ def _hw_data(osdata): if serial is not None: grains['serialnumber'] = serial break - elif salt.utils.which_bin(['fw_printenv']) is not None: + elif salt.utils.path.which_bin(['fw_printenv']) is not None: # ARM Linux devices expose UBOOT env variables via fw_printenv hwdata = { 'manufacturer': 'manufacturer', @@ -2091,7 +2118,7 @@ def _hw_data(osdata): elif osdata['kernel'] == 'FreeBSD': # On FreeBSD /bin/kenv (already in base system) # can be used instead of dmidecode - kenv = salt.utils.which('kenv') + kenv = salt.utils.path.which('kenv') if kenv: # In theory, it will be easier to add new fields to this later fbsd_hwdata = { @@ -2106,7 +2133,7 @@ def _hw_data(osdata): value = __salt__['cmd.run']('{0} {1}'.format(kenv, val)) grains[key] = _clean_value(key, value) elif osdata['kernel'] == 'OpenBSD': - sysctl = salt.utils.which('sysctl') + sysctl = salt.utils.path.which('sysctl') hwdata = {'biosversion': 'hw.version', 'manufacturer': 'hw.vendor', 'productname': 'hw.product', @@ -2117,7 +2144,7 @@ def _hw_data(osdata): if not value.endswith(' value is not available'): grains[key] = _clean_value(key, value) elif osdata['kernel'] == 'NetBSD': - sysctl = salt.utils.which('sysctl') + sysctl = salt.utils.path.which('sysctl') nbsd_hwdata = { 'biosversion': 'machdep.dmi.board-version', 'manufacturer': 'machdep.dmi.system-vendor', @@ -2132,7 +2159,7 @@ def _hw_data(osdata): grains[key] = _clean_value(key, result['stdout']) elif osdata['kernel'] == 'Darwin': grains['manufacturer'] = 'Apple Inc.' - sysctl = salt.utils.which('sysctl') + sysctl = salt.utils.path.which('sysctl') hwdata = {'productname': 'hw.model'} for key, oid in hwdata.items(): value = __salt__['cmd.run']('{0} -b {1}'.format(sysctl, oid)) @@ -2144,7 +2171,7 @@ def _hw_data(osdata): # commands and attempt various lookups. data = "" for (cmd, args) in (('/usr/sbin/prtdiag', '-v'), ('/usr/sbin/prtconf', '-vp'), ('/usr/sbin/virtinfo', '-a')): - if salt.utils.which(cmd): # Also verifies that cmd is executable + if salt.utils.path.which(cmd): # Also verifies that cmd is executable data += __salt__['cmd.run']('{0} {1}'.format(cmd, args)) data += '\n' @@ -2270,7 +2297,7 @@ def _smartos_computenode_data(): # vm_capable # vm_hw_virt - if salt.utils.is_proxy(): + if salt.utils.platform.is_proxy(): return {} grains = {} @@ -2308,7 +2335,7 @@ def _smartos_zone_data(): # hypervisor_uuid # datacenter - if salt.utils.is_proxy(): + if salt.utils.platform.is_proxy(): return {} grains = {} @@ -2317,19 +2344,19 @@ def _smartos_zone_data(): imageversion = re.compile('Image:\\s(.+)') pkgsrcpath = re.compile('PKG_PATH=(.+)') if os.path.isfile('/etc/pkgsrc_version'): - with salt.utils.fopen('/etc/pkgsrc_version', 'r') as fp_: + with salt.utils.files.fopen('/etc/pkgsrc_version', 'r') as fp_: for line in fp_: match = pkgsrcversion.match(line) if match: grains['pkgsrcversion'] = match.group(1) if os.path.isfile('/etc/product'): - with salt.utils.fopen('/etc/product', 'r') as fp_: + with salt.utils.files.fopen('/etc/product', 'r') as fp_: for line in fp_: match = imageversion.match(line) if match: grains['imageversion'] = match.group(1) if os.path.isfile('/opt/local/etc/pkg_install.conf'): - with salt.utils.fopen('/opt/local/etc/pkg_install.conf', 'r') as fp_: + with salt.utils.files.fopen('/opt/local/etc/pkg_install.conf', 'r') as fp_: for line in fp_: match = pkgsrcpath.match(line) if match: @@ -2352,11 +2379,15 @@ def _zpool_data(grains): Provide grains about zpools ''' # quickly return if windows or proxy - if salt.utils.is_windows() or 'proxyminion' in __opts__: + if salt.utils.platform.is_windows() or 'proxyminion' in __opts__: + return {} + + # quickly return if NetBSD (ZFS still under development) + if salt.utils.platform.is_netbsd(): return {} # quickly return if no zpool and zfs command - if not salt.utils.which('zpool'): + if not salt.utils.path.which('zpool'): return {} # collect zpool data @@ -2380,7 +2411,7 @@ def get_server_id(): # Provides: # server_id - if salt.utils.is_proxy(): + if salt.utils.platform.is_proxy(): return {} return {'server_id': abs(hash(__opts__.get('id', '')) % (2 ** 31))} @@ -2394,4 +2425,46 @@ def get_master(): # master return {'master': __opts__.get('master', '')} -# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 + +def default_gateway(): + ''' + Populates grains which describe whether a server has a default gateway + configured or not. Uses `ip -4 route show` and `ip -6 route show` and greps + for a `default` at the beginning of any line. Assuming the standard + `default via ` format for default gateways, it will also parse out the + ip address of the default gateway, and put it in ip4_gw or ip6_gw. + + If the `ip` command is unavailable, no grains will be populated. + + Currently does not support multiple default gateways. The grains will be + set to the first default gateway found. + + List of grains: + + ip4_gw: True # ip/True/False if default ipv4 gateway + ip6_gw: True # ip/True/False if default ipv6 gateway + ip_gw: True # True if either of the above is True, False otherwise + ''' + grains = {} + if not salt.utils.path.which('ip'): + return {} + grains['ip_gw'] = False + grains['ip4_gw'] = False + grains['ip6_gw'] = False + if __salt__['cmd.run']('ip -4 route show | grep "^default"', python_shell=True): + grains['ip_gw'] = True + grains['ip4_gw'] = True + try: + gateway_ip = __salt__['cmd.run']('ip -4 route show | grep "^default via"', python_shell=True).split(' ')[2].strip() + grains['ip4_gw'] = gateway_ip if gateway_ip else True + except Exception as exc: + pass + if __salt__['cmd.run']('ip -6 route show | grep "^default"', python_shell=True): + grains['ip_gw'] = True + grains['ip6_gw'] = True + try: + gateway_ip = __salt__['cmd.run']('ip -6 route show | grep "^default via"', python_shell=True).split(' ')[2].strip() + grains['ip6_gw'] = gateway_ip if gateway_ip else True + except Exception as exc: + pass + return grains diff --git a/salt/grains/disks.py b/salt/grains/disks.py index ce406828424..0d76a579c87 100644 --- a/salt/grains/disks.py +++ b/salt/grains/disks.py @@ -10,14 +10,17 @@ import logging import re # Import salt libs -import salt.utils +import salt.utils.files +import salt.utils.path +import salt.utils.platform # Solve the Chicken and egg problem where grains need to run before any # of the modules are loaded and are generally available for any usage. import salt.modules.cmdmod __salt__ = { - 'cmd.run': salt.modules.cmdmod._run_quiet + 'cmd.run': salt.modules.cmdmod._run_quiet, + 'cmd.run_all': salt.modules.cmdmod._run_all_quiet } log = logging.getLogger(__name__) @@ -27,10 +30,12 @@ def disks(): ''' Return list of disk devices ''' - if salt.utils.is_freebsd(): + if salt.utils.platform.is_freebsd(): return _freebsd_geom() - elif salt.utils.is_linux(): + elif salt.utils.platform.is_linux(): return _linux_disks() + elif salt.utils.platform.is_windows(): + return _windows_disks() else: log.trace('Disk grain does not support OS') @@ -87,7 +92,7 @@ _geom_attribs = [_geomconsts.__dict__[key] for key in def _freebsd_geom(): - geom = salt.utils.which('geom') + geom = salt.utils.path.which('geom') ret = {'disks': {}, 'SSDs': []} devices = __salt__['cmd.run']('{0} disk list'.format(geom)) @@ -127,7 +132,7 @@ def _linux_disks(): ret = {'disks': [], 'SSDs': []} for entry in glob.glob('/sys/block/*/queue/rotational'): - with salt.utils.fopen(entry) as entry_fp: + with salt.utils.files.fopen(entry) as entry_fp: device = entry.split('/')[3] flag = entry_fp.read(1) if flag == '0': @@ -140,3 +145,39 @@ def _linux_disks(): log.trace('Unable to identify device {0} as an SSD or HDD.' ' It does not report 0 or 1'.format(device)) return ret + + +def _windows_disks(): + wmic = salt.utils.path.which('wmic') + + namespace = r'\\root\microsoft\windows\storage' + path = 'MSFT_PhysicalDisk' + where = '(MediaType=3 or MediaType=4)' + get = 'DeviceID,MediaType' + + ret = {'disks': [], 'SSDs': []} + + cmdret = __salt__['cmd.run_all']( + '{0} /namespace:{1} path {2} where {3} get {4} /format:table'.format( + wmic, namespace, path, where, get)) + + if cmdret['retcode'] != 0: + log.trace('Disk grain does not support this version of Windows') + else: + for line in cmdret['stdout'].splitlines(): + info = line.split() + if len(info) != 2 or not info[0].isdigit() or not info[1].isdigit(): + continue + device = r'\\.\PhysicalDrive{0}'.format(info[0]) + mediatype = info[1] + if mediatype == '3': + log.trace('Device {0} reports itself as an HDD'.format(device)) + ret['disks'].append(device) + elif mediatype == '4': + log.trace('Device {0} reports itself as an SSD'.format(device)) + ret['SSDs'].append(device) + else: + log.trace('Unable to identify device {0} as an SSD or HDD.' + 'It does not report 3 or 4'.format(device)) + + return ret diff --git a/salt/grains/esxi.py b/salt/grains/esxi.py index f4cf1b79cdd..c548b24a55e 100644 --- a/salt/grains/esxi.py +++ b/salt/grains/esxi.py @@ -12,7 +12,7 @@ import logging # Import Salt Libs from salt.exceptions import SaltSystemExit -import salt.utils +import salt.utils.platform import salt.modules.vsphere __proxyenabled__ = ['esxi'] @@ -26,7 +26,7 @@ GRAINS_CACHE = {} def __virtual__(): try: - if salt.utils.is_proxy() and __opts__['proxy']['proxytype'] == 'esxi': + if salt.utils.platform.is_proxy() and __opts__['proxy']['proxytype'] == 'esxi': return __virtualname__ except KeyError: pass diff --git a/salt/grains/extra.py b/salt/grains/extra.py index 228446675b8..e89386ba21d 100644 --- a/salt/grains/extra.py +++ b/salt/grains/extra.py @@ -10,7 +10,8 @@ import yaml import logging # Import salt libs -import salt.utils +import salt.utils.files +import salt.utils.platform log = logging.getLogger(__name__) @@ -21,7 +22,14 @@ def shell(): ''' # Provides: # shell - return {'shell': os.environ.get('SHELL', '/bin/sh')} + if salt.utils.platform.is_windows(): + env_var = 'COMSPEC' + default = r'C:\Windows\system32\cmd.exe' + else: + env_var = 'SHELL' + default = '/bin/sh' + + return {'shell': os.environ.get(env_var, default)} def config(): @@ -41,7 +49,7 @@ def config(): 'grains' ) if os.path.isfile(gfn): - with salt.utils.fopen(gfn, 'rb') as fp_: + with salt.utils.files.fopen(gfn, 'rb') as fp_: try: return yaml.safe_load(fp_.read()) except Exception: diff --git a/salt/grains/fx2.py b/salt/grains/fx2.py index fa4069775ed..b4c723346d2 100644 --- a/salt/grains/fx2.py +++ b/salt/grains/fx2.py @@ -7,11 +7,11 @@ in proxy/fx2.py--just enough to get data from the chassis to include in grains. ''' from __future__ import absolute_import -import salt.utils import logging import salt.proxy.fx2 import salt.modules.cmdmod import salt.modules.dracr +import salt.utils.platform __proxyenabled__ = ['fx2'] @@ -25,7 +25,7 @@ GRAINS_CACHE = {} def __virtual__(): try: - if salt.utils.is_proxy() and __opts__['proxy']['proxytype'] == 'fx2': + if salt.utils.platform.is_proxy() and __opts__['proxy']['proxytype'] == 'fx2': return __virtualname__ except KeyError: pass diff --git a/salt/grains/junos.py b/salt/grains/junos.py index 34e33f4cbcb..c2fd0ba1b03 100644 --- a/salt/grains/junos.py +++ b/salt/grains/junos.py @@ -11,7 +11,7 @@ from __future__ import absolute_import import logging # Import Salt libs -import salt.ext.six as six +from salt.ext import six __proxyenabled__ = ['junos'] __virtualname__ = 'junos' diff --git a/salt/grains/marathon.py b/salt/grains/marathon.py index 330e1fe4860..82c3f75716b 100644 --- a/salt/grains/marathon.py +++ b/salt/grains/marathon.py @@ -7,14 +7,14 @@ Generate marathon proxy minion grains. ''' from __future__ import absolute_import -import salt.utils import salt.utils.http +import salt.utils.platform __proxyenabled__ = ['marathon'] __virtualname__ = 'marathon' def __virtual__(): - if not salt.utils.is_proxy() or 'proxy' not in __opts__: + if not salt.utils.platform.is_proxy() or 'proxy' not in __opts__: return False else: return __virtualname__ diff --git a/salt/grains/mdadm.py b/salt/grains/mdadm.py index 2ea0abcf844..c6a1952d0a0 100644 --- a/salt/grains/mdadm.py +++ b/salt/grains/mdadm.py @@ -8,7 +8,7 @@ from __future__ import absolute_import import logging # Import salt libs -import salt.utils +import salt.utils.files log = logging.getLogger(__name__) @@ -19,7 +19,7 @@ def mdadm(): ''' devices = set() try: - with salt.utils.fopen('/proc/mdstat', 'r') as mdstat: + with salt.utils.files.fopen('/proc/mdstat', 'r') as mdstat: for line in mdstat: if line.startswith('Personalities : '): continue diff --git a/salt/grains/mdata.py b/salt/grains/mdata.py index 59093db3338..9e528e8cfe8 100644 --- a/salt/grains/mdata.py +++ b/salt/grains/mdata.py @@ -18,8 +18,9 @@ import json import logging # Import salt libs -import salt.utils import salt.utils.dictupdate +import salt.utils.path +import salt.utils.platform # Solve the Chicken and egg problem where grains need to run before any # of the modules are loaded and are generally available for any usage. @@ -39,10 +40,10 @@ def __virtual__(): Figure out if we need to be loaded ''' ## collect mdata grains in a SmartOS zone - if salt.utils.is_smartos_zone(): + if salt.utils.platform.is_smartos_zone(): return __virtualname__ ## collect mdata grains in a LX zone - if salt.utils.is_linux() and 'BrandZ virtual linux' in os.uname(): + if salt.utils.platform.is_linux() and 'BrandZ virtual linux' in os.uname(): return __virtualname__ return False @@ -54,10 +55,10 @@ def _user_mdata(mdata_list=None, mdata_get=None): grains = {} if not mdata_list: - mdata_list = salt.utils.which('mdata-list') + mdata_list = salt.utils.path.which('mdata-list') if not mdata_get: - mdata_get = salt.utils.which('mdata-get') + mdata_get = salt.utils.path.which('mdata-get') if not mdata_list or not mdata_get: return grains @@ -97,10 +98,10 @@ def _sdc_mdata(mdata_list=None, mdata_get=None): ] if not mdata_list: - mdata_list = salt.utils.which('mdata-list') + mdata_list = salt.utils.path.which('mdata-list') if not mdata_get: - mdata_get = salt.utils.which('mdata-get') + mdata_get = salt.utils.path.which('mdata-get') if not mdata_list or not mdata_get: return grains @@ -154,8 +155,8 @@ def mdata(): Provide grains from the SmartOS metadata ''' grains = {} - mdata_list = salt.utils.which('mdata-list') - mdata_get = salt.utils.which('mdata-get') + mdata_list = salt.utils.path.which('mdata-list') + mdata_get = salt.utils.path.which('mdata-get') grains = salt.utils.dictupdate.update(grains, _user_mdata(mdata_list, mdata_get), merge_lists=True) grains = salt.utils.dictupdate.update(grains, _sdc_mdata(mdata_list, mdata_get), merge_lists=True) diff --git a/salt/grains/metadata.py b/salt/grains/metadata.py index 2372aac6c70..15ed571a76e 100644 --- a/salt/grains/metadata.py +++ b/salt/grains/metadata.py @@ -17,6 +17,7 @@ metadata server set `metadata_server_grains: True`. from __future__ import absolute_import # Import python libs +import json import os import socket @@ -47,14 +48,30 @@ def _search(prefix="latest/"): Recursively look up all grains in the metadata server ''' ret = {} - for line in http.query(os.path.join(HOST, prefix))['body'].split('\n'): + linedata = http.query(os.path.join(HOST, prefix)) + if 'body' not in linedata: + return ret + for line in linedata['body'].split('\n'): if line.endswith('/'): ret[line[:-1]] = _search(prefix=os.path.join(prefix, line)) + elif prefix == 'latest/': + # (gtmanfred) The first level should have a forward slash since + # they have stuff underneath. This will not be doubled up though, + # because lines ending with a slash are checked first. + ret[line] = _search(prefix=os.path.join(prefix, line + '/')) + elif line.endswith(('dynamic', 'meta-data')): + ret[line] = _search(prefix=os.path.join(prefix, line)) elif '=' in line: key, value = line.split('=') ret[value] = _search(prefix=os.path.join(prefix, key)) else: - ret[line] = http.query(os.path.join(HOST, prefix, line))['body'] + retdata = http.query(os.path.join(HOST, prefix, line)).get('body', None) + # (gtmanfred) This try except block is slightly faster than + # checking if the string starts with a curly brace + try: + ret[line] = json.loads(retdata) + except ValueError: + ret[line] = retdata return ret diff --git a/salt/grains/minion_process.py b/salt/grains/minion_process.py index 442e6758cb1..e1aa762b4de 100644 --- a/salt/grains/minion_process.py +++ b/salt/grains/minion_process.py @@ -8,7 +8,7 @@ from __future__ import absolute_import import os # Import salt libs -import salt.utils +import salt.utils.platform try: import pwd @@ -23,14 +23,18 @@ except ImportError: def _uid(): - '''Grain for the minion User ID''' - if salt.utils.is_windows(): + ''' + Grain for the minion User ID + ''' + if salt.utils.platform.is_windows(): return None return os.getuid() def _username(): - '''Grain for the minion username''' + ''' + Grain for the minion username + ''' if pwd: username = pwd.getpwuid(os.getuid()).pw_name else: @@ -40,14 +44,18 @@ def _username(): def _gid(): - '''Grain for the minion Group ID''' - if salt.utils.is_windows(): + ''' + Grain for the minion Group ID + ''' + if salt.utils.platform.is_windows(): return None return os.getgid() def _groupname(): - '''Grain for the minion groupname''' + ''' + Grain for the minion groupname + ''' if grp: groupname = grp.getgrgid(os.getgid()).gr_name else: @@ -67,7 +75,7 @@ def grains(): 'pid': _pid(), } - if not salt.utils.is_windows(): + if not salt.utils.platform.is_windows(): ret['gid'] = _gid() ret['uid'] = _uid() diff --git a/salt/grains/napalm.py b/salt/grains/napalm.py index 6ac52464c8e..fcfbdcfe9fc 100644 --- a/salt/grains/napalm.py +++ b/salt/grains/napalm.py @@ -447,8 +447,8 @@ def optional_args(proxy=None): device2: True ''' - opt_args = _get_device_grain('optional_args', proxy=proxy) - if _FORBIDDEN_OPT_ARGS: + opt_args = _get_device_grain('optional_args', proxy=proxy) or {} + if opt_args and _FORBIDDEN_OPT_ARGS: for arg in _FORBIDDEN_OPT_ARGS: opt_args.pop(arg, None) return {'optional_args': opt_args} diff --git a/salt/grains/nxos.py b/salt/grains/nxos.py index 13fbeb8930c..84df6570727 100644 --- a/salt/grains/nxos.py +++ b/salt/grains/nxos.py @@ -11,7 +11,7 @@ for :mod:`salt.proxy.nxos`. from __future__ import absolute_import # Import Salt Libs -import salt.utils +import salt.utils.platform import salt.modules.nxos import logging @@ -23,7 +23,7 @@ __virtualname__ = 'nxos' def __virtual__(): try: - if salt.utils.is_proxy() and __opts__['proxy']['proxytype'] == 'nxos': + if salt.utils.platform.is_proxy() and __opts__['proxy']['proxytype'] == 'nxos': return __virtualname__ except KeyError: pass diff --git a/salt/grains/panos.py b/salt/grains/panos.py new file mode 100644 index 00000000000..d39de5f94bf --- /dev/null +++ b/salt/grains/panos.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- +''' +Generate baseline proxy minion grains for panos hosts. + +''' + +# Import Python Libs +from __future__ import absolute_import +import logging + +# Import Salt Libs +import salt.utils.platform +import salt.proxy.panos + +__proxyenabled__ = ['panos'] +__virtualname__ = 'panos' + +log = logging.getLogger(__file__) + +GRAINS_CACHE = {'os_family': 'panos'} + + +def __virtual__(): + try: + if salt.utils.platform.is_proxy() and __opts__['proxy']['proxytype'] == 'panos': + return __virtualname__ + except KeyError: + pass + + return False + + +def panos(proxy=None): + if not proxy: + return {} + if proxy['panos.initialized']() is False: + return {} + return {'panos': proxy['panos.grains']()} diff --git a/salt/grains/rest_sample.py b/salt/grains/rest_sample.py index 98a2c0e53f6..490256fb56f 100644 --- a/salt/grains/rest_sample.py +++ b/salt/grains/rest_sample.py @@ -3,7 +3,7 @@ Generate baseline proxy minion grains ''' from __future__ import absolute_import -import salt.utils +import salt.utils.platform __proxyenabled__ = ['rest_sample'] @@ -12,7 +12,7 @@ __virtualname__ = 'rest_sample' def __virtual__(): try: - if salt.utils.is_proxy() and __opts__['proxy']['proxytype'] == 'rest_sample': + if salt.utils.platform.is_proxy() and __opts__['proxy']['proxytype'] == 'rest_sample': return __virtualname__ except KeyError: pass diff --git a/salt/grains/ssh_sample.py b/salt/grains/ssh_sample.py index 7c1cbe90a6c..be12165716c 100644 --- a/salt/grains/ssh_sample.py +++ b/salt/grains/ssh_sample.py @@ -3,7 +3,7 @@ Generate baseline proxy minion grains ''' from __future__ import absolute_import -import salt.utils +import salt.utils.platform __proxyenabled__ = ['ssh_sample'] @@ -12,7 +12,7 @@ __virtualname__ = 'ssh_sample' def __virtual__(): try: - if salt.utils.is_proxy() and __opts__['proxy']['proxytype'] == 'ssh_sample': + if salt.utils.platform.is_proxy() and __opts__['proxy']['proxytype'] == 'ssh_sample': return __virtualname__ except KeyError: pass diff --git a/salt/key.py b/salt/key.py index 88e2acf0e44..52b2d268e1b 100644 --- a/salt/key.py +++ b/salt/key.py @@ -23,11 +23,14 @@ import salt.daemons.masterapi import salt.exceptions import salt.minion import salt.utils +import salt.utils.args import salt.utils.event -import salt.utils.kinds +import salt.utils.files +import salt.utils.sdb +import salt.utils.kinds as kinds # pylint: disable=import-error,no-name-in-module,redefined-builtin -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import input # pylint: enable=import-error,no-name-in-module,redefined-builtin @@ -41,7 +44,7 @@ log = logging.getLogger(__name__) def get_key(opts): - if opts['transport'] in ('zeromq', 'tcp'): + if opts[u'transport'] in (u'zeromq', u'tcp'): return Key(opts) else: return RaetKey(opts) @@ -51,99 +54,99 @@ class KeyCLI(object): ''' Manage key CLI operations ''' - CLI_KEY_MAP = {'list': 'list_status', - 'delete': 'delete_key', - 'gen_signature': 'gen_keys_signature', - 'print': 'key_str', + CLI_KEY_MAP = {u'list': u'list_status', + u'delete': u'delete_key', + u'gen_signature': u'gen_keys_signature', + u'print': u'key_str', } def __init__(self, opts): self.opts = opts self.client = salt.wheel.WheelClient(opts) - if self.opts['transport'] in ('zeromq', 'tcp'): + if self.opts[u'transport'] in (u'zeromq', u'tcp'): self.key = Key else: self.key = RaetKey # instantiate the key object for masterless mode - if not opts.get('eauth'): + if not opts.get(u'eauth'): self.key = self.key(opts) self.auth = None def _update_opts(self): # get the key command - for cmd in ('gen_keys', - 'gen_signature', - 'list', - 'list_all', - 'print', - 'print_all', - 'accept', - 'accept_all', - 'reject', - 'reject_all', - 'delete', - 'delete_all', - 'finger', - 'finger_all', - 'list_all'): # last is default + for cmd in (u'gen_keys', + u'gen_signature', + u'list', + u'list_all', + u'print', + u'print_all', + u'accept', + u'accept_all', + u'reject', + u'reject_all', + u'delete', + u'delete_all', + u'finger', + u'finger_all', + u'list_all'): # last is default if self.opts[cmd]: break # set match if needed - if not cmd.startswith('gen_'): - if cmd == 'list_all': - self.opts['match'] = 'all' - elif cmd.endswith('_all'): - self.opts['match'] = '*' + if not cmd.startswith(u'gen_'): + if cmd == u'list_all': + self.opts[u'match'] = u'all' + elif cmd.endswith(u'_all'): + self.opts[u'match'] = u'*' else: - self.opts['match'] = self.opts[cmd] - if cmd.startswith('accept'): - self.opts['include_rejected'] = self.opts['include_all'] or self.opts['include_rejected'] - self.opts['include_accepted'] = False - elif cmd.startswith('reject'): - self.opts['include_accepted'] = self.opts['include_all'] or self.opts['include_accepted'] - self.opts['include_rejected'] = False - elif cmd == 'gen_keys': - self.opts['keydir'] = self.opts['gen_keys_dir'] - self.opts['keyname'] = self.opts['gen_keys'] + self.opts[u'match'] = self.opts[cmd] + if cmd.startswith(u'accept'): + self.opts[u'include_rejected'] = self.opts[u'include_all'] or self.opts[u'include_rejected'] + self.opts[u'include_accepted'] = False + elif cmd.startswith(u'reject'): + self.opts[u'include_accepted'] = self.opts[u'include_all'] or self.opts[u'include_accepted'] + self.opts[u'include_rejected'] = False + elif cmd == u'gen_keys': + self.opts[u'keydir'] = self.opts[u'gen_keys_dir'] + self.opts[u'keyname'] = self.opts[u'gen_keys'] # match is set to opts, now we can forget about *_all commands - self.opts['fun'] = cmd.replace('_all', '') + self.opts[u'fun'] = cmd.replace(u'_all', u'') def _init_auth(self): if self.auth: return low = {} - skip_perm_errors = self.opts['eauth'] != '' + skip_perm_errors = self.opts[u'eauth'] != u'' - if self.opts['eauth']: - if 'token' in self.opts: + if self.opts[u'eauth']: + if u'token' in self.opts: try: - with salt.utils.fopen(os.path.join(self.opts['cachedir'], '.root_key'), 'r') as fp_: - low['key'] = fp_.readline() + with salt.utils.files.fopen(os.path.join(self.opts[u'cachedir'], u'.root_key'), u'r') as fp_: + low[u'key'] = fp_.readline() except IOError: - low['token'] = self.opts['token'] + low[u'token'] = self.opts[u'token'] # # If using eauth and a token hasn't already been loaded into # low, prompt the user to enter auth credentials - if 'token' not in low and 'key' not in low and self.opts['eauth']: + if u'token' not in low and u'key' not in low and self.opts[u'eauth']: # This is expensive. Don't do it unless we need to. resolver = salt.auth.Resolver(self.opts) - res = resolver.cli(self.opts['eauth']) - if self.opts['mktoken'] and res: + res = resolver.cli(self.opts[u'eauth']) + if self.opts[u'mktoken'] and res: tok = resolver.token_cli( - self.opts['eauth'], + self.opts[u'eauth'], res ) if tok: - low['token'] = tok.get('token', '') + low[u'token'] = tok.get(u'token', u'') if not res: - log.error('Authentication failed') + log.error(u'Authentication failed') return {} low.update(res) - low['eauth'] = self.opts['eauth'] + low[u'eauth'] = self.opts[u'eauth'] else: - low['user'] = salt.utils.get_specific_user() - low['key'] = salt.utils.get_master_key(low['user'], self.opts, skip_perm_errors) + low[u'user'] = salt.utils.get_specific_user() + low[u'key'] = salt.utils.get_master_key(low[u'user'], self.opts, skip_perm_errors) self.auth = low @@ -162,24 +165,24 @@ class KeyCLI(object): return args, kwargs def _run_cmd(self, cmd, args=None): - if not self.opts.get('eauth'): + if not self.opts.get(u'eauth'): cmd = self.CLI_KEY_MAP.get(cmd, cmd) fun = getattr(self.key, cmd) args, kwargs = self._get_args_kwargs(fun, args) ret = fun(*args, **kwargs) - if (isinstance(ret, dict) and 'local' in ret and - cmd not in ('finger', 'finger_all')): - ret.pop('local', None) + if (isinstance(ret, dict) and u'local' in ret and + cmd not in (u'finger', u'finger_all')): + ret.pop(u'local', None) return ret - fstr = 'key.{0}'.format(cmd) + fstr = u'key.{0}'.format(cmd) fun = self.client.functions[fstr] args, kwargs = self._get_args_kwargs(fun, args) low = { - 'fun': fstr, - 'arg': args, - 'kwarg': kwargs, + u'fun': fstr, + u'arg': args, + u'kwarg': kwargs, } self._init_auth() @@ -188,41 +191,41 @@ class KeyCLI(object): # Execute the key request! ret = self.client.cmd_sync(low) - ret = ret['data']['return'] - if (isinstance(ret, dict) and 'local' in ret and - cmd not in ('finger', 'finger_all')): - ret.pop('local', None) + ret = ret[u'data'][u'return'] + if (isinstance(ret, dict) and u'local' in ret and + cmd not in (u'finger', u'finger_all')): + ret.pop(u'local', None) return ret def _filter_ret(self, cmd, ret): - if cmd.startswith('delete'): + if cmd.startswith(u'delete'): return ret keys = {} if self.key.PEND in ret: keys[self.key.PEND] = ret[self.key.PEND] - if self.opts['include_accepted'] and bool(ret.get(self.key.ACC)): + if self.opts[u'include_accepted'] and bool(ret.get(self.key.ACC)): keys[self.key.ACC] = ret[self.key.ACC] - if self.opts['include_rejected'] and bool(ret.get(self.key.REJ)): + if self.opts[u'include_rejected'] and bool(ret.get(self.key.REJ)): keys[self.key.REJ] = ret[self.key.REJ] - if self.opts['include_denied'] and bool(ret.get(self.key.DEN)): + if self.opts[u'include_denied'] and bool(ret.get(self.key.DEN)): keys[self.key.DEN] = ret[self.key.DEN] return keys def _print_no_match(self, cmd, match): - statuses = ['unaccepted'] - if self.opts['include_accepted']: - statuses.append('accepted') - if self.opts['include_rejected']: - statuses.append('rejected') - if self.opts['include_denied']: - statuses.append('denied') + statuses = [u'unaccepted'] + if self.opts[u'include_accepted']: + statuses.append(u'accepted') + if self.opts[u'include_rejected']: + statuses.append(u'rejected') + if self.opts[u'include_denied']: + statuses.append(u'denied') if len(statuses) == 1: stat_str = statuses[0] else: - stat_str = '{0} or {1}'.format(', '.join(statuses[:-1]), statuses[-1]) - msg = 'The key glob \'{0}\' does not match any {1} keys.'.format(match, stat_str) + stat_str = u'{0} or {1}'.format(u', '.join(statuses[:-1]), statuses[-1]) + msg = u'The key glob \'{0}\' does not match any {1} keys.'.format(match, stat_str) print(msg) def run(self): @@ -230,57 +233,57 @@ class KeyCLI(object): Run the logic for saltkey ''' self._update_opts() - cmd = self.opts['fun'] + cmd = self.opts[u'fun'] veri = None ret = None try: - if cmd in ('accept', 'reject', 'delete'): - ret = self._run_cmd('name_match') + if cmd in (u'accept', u'reject', u'delete'): + ret = self._run_cmd(u'name_match') if not isinstance(ret, dict): - salt.output.display_output(ret, 'key', opts=self.opts) + salt.output.display_output(ret, u'key', opts=self.opts) return ret ret = self._filter_ret(cmd, ret) if not ret: - self._print_no_match(cmd, self.opts['match']) + self._print_no_match(cmd, self.opts[u'match']) return - print('The following keys are going to be {0}ed:'.format(cmd.rstrip('e'))) - salt.output.display_output(ret, 'key', opts=self.opts) + print(u'The following keys are going to be {0}ed:'.format(cmd.rstrip(u'e'))) + salt.output.display_output(ret, u'key', opts=self.opts) - if not self.opts.get('yes', False): + if not self.opts.get(u'yes', False): try: - if cmd.startswith('delete'): - veri = input('Proceed? [N/y] ') + if cmd.startswith(u'delete'): + veri = input(u'Proceed? [N/y] ') if not veri: - veri = 'n' + veri = u'n' else: - veri = input('Proceed? [n/Y] ') + veri = input(u'Proceed? [n/Y] ') if not veri: - veri = 'y' + veri = u'y' except KeyboardInterrupt: - raise SystemExit("\nExiting on CTRL-c") + raise SystemExit(u"\nExiting on CTRL-c") # accept/reject/delete the same keys we're printed to the user - self.opts['match_dict'] = ret - self.opts.pop('match', None) + self.opts[u'match_dict'] = ret + self.opts.pop(u'match', None) list_ret = ret - if veri is None or veri.lower().startswith('y'): + if veri is None or veri.lower().startswith(u'y'): ret = self._run_cmd(cmd) - if cmd in ('accept', 'reject', 'delete'): - if cmd == 'delete': + if cmd in (u'accept', u'reject', u'delete'): + if cmd == u'delete': ret = list_ret for minions in ret.values(): for minion in minions: - print('Key for minion {0} {1}ed.'.format(minion, - cmd.rstrip('e'))) + print(u'Key for minion {0} {1}ed.'.format(minion, + cmd.rstrip(u'e'))) elif isinstance(ret, dict): - salt.output.display_output(ret, 'key', opts=self.opts) + salt.output.display_output(ret, u'key', opts=self.opts) else: - salt.output.display_output({'return': ret}, 'key', opts=self.opts) + salt.output.display_output({u'return': ret}, u'key', opts=self.opts) except salt.exceptions.SaltException as exc: - ret = '{0}'.format(exc) - if not self.opts.get('quiet', False): - salt.output.display_output(ret, 'nested', self.opts) + ret = u'{0}'.format(exc) + if not self.opts.get(u'quiet', False): + salt.output.display_output(ret, u'nested', self.opts) return ret @@ -289,17 +292,17 @@ class MultiKeyCLI(KeyCLI): Manage multiple key backends from the CLI ''' def __init__(self, opts): - opts['__multi_key'] = True + opts[u'__multi_key'] = True super(MultiKeyCLI, self).__init__(opts) # Remove the key attribute set in KeyCLI.__init__ - delattr(self, 'key') + delattr(self, u'key') zopts = copy.copy(opts) ropts = copy.copy(opts) self.keys = {} - zopts['transport'] = 'zeromq' - self.keys['ZMQ Keys'] = KeyCLI(zopts) - ropts['transport'] = 'raet' - self.keys['RAET Keys'] = KeyCLI(ropts) + zopts[u'transport'] = u'zeromq' + self.keys[u'ZMQ Keys'] = KeyCLI(zopts) + ropts[u'transport'] = u'raet' + self.keys[u'RAET Keys'] = KeyCLI(ropts) def _call_all(self, fun, *args): ''' @@ -310,97 +313,99 @@ class MultiKeyCLI(KeyCLI): getattr(self.keys[kback], fun)(*args) def list_status(self, status): - self._call_all('list_status', status) + self._call_all(u'list_status', status) def list_all(self): - self._call_all('list_all') + self._call_all(u'list_all') def accept(self, match, include_rejected=False, include_denied=False): - self._call_all('accept', match, include_rejected, include_denied) + self._call_all(u'accept', match, include_rejected, include_denied) def accept_all(self, include_rejected=False, include_denied=False): - self._call_all('accept_all', include_rejected, include_denied) + self._call_all(u'accept_all', include_rejected, include_denied) def delete(self, match): - self._call_all('delete', match) + self._call_all(u'delete', match) def delete_all(self): - self._call_all('delete_all') + self._call_all(u'delete_all') def reject(self, match, include_accepted=False, include_denied=False): - self._call_all('reject', match, include_accepted, include_denied) + self._call_all(u'reject', match, include_accepted, include_denied) def reject_all(self, include_accepted=False, include_denied=False): - self._call_all('reject_all', include_accepted, include_denied) + self._call_all(u'reject_all', include_accepted, include_denied) def print_key(self, match): - self._call_all('print_key', match) + self._call_all(u'print_key', match) def print_all(self): - self._call_all('print_all') + self._call_all(u'print_all') def finger(self, match, hash_type): - self._call_all('finger', match, hash_type) + self._call_all(u'finger', match, hash_type) def finger_all(self, hash_type): - self._call_all('finger_all', hash_type) + self._call_all(u'finger_all', hash_type) def prep_signature(self): - self._call_all('prep_signature') + self._call_all(u'prep_signature') class Key(object): ''' The object that encapsulates saltkey actions ''' - ACC = 'minions' - PEND = 'minions_pre' - REJ = 'minions_rejected' - DEN = 'minions_denied' + ACC = u'minions' + PEND = u'minions_pre' + REJ = u'minions_rejected' + DEN = u'minions_denied' def __init__(self, opts, io_loop=None): self.opts = opts - kind = self.opts.get('__role', '') # application kind - if kind not in salt.utils.kinds.APPL_KINDS: - emsg = ("Invalid application kind = '{0}'.".format(kind)) - log.error(emsg + '\n') + kind = self.opts.get(u'__role', u'') # application kind + if kind not in kinds.APPL_KINDS: + emsg = (u"Invalid application kind = '{0}'.".format(kind)) + log.error(emsg + u'\n') raise ValueError(emsg) self.event = salt.utils.event.get_event( kind, - opts['sock_dir'], - opts['transport'], + opts[u'sock_dir'], + opts[u'transport'], opts=opts, listen=False, io_loop=io_loop ) + self.passphrase = salt.utils.sdb.sdb_get(self.opts['signing_key_pass'], self.opts) + def _check_minions_directories(self): ''' Return the minion keys directory paths ''' - minions_accepted = os.path.join(self.opts['pki_dir'], self.ACC) - minions_pre = os.path.join(self.opts['pki_dir'], self.PEND) - minions_rejected = os.path.join(self.opts['pki_dir'], + minions_accepted = os.path.join(self.opts[u'pki_dir'], self.ACC) + minions_pre = os.path.join(self.opts[u'pki_dir'], self.PEND) + minions_rejected = os.path.join(self.opts[u'pki_dir'], self.REJ) - minions_denied = os.path.join(self.opts['pki_dir'], + minions_denied = os.path.join(self.opts[u'pki_dir'], self.DEN) return minions_accepted, minions_pre, minions_rejected, minions_denied def _get_key_attrs(self, keydir, keyname, keysize, user): if not keydir: - if 'gen_keys_dir' in self.opts: - keydir = self.opts['gen_keys_dir'] + if u'gen_keys_dir' in self.opts: + keydir = self.opts[u'gen_keys_dir'] else: - keydir = self.opts['pki_dir'] + keydir = self.opts[u'pki_dir'] if not keyname: - if 'gen_keys' in self.opts: - keyname = self.opts['gen_keys'] + if u'gen_keys' in self.opts: + keyname = self.opts[u'gen_keys'] else: - keyname = 'minion' + keyname = u'minion' if not keysize: - keysize = self.opts['keysize'] + keysize = self.opts[u'keysize'] return keydir, keyname, keysize, user def gen_keys(self, keydir=None, keyname=None, keysize=None, user=None): @@ -409,8 +414,8 @@ class Key(object): ''' keydir, keyname, keysize, user = self._get_key_attrs(keydir, keyname, keysize, user) - salt.crypt.gen_keys(keydir, keyname, keysize, user) - return salt.utils.pem_finger(os.path.join(keydir, keyname + '.pub')) + salt.crypt.gen_keys(keydir, keyname, keysize, user, self.passphrase) + return salt.utils.pem_finger(os.path.join(keydir, keyname + u'.pub')) def gen_signature(self, privkey, pubkey, sig_path): ''' @@ -418,7 +423,8 @@ class Key(object): ''' return salt.crypt.gen_signature(privkey, pubkey, - sig_path) + sig_path, + self.passphrase) def gen_keys_signature(self, priv, pub, signature_path, auto_create=False, keysize=None): ''' @@ -427,51 +433,52 @@ class Key(object): # check given pub-key if pub: if not os.path.isfile(pub): - return 'Public-key {0} does not exist'.format(pub) + return u'Public-key {0} does not exist'.format(pub) # default to master.pub else: - mpub = self.opts['pki_dir'] + '/' + 'master.pub' + mpub = self.opts[u'pki_dir'] + u'/' + u'master.pub' if os.path.isfile(mpub): pub = mpub # check given priv-key if priv: if not os.path.isfile(priv): - return 'Private-key {0} does not exist'.format(priv) + return u'Private-key {0} does not exist'.format(priv) # default to master_sign.pem else: - mpriv = self.opts['pki_dir'] + '/' + 'master_sign.pem' + mpriv = self.opts[u'pki_dir'] + u'/' + u'master_sign.pem' if os.path.isfile(mpriv): priv = mpriv if not priv: if auto_create: - log.debug('Generating new signing key-pair {0}.* in {1}' - ''.format(self.opts['master_sign_key_name'], - self.opts['pki_dir'])) - salt.crypt.gen_keys(self.opts['pki_dir'], - self.opts['master_sign_key_name'], - keysize or self.opts['keysize'], - self.opts.get('user')) + log.debug( + u'Generating new signing key-pair .%s.* in %s', + self.opts[u'master_sign_key_name'], self.opts[u'pki_dir'] + ) + salt.crypt.gen_keys(self.opts[u'pki_dir'], + self.opts[u'master_sign_key_name'], + keysize or self.opts[u'keysize'], + self.opts.get(u'user'), + self.passphrase) - priv = self.opts['pki_dir'] + '/' + self.opts['master_sign_key_name'] + '.pem' + priv = self.opts[u'pki_dir'] + u'/' + self.opts[u'master_sign_key_name'] + u'.pem' else: - return 'No usable private-key found' + return u'No usable private-key found' if not pub: - return 'No usable public-key found' + return u'No usable public-key found' - log.debug('Using public-key {0}'.format(pub)) - log.debug('Using private-key {0}'.format(priv)) + log.debug(u'Using public-key %s', pub) + log.debug(u'Using private-key %s', priv) if signature_path: if not os.path.isdir(signature_path): - log.debug('target directory {0} does not exist' - ''.format(signature_path)) + log.debug(u'target directory %s does not exist', signature_path) else: - signature_path = self.opts['pki_dir'] + signature_path = self.opts[u'pki_dir'] - sign_path = signature_path + '/' + self.opts['master_pubkey_signature'] + sign_path = signature_path + u'/' + self.opts[u'master_pubkey_signature'] skey = get_key(self.opts) return skey.gen_signature(priv, pub, sign_path) @@ -489,18 +496,18 @@ class Key(object): minions = [] for key, val in six.iteritems(keys): minions.extend(val) - if not self.opts.get('preserve_minion_cache', False) or not preserve_minions: - m_cache = os.path.join(self.opts['cachedir'], self.ACC) + if not self.opts.get(u'preserve_minion_cache', False): + m_cache = os.path.join(self.opts[u'cachedir'], self.ACC) if os.path.isdir(m_cache): for minion in os.listdir(m_cache): if minion not in minions and minion not in preserve_minions: shutil.rmtree(os.path.join(m_cache, minion)) cache = salt.cache.factory(self.opts) - clist = cache.ls(self.ACC) + clist = cache.list(self.ACC) if clist: for minion in clist: if minion not in minions and minion not in preserve_minions: - cache.flush('{0}/{1}'.format(self.ACC, minion)) + cache.flush(u'{0}/{1}'.format(self.ACC, minion)) def check_master(self): ''' @@ -511,8 +518,8 @@ class Key(object): ''' if not os.path.exists( os.path.join( - self.opts['sock_dir'], - 'publish_pull.ipc' + self.opts[u'sock_dir'], + u'publish_pull.ipc' ) ): return False @@ -527,8 +534,8 @@ class Key(object): else: matches = self.list_keys() ret = {} - if ',' in match and isinstance(match, str): - match = match.split(',') + if u',' in match and isinstance(match, six.string_types): + match = match.split(u',') for status, keys in six.iteritems(matches): for key in salt.utils.isorted(keys): if isinstance(match, list): @@ -562,12 +569,12 @@ class Key(object): ''' Return a dict of local keys ''' - ret = {'local': []} - for fn_ in salt.utils.isorted(os.listdir(self.opts['pki_dir'])): - if fn_.endswith('.pub') or fn_.endswith('.pem'): - path = os.path.join(self.opts['pki_dir'], fn_) + ret = {u'local': []} + for fn_ in salt.utils.isorted(os.listdir(self.opts[u'pki_dir'])): + if fn_.endswith(u'.pub') or fn_.endswith(u'.pem'): + path = os.path.join(self.opts[u'pki_dir'], fn_) if os.path.isfile(path): - ret['local'].append(fn_) + ret[u'local'].append(fn_) return ret def list_keys(self): @@ -590,7 +597,7 @@ class Key(object): ret[os.path.basename(dir_)] = [] try: for fn_ in salt.utils.isorted(os.listdir(dir_)): - if not fn_.startswith('.'): + if not fn_.startswith(u'.'): if os.path.isfile(os.path.join(dir_, fn_)): ret[os.path.basename(dir_)].append(fn_) except (OSError, IOError): @@ -612,31 +619,31 @@ class Key(object): ''' acc, pre, rej, den = self._check_minions_directories() ret = {} - if match.startswith('acc'): + if match.startswith(u'acc'): ret[os.path.basename(acc)] = [] for fn_ in salt.utils.isorted(os.listdir(acc)): - if not fn_.startswith('.'): + if not fn_.startswith(u'.'): if os.path.isfile(os.path.join(acc, fn_)): ret[os.path.basename(acc)].append(fn_) - elif match.startswith('pre') or match.startswith('un'): + elif match.startswith(u'pre') or match.startswith(u'un'): ret[os.path.basename(pre)] = [] for fn_ in salt.utils.isorted(os.listdir(pre)): - if not fn_.startswith('.'): + if not fn_.startswith(u'.'): if os.path.isfile(os.path.join(pre, fn_)): ret[os.path.basename(pre)].append(fn_) - elif match.startswith('rej'): + elif match.startswith(u'rej'): ret[os.path.basename(rej)] = [] for fn_ in salt.utils.isorted(os.listdir(rej)): - if not fn_.startswith('.'): + if not fn_.startswith(u'.'): if os.path.isfile(os.path.join(rej, fn_)): ret[os.path.basename(rej)].append(fn_) - elif match.startswith('den') and den is not None: + elif match.startswith(u'den') and den is not None: ret[os.path.basename(den)] = [] for fn_ in salt.utils.isorted(os.listdir(den)): - if not fn_.startswith('.'): + if not fn_.startswith(u'.'): if os.path.isfile(os.path.join(den, fn_)): ret[os.path.basename(den)].append(fn_) - elif match.startswith('all'): + elif match.startswith(u'all'): return self.all_keys() return ret @@ -648,8 +655,8 @@ class Key(object): for status, keys in six.iteritems(self.name_match(match)): ret[status] = {} for key in salt.utils.isorted(keys): - path = os.path.join(self.opts['pki_dir'], status, key) - with salt.utils.fopen(path, 'r') as fp_: + path = os.path.join(self.opts[u'pki_dir'], status, key) + with salt.utils.files.fopen(path, u'r') as fp_: ret[status][key] = fp_.read() return ret @@ -661,8 +668,8 @@ class Key(object): for status, keys in six.iteritems(self.list_keys()): ret[status] = {} for key in salt.utils.isorted(keys): - path = os.path.join(self.opts['pki_dir'], status, key) - with salt.utils.fopen(path, 'r') as fp_: + path = os.path.join(self.opts[u'pki_dir'], status, key) + with salt.utils.files.fopen(path, u'r') as fp_: ret[status][key] = fp_.read() return ret @@ -687,19 +694,19 @@ class Key(object): try: shutil.move( os.path.join( - self.opts['pki_dir'], + self.opts[u'pki_dir'], keydir, key), os.path.join( - self.opts['pki_dir'], + self.opts[u'pki_dir'], self.ACC, key) ) - eload = {'result': True, - 'act': 'accept', - 'id': key} + eload = {u'result': True, + u'act': u'accept', + u'id': key} self.event.fire_event(eload, - salt.utils.event.tagify(prefix='key')) + salt.utils.event.tagify(prefix=u'key')) except (IOError, OSError): pass return ( @@ -716,19 +723,19 @@ class Key(object): try: shutil.move( os.path.join( - self.opts['pki_dir'], + self.opts[u'pki_dir'], self.PEND, key), os.path.join( - self.opts['pki_dir'], + self.opts[u'pki_dir'], self.ACC, key) ) - eload = {'result': True, - 'act': 'accept', - 'id': key} + eload = {u'result': True, + u'act': u'accept', + u'id': key} self.event.fire_event(eload, - salt.utils.event.tagify(prefix='key')) + salt.utils.event.tagify(prefix=u'key')) except (IOError, OSError): pass return self.list_keys() @@ -736,7 +743,7 @@ class Key(object): def delete_key(self, match=None, match_dict=None, - preserve_minions=False, + preserve_minions=None, revoke_auth=False): ''' Delete public keys. If "match" is passed, it is evaluated as a glob. @@ -754,33 +761,32 @@ class Key(object): for key in keys: try: if revoke_auth: - if self.opts.get('rotate_aes_key') is False: - print('Immediate auth revocation specified but AES key rotation not allowed. ' - 'Minion will not be disconnected until the master AES key is rotated.') + if self.opts.get(u'rotate_aes_key') is False: + print(u'Immediate auth revocation specified but AES key rotation not allowed. ' + u'Minion will not be disconnected until the master AES key is rotated.') else: try: client = salt.client.get_local_client(mopts=self.opts) - client.cmd_async(key, 'saltutil.revoke_auth') + client.cmd_async(key, u'saltutil.revoke_auth') except salt.exceptions.SaltClientError: - print('Cannot contact Salt master. ' - 'Connection for {0} will remain up until ' - 'master AES key is rotated or auth is revoked ' - 'with \'saltutil.revoke_auth\'.'.format(key)) - os.remove(os.path.join(self.opts['pki_dir'], status, key)) - eload = {'result': True, - 'act': 'delete', - 'id': key} + print(u'Cannot contact Salt master. ' + u'Connection for {0} will remain up until ' + u'master AES key is rotated or auth is revoked ' + u'with \'saltutil.revoke_auth\'.'.format(key)) + os.remove(os.path.join(self.opts[u'pki_dir'], status, key)) + eload = {u'result': True, + u'act': u'delete', + u'id': key} self.event.fire_event(eload, - salt.utils.event.tagify(prefix='key')) + salt.utils.event.tagify(prefix=u'key')) except (OSError, IOError): pass - if preserve_minions: - preserve_minions_list = matches.get('minions', []) + if self.opts.get(u'preserve_minions') is True: + self.check_minion_cache(preserve_minions=matches.get(u'minions', [])) else: - preserve_minions_list = [] - self.check_minion_cache(preserve_minions=preserve_minions_list) - if self.opts.get('rotate_aes_key'): - salt.crypt.dropfile(self.opts['cachedir'], self.opts['user']) + self.check_minion_cache() + if self.opts.get(u'rotate_aes_key'): + salt.crypt.dropfile(self.opts[u'cachedir'], self.opts[u'user']) return ( self.name_match(match) if match is not None else self.dict_match(matches) @@ -794,12 +800,12 @@ class Key(object): for status, keys in six.iteritems(self.list_keys()): for key in keys[self.DEN]: try: - os.remove(os.path.join(self.opts['pki_dir'], status, key)) - eload = {'result': True, - 'act': 'delete', - 'id': key} + os.remove(os.path.join(self.opts[u'pki_dir'], status, key)) + eload = {u'result': True, + u'act': u'delete', + u'id': key} self.event.fire_event(eload, - salt.utils.event.tagify(prefix='key')) + salt.utils.event.tagify(prefix=u'key')) except (OSError, IOError): pass self.check_minion_cache() @@ -812,17 +818,17 @@ class Key(object): for status, keys in six.iteritems(self.list_keys()): for key in keys: try: - os.remove(os.path.join(self.opts['pki_dir'], status, key)) - eload = {'result': True, - 'act': 'delete', - 'id': key} + os.remove(os.path.join(self.opts[u'pki_dir'], status, key)) + eload = {u'result': True, + u'act': u'delete', + u'id': key} self.event.fire_event(eload, - salt.utils.event.tagify(prefix='key')) + salt.utils.event.tagify(prefix=u'key')) except (OSError, IOError): pass self.check_minion_cache() - if self.opts.get('rotate_aes_key'): - salt.crypt.dropfile(self.opts['cachedir'], self.opts['user']) + if self.opts.get(u'rotate_aes_key'): + salt.crypt.dropfile(self.opts[u'cachedir'], self.opts[u'user']) return self.list_keys() def reject(self, match=None, match_dict=None, include_accepted=False, include_denied=False): @@ -846,24 +852,24 @@ class Key(object): try: shutil.move( os.path.join( - self.opts['pki_dir'], + self.opts[u'pki_dir'], keydir, key), os.path.join( - self.opts['pki_dir'], + self.opts[u'pki_dir'], self.REJ, key) ) - eload = {'result': True, - 'act': 'reject', - 'id': key} + eload = {u'result': True, + u'act': u'reject', + u'id': key} self.event.fire_event(eload, - salt.utils.event.tagify(prefix='key')) + salt.utils.event.tagify(prefix=u'key')) except (IOError, OSError): pass self.check_minion_cache() - if self.opts.get('rotate_aes_key'): - salt.crypt.dropfile(self.opts['cachedir'], self.opts['user']) + if self.opts.get(u'rotate_aes_key'): + salt.crypt.dropfile(self.opts[u'cachedir'], self.opts[u'user']) return ( self.name_match(match) if match is not None else self.dict_match(matches) @@ -878,24 +884,24 @@ class Key(object): try: shutil.move( os.path.join( - self.opts['pki_dir'], + self.opts[u'pki_dir'], self.PEND, key), os.path.join( - self.opts['pki_dir'], + self.opts[u'pki_dir'], self.REJ, key) ) - eload = {'result': True, - 'act': 'reject', - 'id': key} + eload = {u'result': True, + u'act': u'reject', + u'id': key} self.event.fire_event(eload, - salt.utils.event.tagify(prefix='key')) + salt.utils.event.tagify(prefix=u'key')) except (IOError, OSError): pass self.check_minion_cache() - if self.opts.get('rotate_aes_key'): - salt.crypt.dropfile(self.opts['cachedir'], self.opts['user']) + if self.opts.get(u'rotate_aes_key'): + salt.crypt.dropfile(self.opts[u'cachedir'], self.opts[u'user']) return self.list_keys() def finger(self, match, hash_type=None): @@ -903,17 +909,17 @@ class Key(object): Return the fingerprint for a specified key ''' if hash_type is None: - hash_type = __opts__['hash_type'] + hash_type = __opts__[u'hash_type'] matches = self.name_match(match, True) ret = {} for status, keys in six.iteritems(matches): ret[status] = {} for key in keys: - if status == 'local': - path = os.path.join(self.opts['pki_dir'], key) + if status == u'local': + path = os.path.join(self.opts[u'pki_dir'], key) else: - path = os.path.join(self.opts['pki_dir'], status, key) + path = os.path.join(self.opts[u'pki_dir'], status, key) ret[status][key] = salt.utils.pem_finger(path, sum_type=hash_type) return ret @@ -922,16 +928,16 @@ class Key(object): Return fingerprints for all keys ''' if hash_type is None: - hash_type = __opts__['hash_type'] + hash_type = __opts__[u'hash_type'] ret = {} for status, keys in six.iteritems(self.all_keys()): ret[status] = {} for key in keys: - if status == 'local': - path = os.path.join(self.opts['pki_dir'], key) + if status == u'local': + path = os.path.join(self.opts[u'pki_dir'], key) else: - path = os.path.join(self.opts['pki_dir'], status, key) + path = os.path.join(self.opts[u'pki_dir'], status, key) ret[status][key] = salt.utils.pem_finger(path, sum_type=hash_type) return ret @@ -940,9 +946,9 @@ class RaetKey(Key): ''' Manage keys from the raet backend ''' - ACC = 'accepted' - PEND = 'pending' - REJ = 'rejected' + ACC = u'accepted' + PEND = u'pending' + REJ = u'rejected' DEN = None def __init__(self, opts): @@ -954,9 +960,9 @@ class RaetKey(Key): ''' Return the minion keys directory paths ''' - accepted = os.path.join(self.opts['pki_dir'], self.ACC) - pre = os.path.join(self.opts['pki_dir'], self.PEND) - rejected = os.path.join(self.opts['pki_dir'], self.REJ) + accepted = os.path.join(self.opts[u'pki_dir'], self.ACC) + pre = os.path.join(self.opts[u'pki_dir'], self.PEND) + rejected = os.path.join(self.opts[u'pki_dir'], self.REJ) return accepted, pre, rejected, None def check_minion_cache(self, preserve_minions=False): @@ -968,49 +974,50 @@ class RaetKey(Key): for key, val in six.iteritems(keys): minions.extend(val) - m_cache = os.path.join(self.opts['cachedir'], 'minions') - if os.path.isdir(m_cache): - for minion in os.listdir(m_cache): - if minion not in minions: - shutil.rmtree(os.path.join(m_cache, minion)) - cache = salt.cache.factory(self.opts) - clist = cache.ls(self.ACC) - if clist: - for minion in clist: + m_cache = os.path.join(self.opts[u'cachedir'], u'minions') + if not self.opts.get('preserve_minion_cache', False): + if os.path.isdir(m_cache): + for minion in os.listdir(m_cache): if minion not in minions and minion not in preserve_minions: - cache.flush('{0}/{1}'.format(self.ACC, minion)) + shutil.rmtree(os.path.join(m_cache, minion)) + cache = salt.cache.factory(self.opts) + clist = cache.list(self.ACC) + if clist: + for minion in clist: + if minion not in minions and minion not in preserve_minions: + cache.flush(u'{0}/{1}'.format(self.ACC, minion)) - kind = self.opts.get('__role', '') # application kind - if kind not in salt.utils.kinds.APPL_KINDS: - emsg = ("Invalid application kind = '{0}'.".format(kind)) - log.error(emsg + '\n') + kind = self.opts.get(u'__role', u'') # application kind + if kind not in kinds.APPL_KINDS: + emsg = (u"Invalid application kind = '{0}'.".format(kind)) + log.error(emsg + u'\n') raise ValueError(emsg) - role = self.opts.get('id', '') + role = self.opts.get(u'id', u'') if not role: - emsg = ("Invalid id.") - log.error(emsg + "\n") + emsg = (u"Invalid id.") + log.error(emsg + u"\n") raise ValueError(emsg) - name = "{0}_{1}".format(role, kind) - road_cache = os.path.join(self.opts['cachedir'], - 'raet', + name = u"{0}_{1}".format(role, kind) + road_cache = os.path.join(self.opts[u'cachedir'], + u'raet', name, - 'remote') + u'remote') if os.path.isdir(road_cache): for road in os.listdir(road_cache): root, ext = os.path.splitext(road) - if ext not in ['.json', '.msgpack']: + if ext not in (u'.json', u'.msgpack'): continue - prefix, sep, name = root.partition('.') - if not name or prefix != 'estate': + prefix, sep, name = root.partition(u'.') + if not name or prefix != u'estate': continue path = os.path.join(road_cache, road) - with salt.utils.fopen(path, 'rb') as fp_: - if ext == '.json': + with salt.utils.files.fopen(path, u'rb') as fp_: + if ext == u'.json': data = json.load(fp_) - elif ext == '.msgpack': + elif ext == u'.msgpack': data = msgpack.load(fp_) - if data['role'] not in minions: + if data[u'role'] not in minions: os.remove(path) def gen_keys(self, keydir=None, keyname=None, keysize=None, user=None): @@ -1021,10 +1028,10 @@ class RaetKey(Key): d_key = libnacl.dual.DualSecret() keydir, keyname, _, _ = self._get_key_attrs(keydir, keyname, keysize, user) - path = '{0}.key'.format(os.path.join( + path = u'{0}.key'.format(os.path.join( keydir, keyname)) - d_key.save(path, 'msgpack') + d_key.save(path, u'msgpack') def check_master(self): ''' @@ -1037,10 +1044,10 @@ class RaetKey(Key): ''' Return a dict of local keys ''' - ret = {'local': []} - fn_ = os.path.join(self.opts['pki_dir'], 'local.key') + ret = {u'local': []} + fn_ = os.path.join(self.opts[u'pki_dir'], u'local.key') if os.path.isfile(fn_): - ret['local'].append(fn_) + ret[u'local'].append(fn_) return ret def status(self, minion_id, pub, verify): @@ -1056,47 +1063,47 @@ class RaetKey(Key): rej_path = os.path.join(rej, minion_id) # open mode is turned on, force accept the key keydata = { - 'minion_id': minion_id, - 'pub': pub, - 'verify': verify} - if self.opts['open_mode']: # always accept and overwrite - with salt.utils.fopen(acc_path, 'w+b') as fp_: + u'minion_id': minion_id, + u'pub': pub, + u'verify': verify} + if self.opts[u'open_mode']: # always accept and overwrite + with salt.utils.files.fopen(acc_path, u'w+b') as fp_: fp_.write(self.serial.dumps(keydata)) return self.ACC if os.path.isfile(rej_path): - log.debug("Rejection Reason: Keys already rejected.\n") + log.debug(u"Rejection Reason: Keys already rejected.\n") return self.REJ elif os.path.isfile(acc_path): # The minion id has been accepted, verify the key strings - with salt.utils.fopen(acc_path, 'rb') as fp_: + with salt.utils.files.fopen(acc_path, u'rb') as fp_: keydata = self.serial.loads(fp_.read()) - if keydata['pub'] == pub and keydata['verify'] == verify: + if keydata[u'pub'] == pub and keydata[u'verify'] == verify: return self.ACC else: - log.debug("Rejection Reason: Keys not match prior accepted.\n") + log.debug(u"Rejection Reason: Keys not match prior accepted.\n") return self.REJ elif os.path.isfile(pre_path): auto_reject = self.auto_key.check_autoreject(minion_id) auto_sign = self.auto_key.check_autosign(minion_id) - with salt.utils.fopen(pre_path, 'rb') as fp_: + with salt.utils.files.fopen(pre_path, u'rb') as fp_: keydata = self.serial.loads(fp_.read()) - if keydata['pub'] == pub and keydata['verify'] == verify: + if keydata[u'pub'] == pub and keydata[u'verify'] == verify: if auto_reject: self.reject(minion_id) - log.debug("Rejection Reason: Auto reject pended.\n") + log.debug(u"Rejection Reason: Auto reject pended.\n") return self.REJ elif auto_sign: self.accept(minion_id) return self.ACC return self.PEND else: - log.debug("Rejection Reason: Keys not match prior pended.\n") + log.debug(u"Rejection Reason: Keys not match prior pended.\n") return self.REJ # This is a new key, evaluate auto accept/reject files and place # accordingly auto_reject = self.auto_key.check_autoreject(minion_id) auto_sign = self.auto_key.check_autosign(minion_id) - if self.opts['auto_accept']: + if self.opts[u'auto_accept']: w_path = acc_path ret = self.ACC elif auto_sign: @@ -1104,12 +1111,12 @@ class RaetKey(Key): ret = self.ACC elif auto_reject: w_path = rej_path - log.debug("Rejection Reason: Auto reject new.\n") + log.debug(u"Rejection Reason: Auto reject new.\n") ret = self.REJ else: w_path = pre_path ret = self.PEND - with salt.utils.fopen(w_path, 'w+b') as fp_: + with salt.utils.files.fopen(w_path, u'w+b') as fp_: fp_.write(self.serial.dumps(keydata)) return ret @@ -1120,22 +1127,22 @@ class RaetKey(Key): pub: verify: ''' - path = os.path.join(self.opts['pki_dir'], status, minion_id) - with salt.utils.fopen(path, 'r') as fp_: + path = os.path.join(self.opts[u'pki_dir'], status, minion_id) + with salt.utils.files.fopen(path, u'r') as fp_: keydata = self.serial.loads(fp_.read()) - return 'pub: {0}\nverify: {1}'.format( - keydata['pub'], - keydata['verify']) + return u'pub: {0}\nverify: {1}'.format( + keydata[u'pub'], + keydata[u'verify']) def _get_key_finger(self, path): ''' Return a sha256 kingerprint for the key ''' - with salt.utils.fopen(path, 'r') as fp_: + with salt.utils.files.fopen(path, u'r') as fp_: keydata = self.serial.loads(fp_.read()) - key = 'pub: {0}\nverify: {1}'.format( - keydata['pub'], - keydata['verify']) + key = u'pub: {0}\nverify: {1}'.format( + keydata[u'pub'], + keydata[u'verify']) return hashlib.sha256(key).hexdigest() def key_str(self, match): @@ -1181,11 +1188,11 @@ class RaetKey(Key): try: shutil.move( os.path.join( - self.opts['pki_dir'], + self.opts[u'pki_dir'], keydir, key), os.path.join( - self.opts['pki_dir'], + self.opts[u'pki_dir'], self.ACC, key) ) @@ -1205,11 +1212,11 @@ class RaetKey(Key): try: shutil.move( os.path.join( - self.opts['pki_dir'], + self.opts[u'pki_dir'], self.PEND, key), os.path.join( - self.opts['pki_dir'], + self.opts[u'pki_dir'], self.ACC, key) ) @@ -1220,7 +1227,7 @@ class RaetKey(Key): def delete_key(self, match=None, match_dict=None, - preserve_minions=False, + preserve_minions=None, revoke_auth=False): ''' Delete public keys. If "match" is passed, it is evaluated as a glob. @@ -1235,23 +1242,26 @@ class RaetKey(Key): for status, keys in six.iteritems(matches): for key in keys: if revoke_auth: - if self.opts.get('rotate_aes_key') is False: - print('Immediate auth revocation specified but AES key rotation not allowed. ' - 'Minion will not be disconnected until the master AES key is rotated.') + if self.opts.get(u'rotate_aes_key') is False: + print(u'Immediate auth revocation specified but AES key rotation not allowed. ' + u'Minion will not be disconnected until the master AES key is rotated.') else: try: client = salt.client.get_local_client(mopts=self.opts) - client.cmd_async(key, 'saltutil.revoke_auth') + client.cmd_async(key, u'saltutil.revoke_auth') except salt.exceptions.SaltClientError: - print('Cannot contact Salt master. ' - 'Connection for {0} will remain up until ' - 'master AES key is rotated or auth is revoked ' - 'with \'saltutil.revoke_auth\'.'.format(key)) + print(u'Cannot contact Salt master. ' + u'Connection for {0} will remain up until ' + u'master AES key is rotated or auth is revoked ' + u'with \'saltutil.revoke_auth\'.'.format(key)) try: - os.remove(os.path.join(self.opts['pki_dir'], status, key)) + os.remove(os.path.join(self.opts[u'pki_dir'], status, key)) except (OSError, IOError): pass - self.check_minion_cache(preserve_minions=matches.get('minions', [])) + if self.opts.get('preserve_minions') is True: + self.check_minion_cache(preserve_minions=matches.get(u'minions', [])) + else: + self.check_minion_cache() return ( self.name_match(match) if match is not None else self.dict_match(matches) @@ -1264,7 +1274,7 @@ class RaetKey(Key): for status, keys in six.iteritems(self.list_keys()): for key in keys: try: - os.remove(os.path.join(self.opts['pki_dir'], status, key)) + os.remove(os.path.join(self.opts[u'pki_dir'], status, key)) except (OSError, IOError): pass self.check_minion_cache() @@ -1291,11 +1301,11 @@ class RaetKey(Key): try: shutil.move( os.path.join( - self.opts['pki_dir'], + self.opts[u'pki_dir'], keydir, key), os.path.join( - self.opts['pki_dir'], + self.opts[u'pki_dir'], self.REJ, key) ) @@ -1316,11 +1326,11 @@ class RaetKey(Key): try: shutil.move( os.path.join( - self.opts['pki_dir'], + self.opts[u'pki_dir'], self.PEND, key), os.path.join( - self.opts['pki_dir'], + self.opts[u'pki_dir'], self.REJ, key) ) @@ -1334,17 +1344,17 @@ class RaetKey(Key): Return the fingerprint for a specified key ''' if hash_type is None: - hash_type = __opts__['hash_type'] + hash_type = __opts__[u'hash_type'] matches = self.name_match(match, True) ret = {} for status, keys in six.iteritems(matches): ret[status] = {} for key in keys: - if status == 'local': - path = os.path.join(self.opts['pki_dir'], key) + if status == u'local': + path = os.path.join(self.opts[u'pki_dir'], key) else: - path = os.path.join(self.opts['pki_dir'], status, key) + path = os.path.join(self.opts[u'pki_dir'], status, key) ret[status][key] = self._get_key_finger(path) return ret @@ -1353,16 +1363,16 @@ class RaetKey(Key): Return fingerprints for all keys ''' if hash_type is None: - hash_type = __opts__['hash_type'] + hash_type = __opts__[u'hash_type'] ret = {} for status, keys in six.iteritems(self.list_keys()): ret[status] = {} for key in keys: - if status == 'local': - path = os.path.join(self.opts['pki_dir'], key) + if status == u'local': + path = os.path.join(self.opts[u'pki_dir'], key) else: - path = os.path.join(self.opts['pki_dir'], status, key) + path = os.path.join(self.opts[u'pki_dir'], status, key) ret[status][key] = self._get_key_finger(path) return ret @@ -1375,7 +1385,7 @@ class RaetKey(Key): for mid in mids: keydata = self.read_remote(mid, status) if keydata: - keydata['acceptance'] = status + keydata[u'acceptance'] = status data[mid] = keydata return data @@ -1384,10 +1394,10 @@ class RaetKey(Key): ''' Read in a remote key of status ''' - path = os.path.join(self.opts['pki_dir'], status, minion_id) + path = os.path.join(self.opts[u'pki_dir'], status, minion_id) if not os.path.isfile(path): return {} - with salt.utils.fopen(path, 'rb') as fp_: + with salt.utils.files.fopen(path, u'rb') as fp_: return self.serial.loads(fp_.read()) def read_local(self): @@ -1395,24 +1405,24 @@ class RaetKey(Key): Read in the local private keys, return an empy dict if the keys do not exist ''' - path = os.path.join(self.opts['pki_dir'], 'local.key') + path = os.path.join(self.opts[u'pki_dir'], u'local.key') if not os.path.isfile(path): return {} - with salt.utils.fopen(path, 'rb') as fp_: + with salt.utils.files.fopen(path, u'rb') as fp_: return self.serial.loads(fp_.read()) def write_local(self, priv, sign): ''' Write the private key and the signing key to a file on disk ''' - keydata = {'priv': priv, - 'sign': sign} - path = os.path.join(self.opts['pki_dir'], 'local.key') + keydata = {u'priv': priv, + u'sign': sign} + path = os.path.join(self.opts[u'pki_dir'], u'local.key') c_umask = os.umask(191) if os.path.exists(path): #mode = os.stat(path).st_mode os.chmod(path, stat.S_IWUSR | stat.S_IRUSR) - with salt.utils.fopen(path, 'w+') as fp_: + with salt.utils.files.fopen(path, u'w+') as fp_: fp_.write(self.serial.dumps(keydata)) os.chmod(path, stat.S_IRUSR) os.umask(c_umask) @@ -1421,7 +1431,7 @@ class RaetKey(Key): ''' Delete the local private key file ''' - path = os.path.join(self.opts['pki_dir'], 'local.key') + path = os.path.join(self.opts[u'pki_dir'], u'local.key') if os.path.isfile(path): os.remove(path) @@ -1429,6 +1439,6 @@ class RaetKey(Key): ''' Delete the private key directory ''' - path = self.opts['pki_dir'] + path = self.opts[u'pki_dir'] if os.path.exists(path): shutil.rmtree(path) diff --git a/salt/loader.py b/salt/loader.py index 366c16affbf..81e44ebb27a 100644 --- a/salt/loader.py +++ b/salt/loader.py @@ -8,13 +8,13 @@ plugin interfaces used by Salt. # Import python libs from __future__ import absolute_import import os -import imp import sys import time import logging import inspect import tempfile import functools +import types from collections import MutableMapping from zipimport import zipimporter @@ -22,17 +22,29 @@ from zipimport import zipimporter import salt.config import salt.syspaths import salt.utils.context -import salt.utils.lazy +import salt.utils.dictupdate import salt.utils.event +import salt.utils.files +import salt.utils.lazy import salt.utils.odict +import salt.utils.platform +import salt.utils.versions from salt.exceptions import LoaderError from salt.template import check_render_pipe_str from salt.utils.decorators import Depends -from salt.utils import is_proxy # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import reload_module + +if sys.version_info[:2] >= (3, 5): + import importlib.machinery # pylint: disable=no-name-in-module,import-error + import importlib.util # pylint: disable=no-name-in-module,import-error + USE_IMPORTLIB = True +else: + import imp + USE_IMPORTLIB = False + try: import pkg_resources HAS_PKG_RESOURCES = True @@ -42,19 +54,27 @@ except ImportError: log = logging.getLogger(__name__) SALT_BASE_PATH = os.path.abspath(salt.syspaths.INSTALL_DIR) -LOADED_BASE_NAME = 'salt.loaded' +LOADED_BASE_NAME = u'salt.loaded' -if six.PY3: - # pylint: disable=no-member,no-name-in-module,import-error - import importlib.machinery +if USE_IMPORTLIB: + # pylint: disable=no-member + MODULE_KIND_SOURCE = 1 + MODULE_KIND_COMPILED = 2 + MODULE_KIND_EXTENSION = 3 + MODULE_KIND_PKG_DIRECTORY = 5 SUFFIXES = [] for suffix in importlib.machinery.EXTENSION_SUFFIXES: - SUFFIXES.append((suffix, 'rb', 3)) + SUFFIXES.append((suffix, u'rb', MODULE_KIND_EXTENSION)) for suffix in importlib.machinery.BYTECODE_SUFFIXES: - SUFFIXES.append((suffix, 'rb', 2)) + SUFFIXES.append((suffix, u'rb', MODULE_KIND_COMPILED)) for suffix in importlib.machinery.SOURCE_SUFFIXES: - SUFFIXES.append((suffix, 'rb', 1)) - # pylint: enable=no-member,no-name-in-module,import-error + SUFFIXES.append((suffix, u'rb', MODULE_KIND_SOURCE)) + MODULE_KIND_MAP = { + MODULE_KIND_SOURCE: importlib.machinery.SourceFileLoader, + MODULE_KIND_COMPILED: importlib.machinery.SourcelessFileLoader, + MODULE_KIND_EXTENSION: importlib.machinery.ExtensionFileLoader + } + # pylint: enable=no-member else: SUFFIXES = imp.get_suffixes() @@ -62,17 +82,14 @@ else: # which simplifies code readability, it adds some unsupported functions into # the driver's module scope. # We list un-supported functions here. These will be removed from the loaded. +# TODO: remove the need for this cross-module code. Maybe use NotImplemented LIBCLOUD_FUNCS_NOT_SUPPORTED = ( - 'parallels.avail_sizes', - 'parallels.avail_locations', - 'proxmox.avail_sizes', - 'saltify.destroy', - 'saltify.avail_sizes', - 'saltify.avail_images', - 'saltify.avail_locations', - 'rackspace.reboot', - 'openstack.list_locations', - 'rackspace.list_locations' + u'parallels.avail_sizes', + u'parallels.avail_locations', + u'proxmox.avail_sizes', + u'rackspace.reboot', + u'openstack.list_locations', + u'rackspace.list_locations' ) # Will be set to pyximport module at runtime if cython is enabled in config. @@ -125,30 +142,30 @@ def _module_dirs( if tag is None: tag = ext_type sys_types = os.path.join(base_path or SALT_BASE_PATH, int_type or ext_type) - ext_types = os.path.join(opts['extension_modules'], ext_type) + ext_types = os.path.join(opts[u'extension_modules'], ext_type) ext_type_types = [] if ext_dirs: if ext_type_dirs is None: - ext_type_dirs = '{0}_dirs'.format(tag) + ext_type_dirs = u'{0}_dirs'.format(tag) if ext_type_dirs in opts: ext_type_types.extend(opts[ext_type_dirs]) if HAS_PKG_RESOURCES and ext_type_dirs: - for entry_point in pkg_resources.iter_entry_points('salt.loader', ext_type_dirs): + for entry_point in pkg_resources.iter_entry_points(u'salt.loader', ext_type_dirs): loaded_entry_point = entry_point.load() for path in loaded_entry_point(): ext_type_types.append(path) cli_module_dirs = [] # The dirs can be any module dir, or a in-tree _{ext_type} dir - for _dir in opts.get('module_dirs', []): + for _dir in opts.get(u'module_dirs', []): # Prepend to the list to match cli argument ordering maybe_dir = os.path.join(_dir, ext_type) if os.path.isdir(maybe_dir): cli_module_dirs.insert(0, maybe_dir) continue - maybe_dir = os.path.join(_dir, '_{0}'.format(ext_type)) + maybe_dir = os.path.join(_dir, u'_{0}'.format(ext_type)) if os.path.isdir(maybe_dir): cli_module_dirs.insert(0, maybe_dir) @@ -177,7 +194,7 @@ def minion_mods( generated modules in __context__ :param dict utils: Utility functions which should be made available to - Salt modules in __utils__. See `utils_dir` in + Salt modules in __utils__. See `utils_dirs` in salt.config for additional information about configuration. @@ -201,23 +218,23 @@ def minion_mods( ''' # TODO Publish documentation for module whitelisting if not whitelist: - whitelist = opts.get('whitelist_modules', None) + whitelist = opts.get(u'whitelist_modules', None) ret = LazyLoader( - _module_dirs(opts, 'modules', 'module'), + _module_dirs(opts, u'modules', u'module'), opts, - tag='module', - pack={'__context__': context, '__utils__': utils, '__proxy__': proxy}, + tag=u'module', + pack={u'__context__': context, u'__utils__': utils, u'__proxy__': proxy}, whitelist=whitelist, loaded_base_name=loaded_base_name, static_modules=static_modules, ) - ret.pack['__salt__'] = ret + ret.pack[u'__salt__'] = ret # Load any provider overrides from the configuration file providers option # Note: Providers can be pkg, service, user or group - not to be confused # with cloud providers. - providers = opts.get('providers', False) + providers = opts.get(u'providers', False) if providers and isinstance(providers, dict): for mod in providers: # sometimes providers opts is not to diverge modules but @@ -229,17 +246,17 @@ def minion_mods( else: if funcs: for func in funcs: - f_key = '{0}{1}'.format(mod, func[func.rindex('.'):]) + f_key = u'{0}{1}'.format(mod, func[func.rindex(u'.'):]) ret[f_key] = funcs[func] if notify: - evt = salt.utils.event.get_event('minion', opts=opts, listen=False) - evt.fire_event({'complete': True}, tag='/salt/minion/minion_mod_complete') + evt = salt.utils.event.get_event(u'minion', opts=opts, listen=False) + evt.fire_event({u'complete': True}, tag=u'/salt/minion/minion_mod_complete') return ret -def raw_mod(opts, name, functions, mod='modules'): +def raw_mod(opts, name, functions, mod=u'modules'): ''' Returns a single module loaded raw and bypassing the __virtual__ function @@ -253,11 +270,11 @@ def raw_mod(opts, name, functions, mod='modules'): testmod['test.ping']() ''' loader = LazyLoader( - _module_dirs(opts, mod, 'rawmodule'), + _module_dirs(opts, mod, u'module'), opts, - tag='rawmodule', + tag=u'rawmodule', virtual_enable=False, - pack={'__salt__': functions}, + pack={u'__salt__': functions}, ) # if we don't have the module, return an empty dict if name not in loader.file_mapping: @@ -271,14 +288,14 @@ def engines(opts, functions, runners, utils, proxy=None): ''' Return the master services plugins ''' - pack = {'__salt__': functions, - '__runners__': runners, - '__proxy__': proxy, - '__utils__': utils} + pack = {u'__salt__': functions, + u'__runners__': runners, + u'__proxy__': proxy, + u'__utils__': utils} return LazyLoader( - _module_dirs(opts, 'engines'), + _module_dirs(opts, u'engines'), opts, - tag='engines', + tag=u'engines', pack=pack, ) @@ -288,13 +305,13 @@ def proxy(opts, functions=None, returners=None, whitelist=None, utils=None): Returns the proxy module for this salt-proxy-minion ''' ret = LazyLoader( - _module_dirs(opts, 'proxy'), + _module_dirs(opts, u'proxy'), opts, - tag='proxy', - pack={'__salt__': functions, '__ret__': returners, '__utils__': utils}, + tag=u'proxy', + pack={u'__salt__': functions, u'__ret__': returners, u'__utils__': utils}, ) - ret.pack['__proxy__'] = ret + ret.pack[u'__proxy__'] = ret return ret @@ -304,11 +321,11 @@ def returners(opts, functions, whitelist=None, context=None, proxy=None): Returns the returner modules ''' return LazyLoader( - _module_dirs(opts, 'returners', 'returner'), + _module_dirs(opts, u'returners', u'returner'), opts, - tag='returner', + tag=u'returner', whitelist=whitelist, - pack={'__salt__': functions, '__context__': context, '__proxy__': proxy or {}}, + pack={u'__salt__': functions, u'__context__': context, u'__proxy__': proxy or {}}, ) @@ -317,11 +334,11 @@ def utils(opts, whitelist=None, context=None, proxy=proxy): Returns the utility modules ''' return LazyLoader( - _module_dirs(opts, 'utils', ext_type_dirs='utils_dirs'), + _module_dirs(opts, u'utils', ext_type_dirs=u'utils_dirs'), opts, - tag='utils', + tag=u'utils', whitelist=whitelist, - pack={'__context__': context, '__proxy__': proxy or {}}, + pack={u'__context__': context, u'__proxy__': proxy or {}}, ) @@ -329,30 +346,30 @@ def pillars(opts, functions, context=None): ''' Returns the pillars modules ''' - ret = LazyLoader(_module_dirs(opts, 'pillar'), + ret = LazyLoader(_module_dirs(opts, u'pillar'), opts, - tag='pillar', - pack={'__salt__': functions, - '__context__': context, - '__utils__': utils(opts)}) - ret.pack['__ext_pillar__'] = ret - return FilterDictWrapper(ret, '.ext_pillar') + tag=u'pillar', + pack={u'__salt__': functions, + u'__context__': context, + u'__utils__': utils(opts)}) + ret.pack[u'__ext_pillar__'] = ret + return FilterDictWrapper(ret, u'.ext_pillar') def tops(opts): ''' Returns the tops modules ''' - if 'master_tops' not in opts: + if u'master_tops' not in opts: return {} - whitelist = list(opts['master_tops'].keys()) + whitelist = list(opts[u'master_tops'].keys()) ret = LazyLoader( - _module_dirs(opts, 'tops', 'top'), + _module_dirs(opts, u'tops', u'top'), opts, - tag='top', + tag=u'top', whitelist=whitelist, ) - return FilterDictWrapper(ret, '.top') + return FilterDictWrapper(ret, u'.top') def wheels(opts, whitelist=None): @@ -360,9 +377,9 @@ def wheels(opts, whitelist=None): Returns the wheels modules ''' return LazyLoader( - _module_dirs(opts, 'wheel'), + _module_dirs(opts, u'wheel'), opts, - tag='wheel', + tag=u'wheel', whitelist=whitelist, ) @@ -375,13 +392,13 @@ def outputters(opts): :returns: LazyLoader instance, with only outputters present in the keyspace ''' ret = LazyLoader( - _module_dirs(opts, 'output', ext_type_dirs='outputter_dirs'), + _module_dirs(opts, u'output', ext_type_dirs=u'outputter_dirs'), opts, - tag='output', + tag=u'output', ) - wrapped_ret = FilterDictWrapper(ret, '.output') + wrapped_ret = FilterDictWrapper(ret, u'.output') # TODO: this name seems terrible... __salt__ should always be execution mods - ret.pack['__salt__'] = wrapped_ret + ret.pack[u'__salt__'] = wrapped_ret return wrapped_ret @@ -392,9 +409,22 @@ def serializers(opts): :returns: LazyLoader instance, with only serializers present in the keyspace ''' return LazyLoader( - _module_dirs(opts, 'serializers'), + _module_dirs(opts, u'serializers'), opts, - tag='serializers', + tag=u'serializers', + ) + + +def eauth_tokens(opts): + ''' + Returns the tokens modules + :param dict opts: The Salt options dictionary + :returns: LazyLoader instance, with only token backends present in the keyspace + ''' + return LazyLoader( + _module_dirs(opts, 'tokens'), + opts, + tag='tokens', ) @@ -406,11 +436,11 @@ def auth(opts, whitelist=None): :returns: LazyLoader ''' return LazyLoader( - _module_dirs(opts, 'auth'), + _module_dirs(opts, u'auth'), opts, - tag='auth', + tag=u'auth', whitelist=whitelist, - pack={'__salt__': minion_mods(opts)}, + pack={u'__salt__': minion_mods(opts)}, ) @@ -418,11 +448,11 @@ def fileserver(opts, backends): ''' Returns the file server modules ''' - return LazyLoader(_module_dirs(opts, 'fileserver'), + return LazyLoader(_module_dirs(opts, u'fileserver'), opts, - tag='fileserver', + tag=u'fileserver', whitelist=backends, - pack={'__utils__': utils(opts)}) + pack={u'__utils__': utils(opts)}) def roster(opts, runner, whitelist=None): @@ -430,11 +460,11 @@ def roster(opts, runner, whitelist=None): Returns the roster modules ''' return LazyLoader( - _module_dirs(opts, 'roster'), + _module_dirs(opts, u'roster'), opts, - tag='roster', + tag=u'roster', whitelist=whitelist, - pack={'__runner__': runner}, + pack={u'__runner__': runner}, ) @@ -442,12 +472,12 @@ def thorium(opts, functions, runners): ''' Load the thorium runtime modules ''' - pack = {'__salt__': functions, '__runner__': runners, '__context__': {}} - ret = LazyLoader(_module_dirs(opts, 'thorium'), + pack = {u'__salt__': functions, u'__runner__': runners, u'__context__': {}} + ret = LazyLoader(_module_dirs(opts, u'thorium'), opts, - tag='thorium', + tag=u'thorium', pack=pack) - ret.pack['__thorium__'] = ret + ret.pack[u'__thorium__'] = ret return ret @@ -468,15 +498,15 @@ def states(opts, functions, utils, serializers, whitelist=None, proxy=None): statemods = salt.loader.states(__opts__, None, None) ''' ret = LazyLoader( - _module_dirs(opts, 'states'), + _module_dirs(opts, u'states'), opts, - tag='states', - pack={'__salt__': functions, '__proxy__': proxy or {}}, + tag=u'states', + pack={u'__salt__': functions, u'__proxy__': proxy or {}}, whitelist=whitelist, ) - ret.pack['__states__'] = ret - ret.pack['__utils__'] = utils - ret.pack['__serializers__'] = serializers + ret.pack[u'__states__'] = ret + ret.pack[u'__utils__'] = utils + ret.pack[u'__serializers__'] = serializers return ret @@ -489,11 +519,11 @@ def beacons(opts, functions, context=None, proxy=None): keys and funcs as values. ''' return LazyLoader( - _module_dirs(opts, 'beacons'), + _module_dirs(opts, u'beacons'), opts, - tag='beacons', - pack={'__context__': context, '__salt__': functions, '__proxy__': proxy or {}}, - virtual_funcs=['__validate__'], + tag=u'beacons', + pack={u'__context__': context, u'__salt__': functions, u'__proxy__': proxy or {}}, + virtual_funcs=[], ) @@ -506,14 +536,14 @@ def log_handlers(opts): ret = LazyLoader( _module_dirs( opts, - 'log_handlers', - int_type='handlers', - base_path=os.path.join(SALT_BASE_PATH, 'log'), + u'log_handlers', + int_type=u'handlers', + base_path=os.path.join(SALT_BASE_PATH, u'log'), ), opts, - tag='log_handlers', + tag=u'log_handlers', ) - return FilterDictWrapper(ret, '.setup_handlers') + return FilterDictWrapper(ret, u'.setup_handlers') def ssh_wrapper(opts, functions=None, context=None): @@ -523,44 +553,45 @@ def ssh_wrapper(opts, functions=None, context=None): return LazyLoader( _module_dirs( opts, - 'wrapper', - base_path=os.path.join(SALT_BASE_PATH, os.path.join('client', 'ssh')), + u'wrapper', + base_path=os.path.join(SALT_BASE_PATH, os.path.join(u'client', u'ssh')), ), opts, - tag='wrapper', + tag=u'wrapper', pack={ - '__salt__': functions, - '__grains__': opts.get('grains', {}), - '__pillar__': opts.get('pillar', {}), - '__context__': context, + u'__salt__': functions, + u'__grains__': opts.get(u'grains', {}), + u'__pillar__': opts.get(u'pillar', {}), + u'__context__': context, }, ) -def render(opts, functions, states=None): +def render(opts, functions, states=None, proxy=None): ''' Returns the render modules ''' - pack = {'__salt__': functions, - '__grains__': opts.get('grains', {})} + pack = {u'__salt__': functions, + u'__grains__': opts.get(u'grains', {})} if states: - pack['__states__'] = states + pack[u'__states__'] = states + pack[u'__proxy__'] = proxy or {} ret = LazyLoader( _module_dirs( opts, - 'renderers', - 'render', - ext_type_dirs='render_dirs', + u'renderers', + u'render', + ext_type_dirs=u'render_dirs', ), opts, - tag='render', + tag=u'render', pack=pack, ) - rend = FilterDictWrapper(ret, '.render') + rend = FilterDictWrapper(ret, u'.render') - if not check_render_pipe_str(opts['renderer'], rend, opts['renderer_blacklist'], opts['renderer_whitelist']): - err = ('The renderer {0} is unavailable, this error is often because ' - 'the needed software is unavailable'.format(opts['renderer'])) + if not check_render_pipe_str(opts[u'renderer'], rend, opts[u'renderer_blacklist'], opts[u'renderer_whitelist']): + err = (u'The renderer {0} is unavailable, this error is often because ' + u'the needed software is unavailable'.format(opts[u'renderer'])) log.critical(err) raise LoaderError(err) return rend @@ -581,12 +612,12 @@ def grain_funcs(opts, proxy=None): return LazyLoader( _module_dirs( opts, - 'grains', - 'grain', - ext_type_dirs='grains_dirs', + u'grains', + u'grain', + ext_type_dirs=u'grains_dirs', ), opts, - tag='grains', + tag=u'grains', ) @@ -596,30 +627,30 @@ def _load_cached_grains(opts, cfn): corrupted. ''' if not os.path.isfile(cfn): - log.debug('Grains cache file does not exist.') + log.debug(u'Grains cache file does not exist.') return None grains_cache_age = int(time.time() - os.path.getmtime(cfn)) - if grains_cache_age > opts.get('grains_cache_expiration', 300): - log.debug('Grains cache last modified {0} seconds ago and ' - 'cache expiration is set to {1}. ' - 'Grains cache expired. Refreshing.'.format( - grains_cache_age, - opts.get('grains_cache_expiration', 300) - )) + if grains_cache_age > opts.get(u'grains_cache_expiration', 300): + log.debug( + u'Grains cache last modified %s seconds ago and cache ' + u'expiration is set to %s. Grains cache expired. ' + u'Refreshing.', + grains_cache_age, opts.get(u'grains_cache_expiration', 300) + ) return None - if opts.get('refresh_grains_cache', False): - log.debug('refresh_grains_cache requested, Refreshing.') + if opts.get(u'refresh_grains_cache', False): + log.debug(u'refresh_grains_cache requested, Refreshing.') return None - log.debug('Retrieving grains from cache') + log.debug(u'Retrieving grains from cache') try: serial = salt.payload.Serial(opts) - with salt.utils.fopen(cfn, 'rb') as fp_: + with salt.utils.files.fopen(cfn, u'rb') as fp_: cached_grains = serial.load(fp_) if not cached_grains: - log.debug('Cached grains are empty, cache might be corrupted. Refreshing.') + log.debug(u'Cached grains are empty, cache might be corrupted. Refreshing.') return None return cached_grains @@ -650,41 +681,41 @@ def grains(opts, force_refresh=False, proxy=None): import salt.config # if we have no grains, lets try loading from disk (TODO: move to decorator?) cfn = os.path.join( - opts['cachedir'], - 'grains.cache.p' + opts[u'cachedir'], + u'grains.cache.p' ) - if not force_refresh and opts.get('grains_cache', False): + if not force_refresh and opts.get(u'grains_cache', False): cached_grains = _load_cached_grains(opts, cfn) if cached_grains: return cached_grains else: - log.debug('Grains refresh requested. Refreshing grains.') + log.debug(u'Grains refresh requested. Refreshing grains.') - if opts.get('skip_grains', False): + if opts.get(u'skip_grains', False): return {} - grains_deep_merge = opts.get('grains_deep_merge', False) is True - if 'conf_file' in opts: + grains_deep_merge = opts.get(u'grains_deep_merge', False) is True + if u'conf_file' in opts: pre_opts = {} pre_opts.update(salt.config.load_config( - opts['conf_file'], 'SALT_MINION_CONFIG', - salt.config.DEFAULT_MINION_OPTS['conf_file'] + opts[u'conf_file'], u'SALT_MINION_CONFIG', + salt.config.DEFAULT_MINION_OPTS[u'conf_file'] )) default_include = pre_opts.get( - 'default_include', opts['default_include'] + u'default_include', opts[u'default_include'] ) - include = pre_opts.get('include', []) + include = pre_opts.get(u'include', []) pre_opts.update(salt.config.include_config( - default_include, opts['conf_file'], verbose=False + default_include, opts[u'conf_file'], verbose=False )) pre_opts.update(salt.config.include_config( - include, opts['conf_file'], verbose=True + include, opts[u'conf_file'], verbose=True )) - if 'grains' in pre_opts: - opts['grains'] = pre_opts['grains'] + if u'grains' in pre_opts: + opts[u'grains'] = pre_opts[u'grains'] else: - opts['grains'] = {} + opts[u'grains'] = {} else: - opts['grains'] = {} + opts[u'grains'] = {} grains_data = {} funcs = grain_funcs(opts, proxy=proxy) @@ -692,9 +723,9 @@ def grains(opts, force_refresh=False, proxy=None): funcs.clear() # Run core grains for key in funcs: - if not key.startswith('core.'): + if not key.startswith(u'core.'): continue - log.trace('Loading {0} grain'.format(key)) + log.trace(u'Loading %s grain', key) ret = funcs[key]() if not isinstance(ret, dict): continue @@ -705,7 +736,7 @@ def grains(opts, force_refresh=False, proxy=None): # Run the rest of the grains for key in funcs: - if key.startswith('core.') or key == '_errors': + if key.startswith(u'core.') or key == u'_errors': continue try: # Grains are loaded too early to take advantage of the injected @@ -714,19 +745,17 @@ def grains(opts, force_refresh=False, proxy=None): # one parameter. Then the grains can have access to the # proxymodule for retrieving information from the connected # device. - log.trace('Loading {0} grain'.format(key)) + log.trace(u'Loading %s grain', key) if funcs[key].__code__.co_argcount == 1: ret = funcs[key](proxy) else: ret = funcs[key]() except Exception: - if is_proxy(): - log.info('The following CRITICAL message may not be an error; the proxy may not be completely established yet.') + if salt.utils.platform.is_proxy(): + log.info(u'The following CRITICAL message may not be an error; the proxy may not be completely established yet.') log.critical( - 'Failed to load grains defined in grain file {0} in ' - 'function {1}, error:\n'.format( - key, funcs[key] - ), + u'Failed to load grains defined in grain file %s in ' + u'function %s, error:\n', key, funcs[key], exc_info=True ) continue @@ -737,51 +766,56 @@ def grains(opts, force_refresh=False, proxy=None): else: grains_data.update(ret) - if opts.get('proxy_merge_grains_in_module', True) and proxy: + if opts.get(u'proxy_merge_grains_in_module', True) and proxy: try: - proxytype = proxy.opts['proxy']['proxytype'] - if proxytype+'.grains' in proxy: - if proxytype+'.initialized' in proxy and proxy[proxytype+'.initialized'](): + proxytype = proxy.opts[u'proxy'][u'proxytype'] + if proxytype + u'.grains' in proxy: + if proxytype + u'.initialized' in proxy and proxy[proxytype + u'.initialized'](): try: - proxytype = proxy.opts['proxy']['proxytype'] - ret = proxy[proxytype+'.grains']() + proxytype = proxy.opts[u'proxy'][u'proxytype'] + ret = proxy[proxytype + u'.grains']() if grains_deep_merge: salt.utils.dictupdate.update(grains_data, ret) else: grains_data.update(ret) except Exception: - log.critical('Failed to run proxy\'s grains function!', + log.critical(u'Failed to run proxy\'s grains function!', exc_info=True ) except KeyError: pass - grains_data.update(opts['grains']) + grains_data.update(opts[u'grains']) # Write cache if enabled - if opts.get('grains_cache', False): + if opts.get(u'grains_cache', False): cumask = os.umask(0o77) try: - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): # Late import import salt.modules.cmdmod # Make sure cache file isn't read-only - salt.modules.cmdmod._run_quiet('attrib -R "{0}"'.format(cfn)) - with salt.utils.fopen(cfn, 'w+b') as fp_: + salt.modules.cmdmod._run_quiet(u'attrib -R "{0}"'.format(cfn)) + with salt.utils.files.fopen(cfn, u'w+b') as fp_: try: serial = salt.payload.Serial(opts) serial.dump(grains_data, fp_) - except TypeError: - # Can't serialize pydsl - pass - except (IOError, OSError): - msg = 'Unable to write to grains cache file {0}' - log.error(msg.format(cfn)) + except TypeError as e: + log.error(u'Failed to serialize grains cache: %s', e) + raise # re-throw for cleanup + except Exception as e: + log.error(u'Unable to write to grains cache file %s: %s', cfn, e) + # Based on the original exception, the file may or may not have been + # created. If it was, we will remove it now, as the exception means + # the serialized data is not to be trusted, no matter what the + # exception is. + if os.path.isfile(cfn): + os.unlink(cfn) os.umask(cumask) if grains_deep_merge: - salt.utils.dictupdate.update(grains_data, opts['grains']) + salt.utils.dictupdate.update(grains_data, opts[u'grains']) else: - grains_data.update(opts['grains']) + grains_data.update(opts[u'grains']) return grains_data @@ -790,13 +824,13 @@ def call(fun, **kwargs): ''' Directly call a function inside a loader directory ''' - args = kwargs.get('args', []) - dirs = kwargs.get('dirs', []) + args = kwargs.get(u'args', []) + dirs = kwargs.get(u'dirs', []) funcs = LazyLoader( - [os.path.join(SALT_BASE_PATH, 'modules')] + dirs, + [os.path.join(SALT_BASE_PATH, u'modules')] + dirs, None, - tag='modules', + tag=u'modules', virtual_enable=False, ) return funcs[fun](*args) @@ -809,13 +843,13 @@ def runner(opts, utils=None): if utils is None: utils = {} ret = LazyLoader( - _module_dirs(opts, 'runners', 'runner', ext_type_dirs='runner_dirs'), + _module_dirs(opts, u'runners', u'runner', ext_type_dirs=u'runner_dirs'), opts, - tag='runners', - pack={'__utils__': utils}, + tag=u'runners', + pack={u'__utils__': utils}, ) # TODO: change from __salt__ to something else, we overload __salt__ too much - ret.pack['__salt__'] = ret + ret.pack[u'__salt__'] = ret return ret @@ -824,9 +858,9 @@ def queues(opts): Directly call a function inside a loader directory ''' return LazyLoader( - _module_dirs(opts, 'queues', 'queue', ext_type_dirs='queue_dirs'), + _module_dirs(opts, u'queues', u'queue', ext_type_dirs=u'queue_dirs'), opts, - tag='queues', + tag=u'queues', ) @@ -838,13 +872,13 @@ def sdb(opts, functions=None, whitelist=None, utils=None): utils = {} return LazyLoader( - _module_dirs(opts, 'sdb'), + _module_dirs(opts, u'sdb'), opts, - tag='sdb', + tag=u'sdb', pack={ - '__sdb__': functions, - '__opts__': opts, - '__utils__': utils, + u'__sdb__': functions, + u'__opts__': opts, + u'__utils__': utils, }, whitelist=whitelist, ) @@ -859,11 +893,11 @@ def pkgdb(opts): return LazyLoader( _module_dirs( opts, - 'pkgdb', - base_path=os.path.join(SALT_BASE_PATH, 'spm') + u'pkgdb', + base_path=os.path.join(SALT_BASE_PATH, u'spm') ), opts, - tag='pkgdb' + tag=u'pkgdb' ) @@ -876,11 +910,11 @@ def pkgfiles(opts): return LazyLoader( _module_dirs( opts, - 'pkgfiles', - base_path=os.path.join(SALT_BASE_PATH, 'spm') + u'pkgfiles', + base_path=os.path.join(SALT_BASE_PATH, u'spm') ), opts, - tag='pkgfiles' + tag=u'pkgfiles' ) @@ -893,21 +927,19 @@ def clouds(opts): # manager when needed. functions = LazyLoader( _module_dirs(opts, - 'clouds', - 'cloud', - base_path=os.path.join(SALT_BASE_PATH, 'cloud'), - int_type='clouds'), + u'clouds', + u'cloud', + base_path=os.path.join(SALT_BASE_PATH, u'cloud'), + int_type=u'clouds'), opts, - tag='clouds', - pack={'__utils__': salt.loader.utils(opts), - '__active_provider_name__': None}, + tag=u'clouds', + pack={u'__utils__': salt.loader.utils(opts), + u'__active_provider_name__': None}, ) for funcname in LIBCLOUD_FUNCS_NOT_SUPPORTED: log.trace( - '\'{0}\' has been marked as not supported. Removing from the list ' - 'of supported cloud functions'.format( - funcname - ) + u'\'%s\' has been marked as not supported. Removing from the ' + u'list of supported cloud functions', funcname ) functions.pop(funcname, None) return functions @@ -918,9 +950,9 @@ def netapi(opts): Return the network api functions ''' return LazyLoader( - _module_dirs(opts, 'netapi'), + _module_dirs(opts, u'netapi'), opts, - tag='netapi', + tag=u'netapi', ) @@ -929,10 +961,10 @@ def executors(opts, functions=None, context=None, proxy=None): Returns the executor modules ''' return LazyLoader( - _module_dirs(opts, 'executors', 'executor'), + _module_dirs(opts, u'executors', u'executor'), opts, - tag='executor', - pack={'__salt__': functions, '__context__': context or {}, '__proxy__': proxy or {}}, + tag=u'executor', + pack={u'__salt__': functions, u'__context__': context or {}, u'__proxy__': proxy or {}}, ) @@ -941,10 +973,10 @@ def cache(opts, serial): Returns the returner modules ''' return LazyLoader( - _module_dirs(opts, 'cache', 'cache'), + _module_dirs(opts, u'cache', u'cache'), opts, - tag='cache', - pack={'__opts__': opts, '__context__': {'serial': serial}}, + tag=u'cache', + pack={u'__opts__': opts, u'__context__': {u'serial': serial}}, ) @@ -952,16 +984,17 @@ def _generate_module(name): if name in sys.modules: return - code = "'''Salt loaded {0} parent module'''".format(name.split('.')[-1]) - module = imp.new_module(name) + code = u"'''Salt loaded {0} parent module'''".format(name.split(u'.')[-1]) + # ModuleType can't accept a unicode type on PY2 + module = types.ModuleType(str(name)) exec(code, module.__dict__) sys.modules[name] = module def _mod_type(module_path): if module_path.startswith(SALT_BASE_PATH): - return 'int' - return 'ext' + return u'int' + return u'ext' # TODO: move somewhere else? @@ -990,7 +1023,7 @@ class FilterDictWrapper(MutableMapping): def __iter__(self): for key in self._dict: if key.endswith(self.suffix): - yield key.replace(self.suffix, '') + yield key.replace(self.suffix, u'') class LazyLoader(salt.utils.lazy.LazyDict): @@ -1025,7 +1058,7 @@ class LazyLoader(salt.utils.lazy.LazyDict): def __init__(self, module_dirs, opts=None, - tag='module', + tag=u'module', loaded_base_name=None, mod_type_check=None, pack=None, @@ -1044,7 +1077,7 @@ class LazyLoader(salt.utils.lazy.LazyDict): self.pack = {} if pack is None else pack if opts is None: opts = {} - threadsafety = not opts.get('multiprocessing') + threadsafety = not opts.get(u'multiprocessing') self.context_dict = salt.utils.context.ContextDict(threadsafe=threadsafety) self.opts = self.__prep_mod_opts(opts) @@ -1053,8 +1086,8 @@ class LazyLoader(salt.utils.lazy.LazyDict): self.loaded_base_name = loaded_base_name or LOADED_BASE_NAME self.mod_type_check = mod_type_check or _mod_type - if '__context__' not in self.pack: - self.pack['__context__'] = None + if u'__context__' not in self.pack: + self.pack[u'__context__'] = None for k, v in six.iteritems(self.pack): if v is None: # if the value of a pack is None, lets make an empty dict @@ -1075,16 +1108,24 @@ class LazyLoader(salt.utils.lazy.LazyDict): virtual_funcs = [] self.virtual_funcs = virtual_funcs - self.disabled = set(self.opts.get('disable_{0}s'.format(self.tag), [])) + self.disabled = set( + self.opts.get( + u'disable_{0}{1}'.format( + self.tag, + u'' if self.tag[-1] == u's' else u's' + ), + [] + ) + ) self.refresh_file_mapping() super(LazyLoader, self).__init__() # late init the lazy loader # create all of the import namespaces - _generate_module('{0}.int'.format(self.loaded_base_name)) - _generate_module('{0}.int.{1}'.format(self.loaded_base_name, tag)) - _generate_module('{0}.ext'.format(self.loaded_base_name)) - _generate_module('{0}.ext.{1}'.format(self.loaded_base_name, tag)) + _generate_module(u'{0}.int'.format(self.loaded_base_name)) + _generate_module(u'{0}.int.{1}'.format(self.loaded_base_name, tag)) + _generate_module(u'{0}.ext'.format(self.loaded_base_name)) + _generate_module(u'{0}.ext.{1}'.format(self.loaded_base_name, tag)) def __getitem__(self, item): ''' @@ -1102,7 +1143,7 @@ class LazyLoader(salt.utils.lazy.LazyDict): Allow for "direct" attribute access-- this allows jinja templates to access things like `salt.test.ping()` ''' - if mod_name in ('__getstate__', '__setstate__'): + if mod_name in (u'__getstate__', u'__setstate__'): return object.__getattribute__(self, mod_name) # if we have an attribute named that, lets return it. @@ -1130,19 +1171,19 @@ class LazyLoader(salt.utils.lazy.LazyDict): This can range from "not available' to "__virtual__" returned False ''' - mod_name = function_name.split('.')[0] + mod_name = function_name.split(u'.')[0] if mod_name in self.loaded_modules: - return '\'{0}\' is not available.'.format(function_name) + return u'\'{0}\' is not available.'.format(function_name) else: try: reason = self.missing_modules[mod_name] except KeyError: - return '\'{0}\' is not available.'.format(function_name) + return u'\'{0}\' is not available.'.format(function_name) else: if reason is not None: - return '\'{0}\' __virtual__ returned False: {1}'.format(mod_name, reason) + return u'\'{0}\' __virtual__ returned False: {1}'.format(mod_name, reason) else: - return '\'{0}\' __virtual__ returned False'.format(mod_name) + return u'\'{0}\' __virtual__ returned False'.format(mod_name) def refresh_file_mapping(self): ''' @@ -1150,27 +1191,30 @@ class LazyLoader(salt.utils.lazy.LazyDict): ''' # map of suffix to description for imp self.suffix_map = {} - suffix_order = [''] # local list to determine precedence of extensions + suffix_order = [u''] # local list to determine precedence of extensions # Prefer packages (directories) over modules (single files)! for (suffix, mode, kind) in SUFFIXES: self.suffix_map[suffix] = (suffix, mode, kind) suffix_order.append(suffix) - if self.opts.get('cython_enable', True) is True: + if self.opts.get(u'cython_enable', True) is True: try: global pyximport - pyximport = __import__('pyximport') # pylint: disable=import-error + pyximport = __import__(u'pyximport') # pylint: disable=import-error pyximport.install() # add to suffix_map so file_mapping will pick it up - self.suffix_map['.pyx'] = tuple() + self.suffix_map[u'.pyx'] = tuple() except ImportError: - log.info('Cython is enabled in the options but not present ' - 'in the system path. Skipping Cython modules.') + log.info(u'Cython is enabled in the options but not present ' + u'in the system path. Skipping Cython modules.') # Allow for zipimport of modules - if self.opts.get('enable_zip_modules', True) is True: - self.suffix_map['.zip'] = tuple() + if self.opts.get(u'enable_zip_modules', True) is True: + self.suffix_map[u'.zip'] = tuple() # allow for module dirs - self.suffix_map[''] = ('', '', imp.PKG_DIRECTORY) + if USE_IMPORTLIB: + self.suffix_map[u''] = (u'', u'', MODULE_KIND_PKG_DIRECTORY) + else: + self.suffix_map[u''] = (u'', u'', imp.PKG_DIRECTORY) # create mapping of filename (without suffix) to (path, suffix) # The files are added in order of priority, so order *must* be retained. @@ -1185,7 +1229,7 @@ class LazyLoader(salt.utils.lazy.LazyDict): continue # Next mod_dir for filename in files: try: - if filename.startswith('_'): + if filename.startswith(u'_'): # skip private modules # log messages omitted for obviousness continue # Next filename @@ -1195,20 +1239,19 @@ class LazyLoader(salt.utils.lazy.LazyDict): continue # Next filename if f_noext in self.disabled: log.trace( - 'Skipping {0}, it is disabled by configuration'.format( + u'Skipping %s, it is disabled by configuration', filename - ) ) continue # Next filename fpath = os.path.join(mod_dir, filename) # if its a directory, lets allow us to load that - if ext == '': + if ext == u'': # is there something __init__? subfiles = os.listdir(fpath) for suffix in suffix_order: - if '' == suffix: + if u'' == suffix: continue # Next suffix (__init__ must have a suffix) - init_file = '__init__{0}'.format(suffix) + init_file = u'__init__{0}'.format(suffix) if init_file in subfiles: break else: @@ -1217,9 +1260,9 @@ class LazyLoader(salt.utils.lazy.LazyDict): if f_noext in self.file_mapping: curr_ext = self.file_mapping[f_noext][1] #log.debug("****** curr_ext={0} ext={1} suffix_order={2}".format(curr_ext, ext, suffix_order)) - if '' in (curr_ext, ext) and curr_ext != ext: + if u'' in (curr_ext, ext) and curr_ext != ext: log.error( - 'Module/package collision: \'%s\' and \'%s\'', + u'Module/package collision: \'%s\' and \'%s\'', fpath, self.file_mapping[f_noext][0] ) @@ -1232,8 +1275,8 @@ class LazyLoader(salt.utils.lazy.LazyDict): except OSError: continue for smod in self.static_modules: - f_noext = smod.split('.')[-1] - self.file_mapping[f_noext] = (smod, '.o') + f_noext = smod.split(u'.')[-1] + self.file_mapping[f_noext] = (smod, u'.o') def clear(self): ''' @@ -1245,7 +1288,7 @@ class LazyLoader(salt.utils.lazy.LazyDict): self.loaded_modules = {} # if we have been loaded before, lets clear the file mapping since # we obviously want a re-do - if hasattr(self, 'opts'): + if hasattr(self, u'opts'): self.refresh_file_mapping() self.initial_load = False @@ -1253,17 +1296,17 @@ class LazyLoader(salt.utils.lazy.LazyDict): ''' Strip out of the opts any logger instance ''' - if '__grains__' not in self.pack: - self.context_dict['grains'] = opts.get('grains', {}) - self.pack['__grains__'] = salt.utils.context.NamespacedDictWrapper(self.context_dict, 'grains', override_name='grains') + if u'__grains__' not in self.pack: + self.context_dict[u'grains'] = opts.get(u'grains', {}) + self.pack[u'__grains__'] = salt.utils.context.NamespacedDictWrapper(self.context_dict, u'grains', override_name=u'grains') - if '__pillar__' not in self.pack: - self.context_dict['pillar'] = opts.get('pillar', {}) - self.pack['__pillar__'] = salt.utils.context.NamespacedDictWrapper(self.context_dict, 'pillar', override_name='pillar') + if u'__pillar__' not in self.pack: + self.context_dict[u'pillar'] = opts.get(u'pillar', {}) + self.pack[u'__pillar__'] = salt.utils.context.NamespacedDictWrapper(self.context_dict, u'pillar', override_name=u'pillar') mod_opts = {} for key, val in list(opts.items()): - if key == 'logger': + if key == u'logger': continue mod_opts[key] = val return mod_opts @@ -1295,7 +1338,7 @@ class LazyLoader(salt.utils.lazy.LazyDict): # reload only custom "sub"modules for submodule in submodules: # it is a submodule if the name is in a namespace under mod - if submodule.__name__.startswith(mod.__name__ + '.'): + if submodule.__name__.startswith(mod.__name__ + u'.'): reload_module(submodule) self._reload_submodules(submodule) @@ -1306,81 +1349,118 @@ class LazyLoader(salt.utils.lazy.LazyDict): fpath_dirname = os.path.dirname(fpath) try: sys.path.append(fpath_dirname) - if suffix == '.pyx': + if suffix == u'.pyx': mod = pyximport.load_module(name, fpath, tempfile.gettempdir()) - elif suffix == '.o': + elif suffix == u'.o': top_mod = __import__(fpath, globals(), locals(), []) - comps = fpath.split('.') + comps = fpath.split(u'.') if len(comps) < 2: mod = top_mod else: mod = top_mod for subname in comps[1:]: mod = getattr(mod, subname) - elif suffix == '.zip': + elif suffix == u'.zip': mod = zipimporter(fpath).load_module(name) else: desc = self.suffix_map[suffix] # if it is a directory, we don't open a file try: - mod_namespace = '.'.join(( + mod_namespace = u'.'.join(( self.loaded_base_name, self.mod_type_check(fpath), self.tag, name)) except TypeError: - mod_namespace = '{0}.{1}.{2}.{3}'.format( + mod_namespace = u'{0}.{1}.{2}.{3}'.format( self.loaded_base_name, self.mod_type_check(fpath), self.tag, name) - if suffix == '': - mod = imp.load_module(mod_namespace, None, fpath, desc) + if suffix == u'': + if USE_IMPORTLIB: + # pylint: disable=no-member + # Package directory, look for __init__ + loader_details = [ + (importlib.machinery.SourceFileLoader, importlib.machinery.SOURCE_SUFFIXES), + (importlib.machinery.SourcelessFileLoader, importlib.machinery.BYTECODE_SUFFIXES), + (importlib.machinery.ExtensionFileLoader, importlib.machinery.EXTENSION_SUFFIXES), + ] + file_finder = importlib.machinery.FileFinder( + fpath_dirname, + *loader_details + ) + spec = file_finder.find_spec(mod_namespace) + if spec is None: + raise ImportError() + # TODO: Get rid of load_module in favor of + # exec_module below. load_module is deprecated, but + # loading using exec_module has been causing odd things + # with the magic dunders we pack into the loaded + # modules, most notably with salt-ssh's __opts__. + mod = spec.loader.load_module() + # mod = importlib.util.module_from_spec(spec) + # spec.loader.exec_module(mod) + # pylint: enable=no-member + sys.modules[mod_namespace] = mod + else: + mod = imp.load_module(mod_namespace, None, fpath, desc) # reload all submodules if necessary if not self.initial_load: self._reload_submodules(mod) else: - with salt.utils.fopen(fpath, desc[1]) as fn_: - mod = imp.load_module(mod_namespace, fn_, fpath, desc) - + if USE_IMPORTLIB: + # pylint: disable=no-member + loader = MODULE_KIND_MAP[desc[2]](mod_namespace, fpath) + spec = importlib.util.spec_from_file_location( + mod_namespace, fpath, loader=loader + ) + if spec is None: + raise ImportError() + # TODO: Get rid of load_module in favor of + # exec_module below. load_module is deprecated, but + # loading using exec_module has been causing odd things + # with the magic dunders we pack into the loaded + # modules, most notably with salt-ssh's __opts__. + mod = spec.loader.load_module() + #mod = importlib.util.module_from_spec(spec) + #spec.loader.exec_module(mod) + # pylint: enable=no-member + sys.modules[mod_namespace] = mod + else: + with salt.utils.files.fopen(fpath, desc[1]) as fn_: + mod = imp.load_module(mod_namespace, fn_, fpath, desc) except IOError: raise except ImportError as exc: - if 'magic number' in str(exc): - error_msg = 'Failed to import {0} {1}. Bad magic number. If migrating from Python2 to Python3, remove all .pyc files and try again.'.format(self.tag, name) + if u'magic number' in str(exc): + error_msg = u'Failed to import {0} {1}. Bad magic number. If migrating from Python2 to Python3, remove all .pyc files and try again.'.format(self.tag, name) log.warning(error_msg) self.missing_modules[name] = error_msg log.debug( - 'Failed to import {0} {1}:\n'.format( - self.tag, name - ), - exc_info=True + u'Failed to import %s %s:\n', + self.tag, name, exc_info=True ) self.missing_modules[name] = exc return False except Exception as error: log.error( - 'Failed to import {0} {1}, this is due most likely to a ' - 'syntax error:\n'.format( - self.tag, name - ), - exc_info=True + u'Failed to import %s %s, this is due most likely to a ' + u'syntax error:\n', self.tag, name, exc_info=True ) self.missing_modules[name] = error return False except SystemExit as error: log.error( - 'Failed to import {0} {1} as the module called exit()\n'.format( - self.tag, name - ), - exc_info=True + u'Failed to import %s %s as the module called exit()\n', + self.tag, name, exc_info=True ) self.missing_modules[name] = error return False finally: sys.path.remove(fpath_dirname) - if hasattr(mod, '__opts__'): + if hasattr(mod, u'__opts__'): mod.__opts__.update(self.opts) else: mod.__opts__ = self.opts @@ -1389,19 +1469,19 @@ class LazyLoader(salt.utils.lazy.LazyDict): for p_name, p_value in six.iteritems(self.pack): setattr(mod, p_name, p_value) - module_name = mod.__name__.rsplit('.', 1)[-1] + module_name = mod.__name__.rsplit(u'.', 1)[-1] # Call a module's initialization method if it exists - module_init = getattr(mod, '__init__', None) + module_init = getattr(mod, u'__init__', None) if inspect.isfunction(module_init): try: module_init(self.opts) except TypeError as e: log.error(e) except Exception: - err_string = '__init__ failed' + err_string = u'__init__ failed' log.debug( - 'Error loading %s.%s: %s', + u'Error loading %s.%s: %s', self.tag, module_name, err_string, exc_info=True ) self.missing_modules[module_name] = err_string @@ -1411,13 +1491,13 @@ class LazyLoader(salt.utils.lazy.LazyDict): # if virtual modules are enabled, we need to look for the # __virtual__() function inside that module and run it. if self.virtual_enable: - virtual_funcs_to_process = ['__virtual__'] + self.virtual_funcs + virtual_funcs_to_process = [u'__virtual__'] + self.virtual_funcs for virtual_func in virtual_funcs_to_process: virtual_ret, module_name, virtual_err, virtual_aliases = \ - self.process_virtual(mod, module_name) + self.process_virtual(mod, module_name, virtual_func) if virtual_err is not None: log.trace( - 'Error loading %s.%s: %s', + u'Error loading %s.%s: %s', self.tag, module_name, virtual_err ) @@ -1436,22 +1516,20 @@ class LazyLoader(salt.utils.lazy.LazyDict): # containing the names of the proxy types that the module supports. # # Render modules and state modules are OK though - if 'proxy' in self.opts: - if self.tag in ['grains', 'proxy']: - if not hasattr(mod, '__proxyenabled__') or \ - (self.opts['proxy']['proxytype'] not in mod.__proxyenabled__ and - '*' not in mod.__proxyenabled__): - err_string = 'not a proxy_minion enabled module' + if u'proxy' in self.opts: + if self.tag in [u'grains', u'proxy']: + if not hasattr(mod, u'__proxyenabled__') or \ + (self.opts[u'proxy'][u'proxytype'] not in mod.__proxyenabled__ and + u'*' not in mod.__proxyenabled__): + err_string = u'not a proxy_minion enabled module' self.missing_modules[module_name] = err_string self.missing_modules[name] = err_string return False - if getattr(mod, '__load__', False) is not False: + if getattr(mod, u'__load__', False) is not False: log.info( - 'The functions from module \'{0}\' are being loaded from the ' - 'provided __load__ attribute'.format( - module_name - ) + u'The functions from module \'%s\' are being loaded from the ' + u'provided __load__ attribute', module_name ) # If we had another module by the same virtual name, we should put any @@ -1462,8 +1540,8 @@ class LazyLoader(salt.utils.lazy.LazyDict): for x in mod_names )) - for attr in getattr(mod, '__load__', dir(mod)): - if attr.startswith('_'): + for attr in getattr(mod, u'__load__', dir(mod)): + if attr.startswith(u'_'): # private functions are skipped continue func = getattr(mod, attr) @@ -1477,12 +1555,12 @@ class LazyLoader(salt.utils.lazy.LazyDict): # # It default's of course to the found callable attribute name # if no alias is defined. - funcname = getattr(mod, '__func_alias__', {}).get(attr, attr) + funcname = getattr(mod, u'__func_alias__', {}).get(attr, attr) for tgt_mod in mod_names: try: - full_funcname = '.'.join((tgt_mod, funcname)) + full_funcname = u'.'.join((tgt_mod, funcname)) except TypeError: - full_funcname = '{0}.{1}'.format(tgt_mod, funcname) + full_funcname = u'{0}.{1}'.format(tgt_mod, funcname) # Save many references for lookups # Careful not to overwrite existing (higher priority) functions if full_funcname not in self._dict: @@ -1496,8 +1574,10 @@ class LazyLoader(salt.utils.lazy.LazyDict): try: Depends.enforce_dependencies(self._dict, self.tag) except RuntimeError as exc: - log.info('Depends.enforce_dependencies() failed ' - 'for reasons: {0}'.format(exc)) + log.info( + u'Depends.enforce_dependencies() failed for the following ' + u'reason: %s', exc + ) for tgt_mod in mod_names: self.loaded_modules[tgt_mod] = mod_dict[tgt_mod] @@ -1508,9 +1588,9 @@ class LazyLoader(salt.utils.lazy.LazyDict): Load a single item if you have it ''' # if the key doesn't have a '.' then it isn't valid for this mod dict - if not isinstance(key, six.string_types) or '.' not in key: + if not isinstance(key, six.string_types) or u'.' not in key: raise KeyError - mod_name, _ = key.split('.', 1) + mod_name, _ = key.split(u'.', 1) if mod_name in self.missing_modules: return True # if the modulename isn't in the whitelist, don't bother @@ -1566,12 +1646,12 @@ class LazyLoader(salt.utils.lazy.LazyDict): ''' Apply the __outputter__ variable to the functions ''' - if hasattr(mod, '__outputter__'): + if hasattr(mod, u'__outputter__'): outp = mod.__outputter__ if func.__name__ in outp: func.__outputter__ = outp[func.__name__] - def process_virtual(self, mod, module_name, virtual_func='__virtual__'): + def process_virtual(self, mod, module_name, virtual_func=u'__virtual__'): ''' Given a loaded module and its default name determine its virtual name @@ -1597,30 +1677,30 @@ class LazyLoader(salt.utils.lazy.LazyDict): # namespace collisions. And finally it allows modules to return False # if they are not intended to run on the given platform or are missing # dependencies. - virtual_aliases = getattr(mod, '__virtual_aliases__', tuple()) + virtual_aliases = getattr(mod, u'__virtual_aliases__', tuple()) try: error_reason = None - if hasattr(mod, '__virtual__') and inspect.isfunction(mod.__virtual__): + if hasattr(mod, u'__virtual__') and inspect.isfunction(mod.__virtual__): try: start = time.time() virtual = getattr(mod, virtual_func)() if isinstance(virtual, tuple): error_reason = virtual[1] virtual = virtual[0] - if self.opts.get('virtual_timer', False): + if self.opts.get(u'virtual_timer', False): end = time.time() - start - msg = 'Virtual function took {0} seconds for {1}'.format( + msg = u'Virtual function took {0} seconds for {1}'.format( end, module_name) log.warning(msg) except Exception as exc: error_reason = ( - 'Exception raised when processing __virtual__ function' - ' for {0}. Module will not be loaded: {1}'.format( + u'Exception raised when processing __virtual__ function' + u' for {0}. Module will not be loaded: {1}'.format( mod.__name__, exc)) log.error(error_reason, exc_info_on_loglevel=logging.DEBUG) virtual = None # Get the module's virtual name - virtualname = getattr(mod, '__virtualname__', virtual) + virtualname = getattr(mod, u'__virtualname__', virtual) if not virtual: # if __virtual__() evaluates to False then the module # wasn't meant for this platform or it's not supposed to @@ -1630,13 +1710,10 @@ class LazyLoader(salt.utils.lazy.LazyDict): # improperly loaded if virtual is None: log.warning( - '{0}.__virtual__() is wrongly returning `None`. ' - 'It should either return `True`, `False` or a new ' - 'name. If you\'re the developer of the module ' - '\'{1}\', please fix this.'.format( - mod.__name__, - module_name - ) + u'%s.__virtual__() is wrongly returning `None`. ' + u'It should either return `True`, `False` or a new ' + u'name. If you\'re the developer of the module ' + u'\'%s\', please fix this.', mod.__name__, module_name ) return (False, module_name, error_reason, virtual_aliases) @@ -1647,18 +1724,16 @@ class LazyLoader(salt.utils.lazy.LazyDict): if virtual is not True and module_name != virtual: # The module is renaming itself. Updating the module name # with the new name - log.trace('Loaded {0} as virtual {1}'.format( - module_name, virtual - )) + log.trace(u'Loaded %s as virtual %s', module_name, virtual) - if not hasattr(mod, '__virtualname__'): - salt.utils.warn_until( - 'Hydrogen', - 'The \'{0}\' module is renaming itself in its ' - '__virtual__() function ({1} => {2}). Please ' - 'set it\'s virtual name as the ' - '\'__virtualname__\' module attribute. ' - 'Example: "__virtualname__ = \'{2}\'"'.format( + if not hasattr(mod, u'__virtualname__'): + salt.utils.versions.warn_until( + u'Hydrogen', + u'The \'{0}\' module is renaming itself in its ' + u'__virtual__() function ({1} => {2}). Please ' + u'set it\'s virtual name as the ' + u'\'__virtualname__\' module attribute. ' + u'Example: "__virtualname__ = \'{2}\'"'.format( mod.__name__, module_name, virtual @@ -1670,14 +1745,11 @@ class LazyLoader(salt.utils.lazy.LazyDict): # being returned by the __virtual__() function. This # should be considered an error. log.error( - 'The module \'{0}\' is showing some bad usage. Its ' - '__virtualname__ attribute is set to \'{1}\' yet the ' - '__virtual__() function is returning \'{2}\'. These ' - 'values should match!'.format( - mod.__name__, - virtualname, - virtual - ) + u'The module \'%s\' is showing some bad usage. Its ' + u'__virtualname__ attribute is set to \'%s\' yet the ' + u'__virtual__() function is returning \'%s\'. These ' + u'values should match!', + mod.__name__, virtualname, virtual ) module_name = virtualname @@ -1693,20 +1765,14 @@ class LazyLoader(salt.utils.lazy.LazyDict): # in incomplete grains sets, these can be safely ignored # and logged to debug, still, it includes the traceback to # help debugging. - log.debug( - 'KeyError when loading {0}'.format(module_name), - exc_info=True - ) + log.debug(u'KeyError when loading %s', module_name, exc_info=True) except Exception: # If the module throws an exception during __virtual__() # then log the information and continue to the next. log.error( - 'Failed to read the virtual function for ' - '{0}: {1}'.format( - self.tag, module_name - ), - exc_info=True + u'Failed to read the virtual function for %s: %s', + self.tag, module_name, exc_info=True ) return (False, module_name, error_reason, virtual_aliases) diff --git a/salt/log/handlers/__init__.py b/salt/log/handlers/__init__.py index ceb8f502320..f0341a7c117 100644 --- a/salt/log/handlers/__init__.py +++ b/salt/log/handlers/__init__.py @@ -117,7 +117,7 @@ class RotatingFileHandler(ExcInfoOnLogLevelFormatMixIn, logging.handlers.Rotatin ''' handled = False - # Can't use "salt.utils.is_windows()" in this file + # Can't use "salt.utils.platform.is_windows()" in this file if (sys.platform.startswith('win') and logging.raiseExceptions and sys.stderr): # see Python issue 13807 diff --git a/salt/log/handlers/fluent_mod.py b/salt/log/handlers/fluent_mod.py index ccd56b5521c..88b3e2cfdb3 100644 --- a/salt/log/handlers/fluent_mod.py +++ b/salt/log/handlers/fluent_mod.py @@ -90,7 +90,7 @@ from salt.log.mixins import NewStyleClassMixIn import salt.utils.network # Import Third party libs -import salt.ext.six as six +from salt.ext import six log = logging.getLogger(__name__) diff --git a/salt/log/handlers/log4mongo_mod.py b/salt/log/handlers/log4mongo_mod.py index 9408ef18617..122ec3bfa6d 100644 --- a/salt/log/handlers/log4mongo_mod.py +++ b/salt/log/handlers/log4mongo_mod.py @@ -40,7 +40,7 @@ import socket import logging # Import salt libs -import salt.ext.six as six +from salt.ext import six from salt.log.mixins import NewStyleClassMixIn from salt.log.setup import LOG_LEVELS diff --git a/salt/log/handlers/logstash_mod.py b/salt/log/handlers/logstash_mod.py index 7e2b3e929e6..a00435b5be7 100644 --- a/salt/log/handlers/logstash_mod.py +++ b/salt/log/handlers/logstash_mod.py @@ -169,7 +169,7 @@ from salt.log.mixins import NewStyleClassMixIn import salt.utils.network # Import Third party libs -import salt.ext.six as six +from salt.ext import six try: import zmq except ImportError: diff --git a/salt/log/setup.py b/salt/log/setup.py index c15c2bc3d5c..73361391ae6 100644 --- a/salt/log/setup.py +++ b/salt/log/setup.py @@ -27,7 +27,7 @@ import traceback import multiprocessing # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves.urllib.parse import urlparse # pylint: disable=import-error,no-name-in-module # Let's define these custom logging levels before importing the salt.log.mixins @@ -38,6 +38,7 @@ GARBAGE = logging.GARBAGE = 1 QUIET = logging.QUIET = 1000 # Import salt libs +import salt.utils # Can be removed once appendproctitle is moved from salt.textformat import TextFormat from salt.log.handlers import (TemporaryLoggingHandler, StreamHandler, @@ -837,7 +838,7 @@ def setup_multiprocessing_logging(queue=None): This code should be called from within a running multiprocessing process instance. ''' - from salt.utils import is_windows + from salt.utils.platform import is_windows global __MP_LOGGING_CONFIGURED global __MP_LOGGING_QUEUE_HANDLER @@ -997,7 +998,6 @@ def patch_python_logging_handlers(): def __process_multiprocessing_logging_queue(opts, queue): - import salt.utils salt.utils.appendproctitle('MultiprocessingLoggingQueue') # Assign UID/GID of user to proc if set @@ -1006,7 +1006,8 @@ def __process_multiprocessing_logging_queue(opts, queue): if user: check_user(user) - if salt.utils.is_windows(): + from salt.utils.platform import is_windows + if is_windows(): # On Windows, creating a new process doesn't fork (copy the parent # process image). Due to this, we need to setup all of our logging # inside this process. diff --git a/salt/master.py b/salt/master.py index 159bf5b1339..19e11002c20 100644 --- a/salt/master.py +++ b/salt/master.py @@ -26,7 +26,7 @@ except ImportError: # Fall back to pycrypto from Crypto.PublicKey import RSA # pylint: disable=import-error,no-name-in-module,redefined-builtin -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import range # pylint: enable=import-error,no-name-in-module,redefined-builtin @@ -34,7 +34,7 @@ try: import zmq import zmq.eventloop.ioloop # support pyzmq 13.0.x, TODO: remove once we force people to 14.0.x - if not hasattr(zmq.eventloop.ioloop, 'ZMQIOLoop'): + if not hasattr(zmq.eventloop.ioloop, u'ZMQIOLoop'): zmq.eventloop.ioloop.ZMQIOLoop = zmq.eventloop.ioloop.IOLoop LOOP_CLASS = zmq.eventloop.ioloop.ZMQIOLoop HAS_ZMQ = True @@ -67,13 +67,18 @@ import salt.log.setup import salt.utils.args import salt.utils.atomicfile import salt.utils.event -import salt.utils.job -import salt.utils.verify -import salt.utils.minions +import salt.utils.files +import salt.utils.gitfs import salt.utils.gzip_util -import salt.utils.process -import salt.utils.zeromq import salt.utils.jid +import salt.utils.job +import salt.utils.master +import salt.utils.minions +import salt.utils.platform +import salt.utils.process +import salt.utils.schedule +import salt.utils.verify +import salt.utils.zeromq from salt.defaults import DEFAULT_TARGET_DELIM from salt.exceptions import FileserverConfigError from salt.transport import iter_transport_opts @@ -81,8 +86,6 @@ from salt.utils.debug import ( enable_sigusr1_handler, enable_sigusr2_handler, inspect_stack ) from salt.utils.event import tagify -from salt.utils.master import ConnectedCache -from salt.utils.process import default_signals, SignalHandlingMultiprocessingProcess try: import resource @@ -124,16 +127,16 @@ class SMaster(object): # These methods are only used when pickling so will not be used on # non-Windows platforms. def __setstate__(self, state): - self.opts = state['opts'] - self.master_key = state['master_key'] - self.key = state['key'] - SMaster.secrets = state['secrets'] + self.opts = state[u'opts'] + self.master_key = state[u'master_key'] + self.key = state[u'key'] + SMaster.secrets = state[u'secrets'] def __getstate__(self): - return {'opts': self.opts, - 'master_key': self.master_key, - 'key': self.key, - 'secrets': SMaster.secrets} + return {u'opts': self.opts, + u'master_key': self.master_key, + u'key': self.key, + u'secrets': SMaster.secrets} def __prep_key(self): ''' @@ -143,7 +146,7 @@ class SMaster(object): return salt.daemons.masterapi.access_keys(self.opts) -class Maintenance(SignalHandlingMultiprocessingProcess): +class Maintenance(salt.utils.process.SignalHandlingMultiprocessingProcess): ''' A generalized maintenance process which performs maintenance routines. ''' @@ -156,7 +159,7 @@ class Maintenance(SignalHandlingMultiprocessingProcess): super(Maintenance, self).__init__(log_queue=log_queue) self.opts = opts # How often do we perform the maintenance tasks - self.loop_interval = int(self.opts['loop_interval']) + self.loop_interval = int(self.opts[u'loop_interval']) # Track key rotation intervals self.rotate = int(time.time()) # A serializer for general maint operations @@ -167,11 +170,11 @@ class Maintenance(SignalHandlingMultiprocessingProcess): # process so that a register_after_fork() equivalent will work on Windows. def __setstate__(self, state): self._is_child = True - self.__init__(state['opts'], log_queue=state['log_queue']) + self.__init__(state[u'opts'], log_queue=state[u'log_queue']) def __getstate__(self): - return {'opts': self.opts, - 'log_queue': self.log_queue} + return {u'opts': self.opts, + u'log_queue': self.log_queue} def _post_fork_init(self): ''' @@ -184,7 +187,7 @@ class Maintenance(SignalHandlingMultiprocessingProcess): self.fileserver = salt.fileserver.Fileserver(self.opts) # Load Runners ropts = dict(self.opts) - ropts['quiet'] = True + ropts[u'quiet'] = True runner_client = salt.runner.RunnerClient(ropts) # Load Returners self.returners = salt.loader.returners(self.opts, {}) @@ -195,15 +198,15 @@ class Maintenance(SignalHandlingMultiprocessingProcess): returners=self.returners) self.ckminions = salt.utils.minions.CkMinions(self.opts) # Make Event bus for firing - self.event = salt.utils.event.get_master_event(self.opts, self.opts['sock_dir'], listen=False) + self.event = salt.utils.event.get_master_event(self.opts, self.opts[u'sock_dir'], listen=False) # Init any values needed by the git ext pillar self.git_pillar = salt.daemons.masterapi.init_git_pillar(self.opts) self.presence_events = False - if self.opts.get('presence_events', False): + if self.opts.get(u'presence_events', False): tcp_only = True for transport, _ in iter_transport_opts(self.opts): - if transport != 'tcp': + if transport != u'tcp': tcp_only = False if not tcp_only: # For a TCP only transport, the presence events will be @@ -218,7 +221,7 @@ class Maintenance(SignalHandlingMultiprocessingProcess): This is where any data that needs to be cleanly maintained from the master is maintained. ''' - salt.utils.appendproctitle('Maintenance') + salt.utils.appendproctitle(u'Maintenance') # init things that need to be done after the process is forked self._post_fork_init() @@ -250,20 +253,20 @@ class Maintenance(SignalHandlingMultiprocessingProcess): Evaluate accepted keys and create a msgpack file which contains a list ''' - if self.opts['key_cache'] == 'sched': + if self.opts[u'key_cache'] == u'sched': keys = [] #TODO DRY from CKMinions - if self.opts['transport'] in ('zeromq', 'tcp'): - acc = 'minions' + if self.opts[u'transport'] in (u'zeromq', u'tcp'): + acc = u'minions' else: - acc = 'accepted' + acc = u'accepted' - for fn_ in os.listdir(os.path.join(self.opts['pki_dir'], acc)): - if not fn_.startswith('.') and os.path.isfile(os.path.join(self.opts['pki_dir'], acc, fn_)): + for fn_ in os.listdir(os.path.join(self.opts[u'pki_dir'], acc)): + if not fn_.startswith(u'.') and os.path.isfile(os.path.join(self.opts[u'pki_dir'], acc, fn_)): keys.append(fn_) - log.debug('Writing master key cache') + log.debug(u'Writing master key cache') # Write a temporary file securely - with salt.utils.atomicfile.atomic_open(os.path.join(self.opts['pki_dir'], acc, '.key_cache')) as cache_file: + with salt.utils.atomicfile.atomic_open(os.path.join(self.opts[u'pki_dir'], acc, u'.key_cache')) as cache_file: self.serial.dump(keys, cache_file) def handle_key_rotate(self, now): @@ -271,39 +274,39 @@ class Maintenance(SignalHandlingMultiprocessingProcess): Rotate the AES key rotation ''' to_rotate = False - dfn = os.path.join(self.opts['cachedir'], '.dfn') + dfn = os.path.join(self.opts[u'cachedir'], u'.dfn') try: stats = os.stat(dfn) # Basic Windows permissions don't distinguish between # user/group/all. Check for read-only state instead. - if salt.utils.is_windows() and not os.access(dfn, os.W_OK): + if salt.utils.platform.is_windows() and not os.access(dfn, os.W_OK): to_rotate = True # Cannot delete read-only files on Windows. os.chmod(dfn, stat.S_IRUSR | stat.S_IWUSR) elif stats.st_mode == 0o100400: to_rotate = True else: - log.error('Found dropfile with incorrect permissions, ignoring...') + log.error(u'Found dropfile with incorrect permissions, ignoring...') os.remove(dfn) except os.error: pass - if self.opts.get('publish_session'): - if now - self.rotate >= self.opts['publish_session']: + if self.opts.get(u'publish_session'): + if now - self.rotate >= self.opts[u'publish_session']: to_rotate = True if to_rotate: - log.info('Rotating master AES key') + log.info(u'Rotating master AES key') for secret_key, secret_map in six.iteritems(SMaster.secrets): # should be unnecessary-- since no one else should be modifying - with secret_map['secret'].get_lock(): - secret_map['secret'].value = six.b(secret_map['reload']()) - self.event.fire_event({'rotate_{0}_key'.format(secret_key): True}, tag='key') + with secret_map[u'secret'].get_lock(): + secret_map[u'secret'].value = six.b(secret_map[u'reload']()) + self.event.fire_event({u'rotate_{0}_key'.format(secret_key): True}, tag=u'key') self.rotate = now - if self.opts.get('ping_on_rotate'): + if self.opts.get(u'ping_on_rotate'): # Ping all minions to get them to pick up the new key - log.debug('Pinging all connected minions ' - 'due to key rotation') + log.debug(u'Pinging all connected minions ' + u'due to key rotation') salt.utils.master.ping_all_connected_minions(self.opts) def handle_git_pillar(self): @@ -312,9 +315,9 @@ class Maintenance(SignalHandlingMultiprocessingProcess): ''' try: for pillar in self.git_pillar: - pillar.update() + pillar.fetch_remotes() except Exception as exc: - log.error('Exception caught while updating git_pillar', + log.error(u'Exception caught while updating git_pillar', exc_info=True) def handle_schedule(self): @@ -328,9 +331,7 @@ class Maintenance(SignalHandlingMultiprocessingProcess): if self.schedule.loop_interval < self.loop_interval: self.loop_interval = self.schedule.loop_interval except Exception as exc: - log.error( - 'Exception {0} occurred in scheduled job'.format(exc) - ) + log.error(u'Exception %s occurred in scheduled job', exc) def handle_presence(self, old_present): ''' @@ -342,13 +343,13 @@ class Maintenance(SignalHandlingMultiprocessingProcess): lost = old_present.difference(present) if new or lost: # Fire new minions present event - data = {'new': list(new), - 'lost': list(lost)} - self.event.fire_event(data, tagify('change', 'presence')) - data = {'present': list(present)} + data = {u'new': list(new), + u'lost': list(lost)} + self.event.fire_event(data, tagify(u'change', u'presence')) + data = {u'present': list(present)} # On the first run it may need more time for the EventPublisher # to come up and be ready. Set the timeout to account for this. - self.event.fire_event(data, tagify('present', 'presence'), timeout=3) + self.event.fire_event(data, tagify(u'present', u'presence'), timeout=3) old_present.clear() old_present.update(present) @@ -371,14 +372,14 @@ class Master(SMaster): # PyZMQ <= 2.1.9 does not have zmq_version_info, fall back to # using zmq.zmq_version() and build a version info tuple. zmq_version_info = tuple( - [int(x) for x in zmq.zmq_version().split('.')] + [int(x) for x in zmq.zmq_version().split(u'.')] ) if zmq_version_info < (3, 2): log.warning( - 'You have a version of ZMQ less than ZMQ 3.2! There are ' - 'known connection keep-alive issues with ZMQ < 3.2 which ' - 'may result in loss of contact with minions. Please ' - 'upgrade your ZMQ!' + u'You have a version of ZMQ less than ZMQ 3.2! There are ' + u'known connection keep-alive issues with ZMQ < 3.2 which ' + u'may result in loss of contact with minions. Please ' + u'upgrade your ZMQ!' ) SMaster.__init__(self, opts) @@ -392,43 +393,39 @@ class Master(SMaster): # hard limit,but raising to anything above soft limit fails... mof_h = mof_s log.info( - 'Current values for max open files soft/hard setting: ' - '{0}/{1}'.format( - mof_s, mof_h - ) + u'Current values for max open files soft/hard setting: %s/%s', + mof_s, mof_h ) # Let's grab, from the configuration file, the value to raise max open # files to - mof_c = self.opts['max_open_files'] + mof_c = self.opts[u'max_open_files'] if mof_c > mof_h: # The configured value is higher than what's allowed log.info( - 'The value for the \'max_open_files\' setting, {0}, is higher ' - 'than what the user running salt is allowed to raise to, {1}. ' - 'Defaulting to {1}.'.format(mof_c, mof_h) + u'The value for the \'max_open_files\' setting, %s, is higher ' + u'than the highest value the user running salt is allowed to ' + u'set (%s). Defaulting to %s.', mof_c, mof_h, mof_h ) mof_c = mof_h if mof_s < mof_c: # There's room to raise the value. Raise it! - log.info('Raising max open files value to {0}'.format(mof_c)) + log.info(u'Raising max open files value to %s', mof_c) resource.setrlimit(resource.RLIMIT_NOFILE, (mof_c, mof_h)) try: mof_s, mof_h = resource.getrlimit(resource.RLIMIT_NOFILE) log.info( - 'New values for max open files soft/hard values: ' - '{0}/{1}'.format(mof_s, mof_h) + u'New values for max open files soft/hard values: %s/%s', + mof_s, mof_h ) except ValueError: # https://github.com/saltstack/salt/issues/1991#issuecomment-13025595 # A user under macOS reported that our 100000 default value is # still too high. log.critical( - 'Failed to raise max open files setting to {0}. If this ' - 'value is too low. The salt-master will most likely fail ' - 'to run properly.'.format( - mof_c - ) + u'Failed to raise max open files setting to %s. If this ' + u'value is too low, the salt-master will most likely fail ' + u'to run properly.', mof_c ) def _pre_flight(self): @@ -440,18 +437,18 @@ class Master(SMaster): critical_errors = [] try: - os.chdir('/') + os.chdir(u'/') except OSError as err: errors.append( - 'Cannot change to root directory ({0})'.format(err) + u'Cannot change to root directory ({0})'.format(err) ) - if self.opts.get('fileserver_verify_config', True): + if self.opts.get(u'fileserver_verify_config', True): fileserver = salt.fileserver.Fileserver(self.opts) if not fileserver.servers: errors.append( - 'Failed to load fileserver backends, the configured backends ' - 'are: {0}'.format(', '.join(self.opts['fileserver_backend'])) + u'Failed to load fileserver backends, the configured backends ' + u'are: {0}'.format(u', '.join(self.opts[u'fileserver_backend'])) ) else: # Run init() for all backends which support the function, to @@ -459,37 +456,37 @@ class Master(SMaster): try: fileserver.init() except FileserverConfigError as exc: - critical_errors.append('{0}'.format(exc)) + critical_errors.append(u'{0}'.format(exc)) - if not self.opts['fileserver_backend']: - errors.append('No fileserver backends are configured') + if not self.opts[u'fileserver_backend']: + errors.append(u'No fileserver backends are configured') # Check to see if we need to create a pillar cache dir - if self.opts['pillar_cache'] and not os.path.isdir(os.path.join(self.opts['cachedir'], 'pillar_cache')): + if self.opts[u'pillar_cache'] and not os.path.isdir(os.path.join(self.opts[u'cachedir'], u'pillar_cache')): try: prev_umask = os.umask(0o077) - os.mkdir(os.path.join(self.opts['cachedir'], 'pillar_cache')) + os.mkdir(os.path.join(self.opts[u'cachedir'], u'pillar_cache')) os.umask(prev_umask) except OSError: pass - if self.opts.get('git_pillar_verify_config', True): - non_legacy_git_pillars = [ - x for x in self.opts.get('ext_pillar', []) - if 'git' in x - and not isinstance(x['git'], six.string_types) + if self.opts.get(u'git_pillar_verify_config', True): + git_pillars = [ + x for x in self.opts.get(u'ext_pillar', []) + if u'git' in x + and not isinstance(x[u'git'], six.string_types) ] - if non_legacy_git_pillars: + if git_pillars: try: new_opts = copy.deepcopy(self.opts) from salt.pillar.git_pillar \ import PER_REMOTE_OVERRIDES as per_remote_overrides, \ PER_REMOTE_ONLY as per_remote_only - for repo in non_legacy_git_pillars: - new_opts['ext_pillar'] = [repo] + for repo in git_pillars: + new_opts[u'ext_pillar'] = [repo] try: git_pillar = salt.utils.gitfs.GitPillar(new_opts) - git_pillar.init_remotes(repo['git'], + git_pillar.init_remotes(repo[u'git'], per_remote_overrides, per_remote_only) except FileserverConfigError as exc: @@ -502,7 +499,7 @@ class Master(SMaster): log.error(error) for error in critical_errors: log.critical(error) - log.critical('Master failed pre flight checks, exiting\n') + log.critical(u'Master failed pre flight checks, exiting\n') sys.exit(salt.defaults.exitcodes.EX_GENERIC) def start(self): @@ -510,11 +507,7 @@ class Master(SMaster): Turn on the master server components ''' self._pre_flight() - log.info( - 'salt-master is starting as user \'{0}\''.format( - salt.utils.get_user() - ) - ) + log.info(u'salt-master is starting as user \'%s\'', salt.utils.get_user()) enable_sigusr1_handler() enable_sigusr2_handler() @@ -524,87 +517,89 @@ class Master(SMaster): # Reset signals to default ones before adding processes to the process # manager. We don't want the processes being started to inherit those # signal handlers - with default_signals(signal.SIGINT, signal.SIGTERM): + with salt.utils.process.default_signals(signal.SIGINT, signal.SIGTERM): # Setup the secrets here because the PubServerChannel may need # them as well. - SMaster.secrets['aes'] = {'secret': multiprocessing.Array(ctypes.c_char, - six.b(salt.crypt.Crypticle.generate_key_string())), - 'reload': salt.crypt.Crypticle.generate_key_string - } - log.info('Creating master process manager') + SMaster.secrets[u'aes'] = { + u'secret': multiprocessing.Array( + ctypes.c_char, + six.b(salt.crypt.Crypticle.generate_key_string()) + ), + u'reload': salt.crypt.Crypticle.generate_key_string + } + log.info(u'Creating master process manager') # Since there are children having their own ProcessManager we should wait for kill more time. self.process_manager = salt.utils.process.ProcessManager(wait_for_kill=5) pub_channels = [] - log.info('Creating master publisher process') + log.info(u'Creating master publisher process') for transport, opts in iter_transport_opts(self.opts): chan = salt.transport.server.PubServerChannel.factory(opts) chan.pre_fork(self.process_manager) pub_channels.append(chan) - log.info('Creating master event publisher process') + log.info(u'Creating master event publisher process') self.process_manager.add_process(salt.utils.event.EventPublisher, args=(self.opts,)) - if self.opts.get('reactor'): - if isinstance(self.opts['engines'], list): + if self.opts.get(u'reactor'): + if isinstance(self.opts[u'engines'], list): rine = False - for item in self.opts['engines']: - if 'reactor' in item: + for item in self.opts[u'engines']: + if u'reactor' in item: rine = True break if not rine: - self.opts['engines'].append({'reactor': {}}) + self.opts[u'engines'].append({u'reactor': {}}) else: - if 'reactor' not in self.opts['engines']: - log.info('Enabling the reactor engine') - self.opts['engines']['reactor'] = {} + if u'reactor' not in self.opts[u'engines']: + log.info(u'Enabling the reactor engine') + self.opts[u'engines'][u'reactor'] = {} salt.engines.start_engines(self.opts, self.process_manager) # must be after channels - log.info('Creating master maintenance process') + log.info(u'Creating master maintenance process') self.process_manager.add_process(Maintenance, args=(self.opts,)) - if self.opts.get('event_return'): - log.info('Creating master event return process') + if self.opts.get(u'event_return'): + log.info(u'Creating master event return process') self.process_manager.add_process(salt.utils.event.EventReturn, args=(self.opts,)) - ext_procs = self.opts.get('ext_processes', []) + ext_procs = self.opts.get(u'ext_processes', []) for proc in ext_procs: - log.info('Creating ext_processes process: {0}'.format(proc)) + log.info(u'Creating ext_processes process: %s', proc) try: - mod = '.'.join(proc.split('.')[:-1]) - cls = proc.split('.')[-1] + mod = u'.'.join(proc.split(u'.')[:-1]) + cls = proc.split(u'.')[-1] _tmp = __import__(mod, globals(), locals(), [cls], -1) cls = _tmp.__getattribute__(cls) self.process_manager.add_process(cls, args=(self.opts,)) except Exception: - log.error(('Error creating ext_processes ' - 'process: {0}').format(proc)) + log.error(u'Error creating ext_processes process: %s', proc) - if HAS_HALITE and 'halite' in self.opts: - log.info('Creating master halite process') - self.process_manager.add_process(Halite, args=(self.opts['halite'],)) + if HAS_HALITE and u'halite' in self.opts: + log.info(u'Creating master halite process') + self.process_manager.add_process(Halite, args=(self.opts[u'halite'],)) # TODO: remove, or at least push into the transport stuff (pre-fork probably makes sense there) - if self.opts['con_cache']: - log.info('Creating master concache process') - self.process_manager.add_process(ConnectedCache, args=(self.opts,)) + if self.opts[u'con_cache']: + log.info(u'Creating master concache process') + self.process_manager.add_process(salt.utils.master.ConnectedCache, args=(self.opts,)) # workaround for issue #16315, race condition - log.debug('Sleeping for two seconds to let concache rest') + log.debug(u'Sleeping for two seconds to let concache rest') time.sleep(2) - log.info('Creating master request server process') + log.info(u'Creating master request server process') kwargs = {} - if salt.utils.is_windows(): - kwargs['log_queue'] = salt.log.setup.get_multiprocessing_logging_queue() - kwargs['secrets'] = SMaster.secrets + if salt.utils.platform.is_windows(): + kwargs[u'log_queue'] = salt.log.setup.get_multiprocessing_logging_queue() + kwargs[u'secrets'] = SMaster.secrets self.process_manager.add_process( ReqServer, args=(self.opts, self.key, self.master_key), kwargs=kwargs, - name='ReqServer') + name=u'ReqServer') # Install the SIGINT/SIGTERM handlers if not done so far if signal.getsignal(signal.SIGINT) is signal.SIG_DFL: @@ -627,7 +622,7 @@ class Master(SMaster): sys.exit(0) -class Halite(SignalHandlingMultiprocessingProcess): +class Halite(salt.utils.process.SignalHandlingMultiprocessingProcess): ''' Manage the Halite server ''' @@ -645,11 +640,11 @@ class Halite(SignalHandlingMultiprocessingProcess): # process so that a register_after_fork() equivalent will work on Windows. def __setstate__(self, state): self._is_child = True - self.__init__(state['hopts'], log_queue=state['log_queue']) + self.__init__(state[u'hopts'], log_queue=state[u'log_queue']) def __getstate__(self): - return {'hopts': self.hopts, - 'log_queue': self.log_queue} + return {u'hopts': self.hopts, + u'log_queue': self.log_queue} def run(self): ''' @@ -659,7 +654,7 @@ class Halite(SignalHandlingMultiprocessingProcess): halite.start(self.hopts) -class ReqServer(SignalHandlingMultiprocessingProcess): +class ReqServer(salt.utils.process.SignalHandlingMultiprocessingProcess): ''' Starts up the master request server, minions send results to this interface. @@ -687,15 +682,15 @@ class ReqServer(SignalHandlingMultiprocessingProcess): # process so that a register_after_fork() equivalent will work on Windows. def __setstate__(self, state): self._is_child = True - self.__init__(state['opts'], state['key'], state['mkey'], - log_queue=state['log_queue'], secrets=state['secrets']) + self.__init__(state[u'opts'], state[u'key'], state[u'mkey'], + log_queue=state[u'log_queue'], secrets=state[u'secrets']) def __getstate__(self): - return {'opts': self.opts, - 'key': self.key, - 'mkey': self.master_key, - 'log_queue': self.log_queue, - 'secrets': self.secrets} + return {u'opts': self.opts, + u'key': self.key, + u'mkey': self.master_key, + u'log_queue': self.log_queue, + u'secrets': self.secrets} def _handle_signals(self, signum, sigframe): # pylint: disable=unused-argument self.destroy(signum) @@ -711,10 +706,10 @@ class ReqServer(SignalHandlingMultiprocessingProcess): if self.secrets is not None: SMaster.secrets = self.secrets - dfn = os.path.join(self.opts['cachedir'], '.dfn') + dfn = os.path.join(self.opts[u'cachedir'], u'.dfn') if os.path.isfile(dfn): try: - if salt.utils.is_windows() and not os.access(dfn, os.W_OK): + if salt.utils.platform.is_windows() and not os.access(dfn, os.W_OK): # Cannot delete read-only files on Windows. os.chmod(dfn, stat.S_IRUSR | stat.S_IWUSR) os.remove(dfn) @@ -722,7 +717,7 @@ class ReqServer(SignalHandlingMultiprocessingProcess): pass # Wait for kill should be less then parent's ProcessManager. - self.process_manager = salt.utils.process.ProcessManager(name='ReqServer_ProcessManager', + self.process_manager = salt.utils.process.ProcessManager(name=u'ReqServer_ProcessManager', wait_for_kill=1) req_channels = [] @@ -731,26 +726,26 @@ class ReqServer(SignalHandlingMultiprocessingProcess): chan = salt.transport.server.ReqServerChannel.factory(opts) chan.pre_fork(self.process_manager) req_channels.append(chan) - if transport != 'tcp': + if transport != u'tcp': tcp_only = False kwargs = {} - if salt.utils.is_windows(): - kwargs['log_queue'] = self.log_queue + if salt.utils.platform.is_windows(): + kwargs[u'log_queue'] = self.log_queue # Use one worker thread if only the TCP transport is set up on # Windows and we are using Python 2. There is load balancer # support on Windows for the TCP transport when using Python 3. - if tcp_only and six.PY2 and int(self.opts['worker_threads']) != 1: - log.warning('TCP transport supports only 1 worker on Windows ' - 'when using Python 2.') - self.opts['worker_threads'] = 1 + if tcp_only and six.PY2 and int(self.opts[u'worker_threads']) != 1: + log.warning(u'TCP transport supports only 1 worker on Windows ' + u'when using Python 2.') + self.opts[u'worker_threads'] = 1 # Reset signals to default ones before adding processes to the process # manager. We don't want the processes being started to inherit those # signal handlers - with default_signals(signal.SIGINT, signal.SIGTERM): - for ind in range(int(self.opts['worker_threads'])): - name = 'MWorker-{0}'.format(ind) + with salt.utils.process.default_signals(signal.SIGINT, signal.SIGTERM): + for ind in range(int(self.opts[u'worker_threads'])): + name = u'MWorker-{0}'.format(ind) self.process_manager.add_process(MWorker, args=(self.opts, self.master_key, @@ -768,7 +763,7 @@ class ReqServer(SignalHandlingMultiprocessingProcess): self.__bind() def destroy(self, signum=signal.SIGTERM): - if hasattr(self, 'process_manager'): + if hasattr(self, u'process_manager'): self.process_manager.stop_restarting() self.process_manager.send_signal_to_processes(signum) self.process_manager.kill_children() @@ -777,7 +772,7 @@ class ReqServer(SignalHandlingMultiprocessingProcess): self.destroy() -class MWorker(SignalHandlingMultiprocessingProcess): +class MWorker(salt.utils.process.SignalHandlingMultiprocessingProcess): ''' The worker multiprocess instance to manage the backend operations for the salt master. @@ -799,8 +794,8 @@ class MWorker(SignalHandlingMultiprocessingProcess): :rtype: MWorker :return: Master worker ''' - kwargs['name'] = name - SignalHandlingMultiprocessingProcess.__init__(self, **kwargs) + kwargs[u'name'] = name + super(MWorker, self).__init__(**kwargs) self.opts = opts self.req_channels = req_channels @@ -815,25 +810,25 @@ class MWorker(SignalHandlingMultiprocessingProcess): # non-Windows platforms. def __setstate__(self, state): self._is_child = True - SignalHandlingMultiprocessingProcess.__init__(self, log_queue=state['log_queue']) - self.opts = state['opts'] - self.req_channels = state['req_channels'] - self.mkey = state['mkey'] - self.key = state['key'] - self.k_mtime = state['k_mtime'] - SMaster.secrets = state['secrets'] + super(MWorker, self).__init__(log_queue=state[u'log_queue']) + self.opts = state[u'opts'] + self.req_channels = state[u'req_channels'] + self.mkey = state[u'mkey'] + self.key = state[u'key'] + self.k_mtime = state[u'k_mtime'] + SMaster.secrets = state[u'secrets'] def __getstate__(self): - return {'opts': self.opts, - 'req_channels': self.req_channels, - 'mkey': self.mkey, - 'key': self.key, - 'k_mtime': self.k_mtime, - 'log_queue': self.log_queue, - 'secrets': SMaster.secrets} + return {u'opts': self.opts, + u'req_channels': self.req_channels, + u'mkey': self.mkey, + u'key': self.key, + u'k_mtime': self.k_mtime, + u'log_queue': self.log_queue, + u'secrets': SMaster.secrets} def _handle_signals(self, signum, sigframe): - for channel in getattr(self, 'req_channels', ()): + for channel in getattr(self, u'req_channels', ()): channel.close() super(MWorker, self)._handle_signals(signum, sigframe) @@ -876,10 +871,10 @@ class MWorker(SignalHandlingMultiprocessingProcess): :param dict payload: The payload route to the appropriate handler ''' - key = payload['enc'] - load = payload['load'] - ret = {'aes': self._handle_aes, - 'clear': self._handle_clear}[key](load) + key = payload[u'enc'] + load = payload[u'load'] + ret = {u'aes': self._handle_aes, + u'clear': self._handle_clear}[key](load) raise tornado.gen.Return(ret) def _handle_clear(self, load): @@ -890,10 +885,10 @@ class MWorker(SignalHandlingMultiprocessingProcess): :return: The result of passing the load to a function in ClearFuncs corresponding to the command specified in the load's 'cmd' key. ''' - log.trace('Clear payload received with command {cmd}'.format(**load)) - if load['cmd'].startswith('__'): + log.trace(u'Clear payload received with command %s', load[u'cmd']) + if load[u'cmd'].startswith(u'__'): return False - return getattr(self.clear_funcs, load['cmd'])(load), {'fun': 'send_clear'} + return getattr(self.clear_funcs, load[u'cmd'])(load), {u'fun': u'send_clear'} def _handle_aes(self, data): ''' @@ -903,13 +898,13 @@ class MWorker(SignalHandlingMultiprocessingProcess): :return: The result of passing the load to a function in AESFuncs corresponding to the command specified in the load's 'cmd' key. ''' - if 'cmd' not in data: - log.error('Received malformed command {0}'.format(data)) + if u'cmd' not in data: + log.error(u'Received malformed command %s', data) return {} - log.trace('AES payload received with command {0}'.format(data['cmd'])) - if data['cmd'].startswith('__'): + log.trace(u'AES payload received with command %s', data[u'cmd']) + if data[u'cmd'].startswith(u'__'): return False - return self.aes_funcs.run_func(data['cmd'], data) + return self.aes_funcs.run_func(data[u'cmd'], data) def run(self): ''' @@ -942,11 +937,11 @@ class AESFuncs(object): :returns: Instance for handling AES operations ''' self.opts = opts - self.event = salt.utils.event.get_master_event(self.opts, self.opts['sock_dir'], listen=False) + self.event = salt.utils.event.get_master_event(self.opts, self.opts[u'sock_dir'], listen=False) self.serial = salt.payload.Serial(opts) self.ckminions = salt.utils.minions.CkMinions(opts) # Make a client - self.local = salt.client.get_local_client(self.opts['conf_file']) + self.local = salt.client.get_local_client(self.opts[u'conf_file']) # Create the master minion to access the external job cache self.mminion = salt.minion.MasterMinion( self.opts, @@ -985,29 +980,31 @@ class AESFuncs(object): ''' if not salt.utils.verify.valid_id(self.opts, id_): return False - pub_path = os.path.join(self.opts['pki_dir'], 'minions', id_) + pub_path = os.path.join(self.opts[u'pki_dir'], u'minions', id_) try: - with salt.utils.fopen(pub_path, 'r') as fp_: + with salt.utils.files.fopen(pub_path, u'r') as fp_: minion_pub = fp_.read() pub = RSA.importKey(minion_pub) except (IOError, OSError): - log.warning('Salt minion claiming to be {0} attempted to communicate ' - 'with master but key could not be read and verification was ' - 'denied.'.format(id_)) + log.warning( + u'Salt minion claiming to be %s attempted to communicate with ' + u'master, but key could not be read and verification was denied.', + id_ + ) return False except (ValueError, IndexError, TypeError) as err: - log.error('Unable to load public key "{0}": {1}' - .format(pub_path, err)) + log.error(u'Unable to load public key "%s": %s', pub_path, err) try: - if salt.crypt.public_decrypt(pub, token) == b'salt': + if salt.crypt.public_decrypt(pub, token) == b'salt': # future lint: disable=non-unicode-string return True except ValueError as err: - log.error('Unable to decrypt token: {0}'.format(err)) + log.error(u'Unable to decrypt token: %s', err) - log.error('Salt minion claiming to be {0} has attempted to ' - 'communicate with the master and could not be verified' - .format(id_)) + log.error( + u'Salt minion claiming to be %s has attempted to communicate with ' + u'the master and could not be verified', id_ + ) return False def verify_minion(self, id_, token): @@ -1033,48 +1030,46 @@ class AESFuncs(object): :return: A boolean indicating if the minion is allowed to publish the command in the load ''' # Verify that the load is valid - if 'peer' not in self.opts: + if u'peer' not in self.opts: return False - if not isinstance(self.opts['peer'], dict): + if not isinstance(self.opts[u'peer'], dict): return False - if any(key not in clear_load for key in ('fun', 'arg', 'tgt', 'ret', 'tok', 'id')): + if any(key not in clear_load for key in (u'fun', u'arg', u'tgt', u'ret', u'tok', u'id')): return False # If the command will make a recursive publish don't run - if clear_load['fun'].startswith('publish.'): + if clear_load[u'fun'].startswith(u'publish.'): return False # Check the permissions for this minion - if not self.__verify_minion(clear_load['id'], clear_load['tok']): + if not self.__verify_minion(clear_load[u'id'], clear_load[u'tok']): # The minion is not who it says it is! # We don't want to listen to it! log.warning( - ( - 'Minion id {0} is not who it says it is and is attempting ' - 'to issue a peer command' - ).format(clear_load['id']) + u'Minion id %s is not who it says it is and is attempting ' + u'to issue a peer command', clear_load[u'id'] ) return False - clear_load.pop('tok') + clear_load.pop(u'tok') perms = [] - for match in self.opts['peer']: - if re.match(match, clear_load['id']): + for match in self.opts[u'peer']: + if re.match(match, clear_load[u'id']): # This is the list of funcs/modules! - if isinstance(self.opts['peer'][match], list): - perms.extend(self.opts['peer'][match]) - if ',' in clear_load['fun']: + if isinstance(self.opts[u'peer'][match], list): + perms.extend(self.opts[u'peer'][match]) + if u',' in clear_load[u'fun']: # 'arg': [['cat', '/proc/cpuinfo'], [], ['foo']] - clear_load['fun'] = clear_load['fun'].split(',') + clear_load[u'fun'] = clear_load[u'fun'].split(u',') arg_ = [] - for arg in clear_load['arg']: + for arg in clear_load[u'arg']: arg_.append(arg.split()) - clear_load['arg'] = arg_ + clear_load[u'arg'] = arg_ # finally, check the auth of the load return self.ckminions.auth_check( perms, - clear_load['fun'], - clear_load['arg'], - clear_load['tgt'], - clear_load.get('tgt_type', 'glob'), + clear_load[u'fun'], + clear_load[u'arg'], + clear_load[u'tgt'], + clear_load.get(u'tgt_type', u'glob'), publish_validate=True) def __verify_load(self, load, verify_keys): @@ -1092,27 +1087,20 @@ class AESFuncs(object): ''' if any(key not in load for key in verify_keys): return False - if 'tok' not in load: + if u'tok' not in load: log.error( - 'Received incomplete call from {0} for \'{1}\', missing \'{2}\'' - .format( - load['id'], - inspect_stack()['co_name'], - 'tok' - )) - return False - if not self.__verify_minion(load['id'], load['tok']): - # The minion is not who it says it is! - # We don't want to listen to it! - log.warning( - 'Minion id {0} is not who it says it is!'.format( - load['id'] - ) + u'Received incomplete call from %s for \'%s\', missing \'%s\'', + load[u'id'], inspect_stack()[u'co_name'], u'tok' ) return False + if not self.__verify_minion(load[u'id'], load[u'tok']): + # The minion is not who it says it is! + # We don't want to listen to it! + log.warning(u'Minion id %s is not who it says it is!', load[u'id']) + return False - if 'tok' in load: - load.pop('tok') + if u'tok' in load: + load.pop(u'tok') return load @@ -1124,7 +1112,7 @@ class AESFuncs(object): :param dict load: A payload received from a minion :return: The results from an external node classifier ''' - load = self.__verify_load(load, ('id', 'tok')) + load = self.__verify_load(load, (u'id', u'tok')) if load is False: return {} return self.masterapi._master_tops(load, skip_verify=True) @@ -1144,22 +1132,22 @@ class AESFuncs(object): for saltenv in envs: if saltenv not in file_roots: file_roots[saltenv] = [] - mopts['file_roots'] = file_roots - mopts['top_file_merging_strategy'] = self.opts['top_file_merging_strategy'] - mopts['env_order'] = self.opts['env_order'] - mopts['default_top'] = self.opts['default_top'] - if load.get('env_only'): + mopts[u'file_roots'] = file_roots + mopts[u'top_file_merging_strategy'] = self.opts[u'top_file_merging_strategy'] + mopts[u'env_order'] = self.opts[u'env_order'] + mopts[u'default_top'] = self.opts[u'default_top'] + if load.get(u'env_only'): return mopts - mopts['renderer'] = self.opts['renderer'] - mopts['failhard'] = self.opts['failhard'] - mopts['state_top'] = self.opts['state_top'] - mopts['state_top_saltenv'] = self.opts['state_top_saltenv'] - mopts['nodegroups'] = self.opts['nodegroups'] - mopts['state_auto_order'] = self.opts['state_auto_order'] - mopts['state_events'] = self.opts['state_events'] - mopts['state_aggregate'] = self.opts['state_aggregate'] - mopts['jinja_lstrip_blocks'] = self.opts['jinja_lstrip_blocks'] - mopts['jinja_trim_blocks'] = self.opts['jinja_trim_blocks'] + mopts[u'renderer'] = self.opts[u'renderer'] + mopts[u'failhard'] = self.opts[u'failhard'] + mopts[u'state_top'] = self.opts[u'state_top'] + mopts[u'state_top_saltenv'] = self.opts[u'state_top_saltenv'] + mopts[u'nodegroups'] = self.opts[u'nodegroups'] + mopts[u'state_auto_order'] = self.opts[u'state_auto_order'] + mopts[u'state_events'] = self.opts[u'state_events'] + mopts[u'state_aggregate'] = self.opts[u'state_aggregate'] + mopts[u'jinja_lstrip_blocks'] = self.opts[u'jinja_lstrip_blocks'] + mopts[u'jinja_trim_blocks'] = self.opts[u'jinja_trim_blocks'] return mopts def _mine_get(self, load): @@ -1171,7 +1159,7 @@ class AESFuncs(object): :rtype: dict :return: Mine data from the specified minions ''' - load = self.__verify_load(load, ('id', 'tgt', 'fun', 'tok')) + load = self.__verify_load(load, (u'id', u'tgt', u'fun', u'tok')) if load is False: return {} else: @@ -1186,7 +1174,7 @@ class AESFuncs(object): :rtype: bool :return: True if the data has been stored in the mine ''' - load = self.__verify_load(load, ('id', 'data', 'tok')) + load = self.__verify_load(load, (u'id', u'data', u'tok')) if load is False: return {} return self.masterapi._mine(load, skip_verify=True) @@ -1200,7 +1188,7 @@ class AESFuncs(object): :rtype: bool :return: Boolean indicating whether or not the given function was deleted from the mine ''' - load = self.__verify_load(load, ('id', 'fun', 'tok')) + load = self.__verify_load(load, (u'id', u'fun', u'tok')) if load is False: return {} else: @@ -1212,7 +1200,7 @@ class AESFuncs(object): :param dict load: A payload received from a minion ''' - load = self.__verify_load(load, ('id', 'tok')) + load = self.__verify_load(load, (u'id', u'tok')) if load is False: return {} else: @@ -1223,51 +1211,43 @@ class AESFuncs(object): Allows minions to send files to the master, files are sent to the master file cache ''' - if any(key not in load for key in ('id', 'path', 'loc')): + if any(key not in load for key in (u'id', u'path', u'loc')): return False - if not isinstance(load['path'], list): + if not isinstance(load[u'path'], list): return False - if not self.opts['file_recv']: + if not self.opts[u'file_recv']: return False - if not salt.utils.verify.valid_id(self.opts, load['id']): + if not salt.utils.verify.valid_id(self.opts, load[u'id']): return False - file_recv_max_size = 1024*1024 * self.opts['file_recv_max_size'] + file_recv_max_size = 1024*1024 * self.opts[u'file_recv_max_size'] - if 'loc' in load and load['loc'] < 0: - log.error('Invalid file pointer: load[loc] < 0') + if u'loc' in load and load[u'loc'] < 0: + log.error(u'Invalid file pointer: load[loc] < 0') return False - if len(load['data']) + load.get('loc', 0) > file_recv_max_size: + if len(load[u'data']) + load.get(u'loc', 0) > file_recv_max_size: log.error( - 'file_recv_max_size limit of %d MB exceeded! %s will be ' - 'truncated. To successfully push this file, adjust ' - 'file_recv_max_size to an integer (in MB) large enough to ' - 'accommodate it.', file_recv_max_size, load['path'] + u'file_recv_max_size limit of %d MB exceeded! %s will be ' + u'truncated. To successfully push this file, adjust ' + u'file_recv_max_size to an integer (in MB) large enough to ' + u'accommodate it.', file_recv_max_size, load[u'path'] ) return False - if 'tok' not in load: + if u'tok' not in load: log.error( - 'Received incomplete call from {0} for \'{1}\', missing ' - '\'{2}\''.format( - load['id'], - inspect_stack()['co_name'], - 'tok' - ) + u'Received incomplete call from %s for \'%s\', missing \'%s\'', + load[u'id'], inspect_stack()[u'co_name'], u'tok' ) return False - if not self.__verify_minion(load['id'], load['tok']): + if not self.__verify_minion(load[u'id'], load[u'tok']): # The minion is not who it says it is! # We don't want to listen to it! - log.warning( - 'Minion id {0} is not who it says it is!'.format( - load['id'] - ) - ) + log.warning(u'Minion id %s is not who it says it is!', load[u'id']) return {} - load.pop('tok') + load.pop(u'tok') # Join path - sep_path = os.sep.join(load['path']) + sep_path = os.sep.join(load[u'path']) # Path normalization should have been done by the sending # minion but we can't guarantee it. Re-do it here. @@ -1275,20 +1255,22 @@ class AESFuncs(object): # Ensure that this safety check is done after the path # have been normalized. - if os.path.isabs(normpath) or '../' in load['path']: + if os.path.isabs(normpath) or u'../' in load[u'path']: # Can overwrite master files!! return False cpath = os.path.join( - self.opts['cachedir'], - 'minions', - load['id'], - 'files', + self.opts[u'cachedir'], + u'minions', + load[u'id'], + u'files', normpath) # One last safety check here - if not os.path.normpath(cpath).startswith(self.opts['cachedir']): - log.warning('Attempt to write received file outside of master cache ' - 'directory! Requested file write: {0}. Access denied.'.format(cpath)) + if not os.path.normpath(cpath).startswith(self.opts[u'cachedir']): + log.warning( + u'Attempt to write received file outside of master cache ' + u'directory! Requested path: %s. Access denied.', cpath + ) return False cdir = os.path.dirname(cpath) if not os.path.isdir(cdir): @@ -1296,17 +1278,15 @@ class AESFuncs(object): os.makedirs(cdir) except os.error: pass - if os.path.isfile(cpath) and load['loc'] != 0: - mode = 'ab' + if os.path.isfile(cpath) and load[u'loc'] != 0: + mode = u'ab' else: - mode = 'wb' - with salt.utils.fopen(cpath, mode) as fp_: - if load['loc']: - fp_.seek(load['loc']) - if six.PY3: - fp_.write(load['data'].encode(__salt_system_encoding__)) - else: - fp_.write(load['data']) + mode = u'wb' + with salt.utils.files.fopen(cpath, mode) as fp_: + if load[u'loc']: + fp_.seek(load[u'loc']) + + fp_.write(load[u'data']) return True def _pillar(self, load): @@ -1318,29 +1298,29 @@ class AESFuncs(object): :rtype: dict :return: The pillar data for the minion ''' - if any(key not in load for key in ('id', 'grains')): + if any(key not in load for key in (u'id', u'grains')): return False - if not salt.utils.verify.valid_id(self.opts, load['id']): + if not salt.utils.verify.valid_id(self.opts, load[u'id']): return False - load['grains']['id'] = load['id'] + load[u'grains'][u'id'] = load[u'id'] - pillar_dirs = {} pillar = salt.pillar.get_pillar( self.opts, - load['grains'], - load['id'], - load.get('saltenv', load.get('env')), - ext=load.get('ext'), - pillar=load.get('pillar_override', {}), - pillarenv=load.get('pillarenv')) - data = pillar.compile_pillar(pillar_dirs=pillar_dirs) + load[u'grains'], + load[u'id'], + load.get(u'saltenv', load.get(u'env')), + ext=load.get(u'ext'), + pillar_override=load.get(u'pillar_override', {}), + pillarenv=load.get(u'pillarenv'), + extra_minion_data=load.get(u'extra_minion_data')) + data = pillar.compile_pillar() self.fs_.update_opts() - if self.opts.get('minion_data_cache', False): - self.masterapi.cache.store('minions/{0}'.format(load['id']), - 'data', - {'grains': load['grains'], - 'pillar': data}) - self.event.fire_event({'Minion data cache refresh': load['id']}, tagify(load['id'], 'refresh', 'minion')) + if self.opts.get(u'minion_data_cache', False): + self.masterapi.cache.store(u'minions/{0}'.format(load[u'id']), + u'data', + {u'grains': load[u'grains'], + u'pillar': data}) + self.event.fire_event({u'Minion data cache refresh': load[u'id']}, tagify(load[u'id'], u'refresh', u'minion')) return data def _minion_event(self, load): @@ -1350,7 +1330,7 @@ class AESFuncs(object): :param dict load: The minion payload ''' - load = self.__verify_load(load, ('id', 'tok')) + load = self.__verify_load(load, (u'id', u'tok')) if load is False: return {} # Route to master event bus @@ -1362,20 +1342,20 @@ class AESFuncs(object): ''' Act on specific events from minions ''' - id_ = load['id'] - if load.get('tag', '') == '_salt_error': + id_ = load[u'id'] + if load.get(u'tag', u'') == u'_salt_error': log.error( - 'Received minion error from [{minion}]: {data}' - .format(minion=id_, data=load['data']['message']) + u'Received minion error from [%s]: %s', + id_, load[u'data'][u'message'] ) - for event in load.get('events', []): - event_data = event.get('data', {}) - if 'minions' in event_data: - jid = event_data.get('jid') + for event in load.get(u'events', []): + event_data = event.get(u'data', {}) + if u'minions' in event_data: + jid = event_data.get(u'jid') if not jid: continue - minions = event_data['minions'] + minions = event_data[u'minions'] try: salt.utils.job.store_minions( self.opts, @@ -1385,8 +1365,8 @@ class AESFuncs(object): syndic_id=id_) except (KeyError, salt.exceptions.SaltCacheError) as exc: log.error( - 'Could not add minion(s) {0} for job {1}: {2}' - .format(minions, jid, exc) + u'Could not add minion(s) %s for job %s: %s', + minions, jid, exc ) def _return(self, load): @@ -1399,29 +1379,36 @@ class AESFuncs(object): :param dict load: The minion payload ''' - if self.opts['require_minion_sign_messages'] and 'sig' not in load: - log.critical('_return: Master is requiring minions to sign their messages, but there is no signature in this payload from {0}.'.format(load['id'])) + if self.opts[u'require_minion_sign_messages'] and u'sig' not in load: + log.critical( + u'_return: Master is requiring minions to sign their ' + u'messages, but there is no signature in this payload from ' + u'%s.', load[u'id'] + ) return False - if 'sig' in load: - log.trace('Verifying signed event publish from minion') - sig = load.pop('sig') - this_minion_pubkey = os.path.join(self.opts['pki_dir'], 'minions/{0}'.format(load['id'])) + if u'sig' in load: + log.trace(u'Verifying signed event publish from minion') + sig = load.pop(u'sig') + this_minion_pubkey = os.path.join(self.opts[u'pki_dir'], u'minions/{0}'.format(load[u'id'])) serialized_load = salt.serializers.msgpack.serialize(load) if not salt.crypt.verify_signature(this_minion_pubkey, serialized_load, sig): - log.info('Failed to verify event signature from minion {0}.'.format(load['id'])) - if self.opts['drop_messages_signature_fail']: - log.critical('Drop_messages_signature_fail is enabled, dropping message from {0}'.format(load['id'])) + log.info(u'Failed to verify event signature from minion %s.', load[u'id']) + if self.opts[u'drop_messages_signature_fail']: + log.critical( + u'Drop_messages_signature_fail is enabled, dropping ' + u'message from %s', load[u'id'] + ) return False else: - log.info('But \'drop_message_signature_fail\' is disabled, so message is still accepted.') - load['sig'] = sig + log.info(u'But \'drop_message_signature_fail\' is disabled, so message is still accepted.') + load[u'sig'] = sig try: salt.utils.job.store_job( self.opts, load, event=self.event, mminion=self.mminion) except salt.exceptions.SaltCacheError: - log.error('Could not store job information for load: {0}'.format(load)) + log.error(u'Could not store job information for load: %s', load) def _syndic_return(self, load): ''' @@ -1431,37 +1418,37 @@ class AESFuncs(object): :param dict load: The minion payload ''' # Verify the load - if any(key not in load for key in ('return', 'jid', 'id')): + if any(key not in load for key in (u'return', u'jid', u'id')): return None # if we have a load, save it - if load.get('load'): - fstr = '{0}.save_load'.format(self.opts['master_job_cache']) - self.mminion.returners[fstr](load['jid'], load['load']) + if load.get(u'load'): + fstr = u'{0}.save_load'.format(self.opts[u'master_job_cache']) + self.mminion.returners[fstr](load[u'jid'], load[u'load']) # Register the syndic - syndic_cache_path = os.path.join(self.opts['cachedir'], 'syndics', load['id']) + syndic_cache_path = os.path.join(self.opts[u'cachedir'], u'syndics', load[u'id']) if not os.path.exists(syndic_cache_path): path_name = os.path.split(syndic_cache_path)[0] if not os.path.exists(path_name): os.makedirs(path_name) - with salt.utils.fopen(syndic_cache_path, 'w') as wfh: - wfh.write('') + with salt.utils.files.fopen(syndic_cache_path, u'w') as wfh: + wfh.write(u'') # Format individual return loads - for key, item in six.iteritems(load['return']): - ret = {'jid': load['jid'], - 'id': key} + for key, item in six.iteritems(load[u'return']): + ret = {u'jid': load[u'jid'], + u'id': key} ret.update(item) - if 'master_id' in load: - ret['master_id'] = load['master_id'] - if 'fun' in load: - ret['fun'] = load['fun'] - if 'arg' in load: - ret['fun_args'] = load['arg'] - if 'out' in load: - ret['out'] = load['out'] - if 'sig' in load: - ret['sig'] = load['sig'] + if u'master_id' in load: + ret[u'master_id'] = load[u'master_id'] + if u'fun' in load: + ret[u'fun'] = load[u'fun'] + if u'arg' in load: + ret[u'fun_args'] = load[u'arg'] + if u'out' in load: + ret[u'out'] = load[u'out'] + if u'sig' in load: + ret[u'sig'] = load[u'sig'] self._return(ret) @@ -1474,7 +1461,7 @@ class AESFuncs(object): :rtype: dict :return: The runner function data ''' - load = self.__verify_load(clear_load, ('fun', 'arg', 'id', 'tok')) + load = self.__verify_load(clear_load, (u'fun', u'arg', u'id', u'tok')) if load is False: return {} else: @@ -1490,21 +1477,21 @@ class AESFuncs(object): :rtype: dict :return: Return data corresponding to a given JID ''' - load = self.__verify_load(load, ('jid', 'id', 'tok')) + load = self.__verify_load(load, (u'jid', u'id', u'tok')) if load is False: return {} # Check that this minion can access this data auth_cache = os.path.join( - self.opts['cachedir'], - 'publish_auth') + self.opts[u'cachedir'], + u'publish_auth') if not os.path.isdir(auth_cache): os.makedirs(auth_cache) - jid_fn = os.path.join(auth_cache, str(load['jid'])) - with salt.utils.fopen(jid_fn, 'r') as fp_: - if not load['id'] == fp_.read(): + jid_fn = os.path.join(auth_cache, str(load[u'jid'])) + with salt.utils.files.fopen(jid_fn, u'r') as fp_: + if not load[u'id'] == fp_.read(): return {} # Grab the latest and return - return self.local.get_cache_returns(load['jid']) + return self.local.get_cache_returns(load[u'jid']) def minion_pub(self, clear_load): ''' @@ -1588,10 +1575,13 @@ class AESFuncs(object): :rtype: bool :return: True if key was revoked, False if not ''' - load = self.__verify_load(load, ('id', 'tok')) + load = self.__verify_load(load, (u'id', u'tok')) - if not self.opts.get('allow_minion_key_revoke', False): - log.warning('Minion {0} requested key revoke, but allow_minion_key_revoke is False'.format(load['id'])) + if not self.opts.get(u'allow_minion_key_revoke', False): + log.warning( + u'Minion %s requested key revoke, but allow_minion_key_revoke ' + u'is set to False', load[u'id'] + ) return load if load is False: @@ -1607,44 +1597,38 @@ class AESFuncs(object): :return: The result of the master function that was called ''' # Don't honor private functions - if func.startswith('__'): + if func.startswith(u'__'): # TODO: return some error? Seems odd to return {} - return {}, {'fun': 'send'} + return {}, {u'fun': u'send'} # Run the func if hasattr(self, func): try: start = time.time() ret = getattr(self, func)(load) log.trace( - 'Master function call {0} took {1} seconds'.format( - func, time.time() - start - ) + u'Master function call %s took %s seconds', + func, time.time() - start ) except Exception: - ret = '' - log.error( - 'Error in function {0}:\n'.format(func), - exc_info=True - ) + ret = u'' + log.error(u'Error in function %s:\n', func, exc_info=True) else: log.error( - 'Received function {0} which is unavailable on the master, ' - 'returning False'.format( - func - ) + u'Received function %s which is unavailable on the master, ' + u'returning False', func ) - return False, {'fun': 'send'} + return False, {u'fun': u'send'} # Don't encrypt the return value for the _return func # (we don't care about the return value, so why encrypt it?) - if func == '_return': - return ret, {'fun': 'send'} - if func == '_pillar' and 'id' in load: - if load.get('ver') != '2' and self.opts['pillar_version'] == 1: + if func == u'_return': + return ret, {u'fun': u'send'} + if func == u'_pillar' and u'id' in load: + if load.get(u'ver') != u'2' and self.opts[u'pillar_version'] == 1: # Authorized to return old pillar proto - return ret, {'fun': 'send'} - return ret, {'fun': 'send_private', 'key': 'pillar', 'tgt': load['id']} + return ret, {u'fun': u'send'} + return ret, {u'fun': u'send_private', u'key': u'pillar', u'tgt': load[u'id']} # Encrypt the return - return ret, {'fun': 'send'} + return ret, {u'fun': u'send'} class ClearFuncs(object): @@ -1660,9 +1644,9 @@ class ClearFuncs(object): self.opts = opts self.key = key # Create the event manager - self.event = salt.utils.event.get_master_event(self.opts, self.opts['sock_dir'], listen=False) + self.event = salt.utils.event.get_master_event(self.opts, self.opts[u'sock_dir'], listen=False) # Make a client - self.local = salt.client.get_local_client(self.opts['conf_file']) + self.local = salt.client.get_local_client(self.opts[u'conf_file']) # Make an minion checker object self.ckminions = salt.utils.minions.CkMinions(opts) # Make an Auth object @@ -1684,153 +1668,126 @@ class ClearFuncs(object): Send a master control function back to the runner system ''' # All runner ops pass through eauth - if 'token' in clear_load: - # Authenticate - token = self.loadauth.authenticate_token(clear_load) + auth_type, err_name, key, sensitive_load_keys = self._prep_auth_info(clear_load) - if not token: - return dict(error=dict(name='TokenAuthenticationError', - message='Authentication failure of type "token" occurred.')) + # Authenticate + auth_check = self.loadauth.check_authentication(clear_load, auth_type, key=key) + error = auth_check.get(u'error') - # Authorize - if self.opts['keep_acl_in_token'] and 'auth_list' in token: - auth_list = token['auth_list'] - else: - clear_load['eauth'] = token['eauth'] - clear_load['username'] = token['name'] - auth_list = self.loadauth.get_auth_list(clear_load) + if error: + # Authentication error occurred: do not continue. + return {u'error': error} - if not self.ckminions.runner_check(auth_list, clear_load['fun']): - return dict(error=dict(name='TokenAuthenticationError', - message=('Authentication failure of type "token" occurred for ' - 'user {0}.').format(token['name']))) - clear_load.pop('token') - username = token['name'] - elif 'eauth' in clear_load: - if not self.loadauth.authenticate_eauth(clear_load): - return dict(error=dict(name='EauthAuthenticationError', - message=('Authentication failure of type "eauth" occurred for ' - 'user {0}.').format(clear_load.get('username', 'UNKNOWN')))) + # Authorize + username = auth_check.get(u'username') + if auth_type != u'user': + runner_check = self.ckminions.runner_check( + auth_check.get(u'auth_list', []), + clear_load[u'fun'], + clear_load.get(u'kwarg', {}) + ) + if not runner_check: + return {u'error': {u'name': err_name, + u'message': u'Authentication failure of type "{0}" occurred for ' + u'user {1}.'.format(auth_type, username)}} + elif isinstance(runner_check, dict) and u'error' in runner_check: + # A dictionary with an error name/message was handled by ckminions.runner_check + return runner_check - auth_list = self.loadauth.get_auth_list(clear_load) - if not self.ckminions.runner_check(auth_list, clear_load['fun']): - return dict(error=dict(name='EauthAuthenticationError', - message=('Authentication failure of type "eauth" occurred for ' - 'user {0}.').format(clear_load.get('username', 'UNKNOWN')))) - - # No error occurred, consume the password from the clear_load if - # passed - username = clear_load.pop('username', 'UNKNOWN') - clear_load.pop('password', None) + # No error occurred, consume sensitive settings from the clear_load if passed. + for item in sensitive_load_keys: + clear_load.pop(item, None) else: - if not self.loadauth.authenticate_key(clear_load, self.key): - return dict(error=dict(name='UserAuthenticationError', - message='Authentication failure of type "user" occurred')) - - if 'user' in clear_load: - username = clear_load['user'] + if u'user' in clear_load: + username = clear_load[u'user'] if salt.auth.AuthUser(username).is_sudo(): - username = self.opts.get('user', 'root') + username = self.opts.get(u'user', u'root') else: username = salt.utils.get_user() # Authorized. Do the job! try: - fun = clear_load.pop('fun') + fun = clear_load.pop(u'fun') runner_client = salt.runner.RunnerClient(self.opts) return runner_client.async(fun, - clear_load.get('kwarg', {}), + clear_load.get(u'kwarg', {}), username) except Exception as exc: - log.error('Exception occurred while ' - 'introspecting {0}: {1}'.format(fun, exc)) - return dict(error=dict(name=exc.__class__.__name__, - args=exc.args, - message=str(exc))) + log.error(u'Exception occurred while introspecting %s: %s', fun, exc) + return {u'error': {u'name': exc.__class__.__name__, + u'args': exc.args, + u'message': str(exc)}} def wheel(self, clear_load): ''' Send a master control function back to the wheel system ''' # All wheel ops pass through eauth - username = None - if 'token' in clear_load: - # Authenticate - token = self.loadauth.authenticate_token(clear_load) - if not token: - return dict(error=dict(name='TokenAuthenticationError', - message='Authentication failure of type "token" occurred.')) + auth_type, err_name, key, sensitive_load_keys = self._prep_auth_info(clear_load) - # Authorize - if self.opts['keep_acl_in_token'] and 'auth_list' in token: - auth_list = token['auth_list'] - else: - clear_load['eauth'] = token['eauth'] - clear_load['username'] = token['name'] - auth_list = self.loadauth.get_auth_list(clear_load) - if not self.ckminions.wheel_check(auth_list, clear_load['fun']): - return dict(error=dict(name='TokenAuthenticationError', - message=('Authentication failure of type "token" occurred for ' - 'user {0}.').format(token['name']))) - clear_load.pop('token') - username = token['name'] - elif 'eauth' in clear_load: - if not self.loadauth.authenticate_eauth(clear_load): - return dict(error=dict(name='EauthAuthenticationError', - message=('Authentication failure of type "eauth" occurred for ' - 'user {0}.').format(clear_load.get('username', 'UNKNOWN')))) + # Authenticate + auth_check = self.loadauth.check_authentication(clear_load, auth_type, key=key) + error = auth_check.get(u'error') - auth_list = self.loadauth.get_auth_list(clear_load) - if not self.ckminions.wheel_check(auth_list, clear_load['fun']): - return dict(error=dict(name='EauthAuthenticationError', - message=('Authentication failure of type "eauth" occurred for ' - 'user {0}.').format(clear_load.get('username', 'UNKNOWN')))) + if error: + # Authentication error occurred: do not continue. + return {u'error': error} - # No error occurred, consume the password from the clear_load if - # passed - clear_load.pop('password', None) - username = clear_load.pop('username', 'UNKNOWN') + # Authorize + username = auth_check.get(u'username') + if auth_type != u'user': + wheel_check = self.ckminions.wheel_check( + auth_check.get(u'auth_list', []), + clear_load[u'fun'], + clear_load.get(u'kwarg', {}) + ) + if not wheel_check: + return {u'error': {u'name': err_name, + u'message': u'Authentication failure of type "{0}" occurred for ' + u'user {1}.'.format(auth_type, username)}} + elif isinstance(wheel_check, dict) and u'error' in wheel_check: + # A dictionary with an error name/message was handled by ckminions.wheel_check + return wheel_check + + # No error occurred, consume sensitive settings from the clear_load if passed. + for item in sensitive_load_keys: + clear_load.pop(item, None) else: - if not self.loadauth.authenticate_key(clear_load, self.key): - return dict(error=dict(name='UserAuthenticationError', - message='Authentication failure of type "user" occurred')) - - if 'user' in clear_load: - username = clear_load['user'] + if u'user' in clear_load: + username = clear_load[u'user'] if salt.auth.AuthUser(username).is_sudo(): - username = self.opts.get('user', 'root') + username = self.opts.get(u'user', u'root') else: username = salt.utils.get_user() # Authorized. Do the job! try: - jid = salt.utils.jid.gen_jid() - fun = clear_load.pop('fun') - tag = tagify(jid, prefix='wheel') - data = {'fun': "wheel.{0}".format(fun), - 'jid': jid, - 'tag': tag, - 'user': username} + jid = salt.utils.jid.gen_jid(self.opts) + fun = clear_load.pop(u'fun') + tag = tagify(jid, prefix=u'wheel') + data = {u'fun': u"wheel.{0}".format(fun), + u'jid': jid, + u'tag': tag, + u'user': username} - self.event.fire_event(data, tagify([jid, 'new'], 'wheel')) + self.event.fire_event(data, tagify([jid, u'new'], u'wheel')) ret = self.wheel_.call_func(fun, full_return=True, **clear_load) - data['return'] = ret['return'] - data['success'] = ret['success'] - self.event.fire_event(data, tagify([jid, 'ret'], 'wheel')) - return {'tag': tag, - 'data': data} + data[u'return'] = ret[u'return'] + data[u'success'] = ret[u'success'] + self.event.fire_event(data, tagify([jid, u'ret'], u'wheel')) + return {u'tag': tag, + u'data': data} except Exception as exc: - log.error('Exception occurred while ' - 'introspecting {0}: {1}'.format(fun, exc)) - data['return'] = 'Exception occurred in wheel {0}: {1}: {2}'.format( + log.error(u'Exception occurred while introspecting %s: %s', fun, exc) + data[u'return'] = u'Exception occurred in wheel {0}: {1}: {2}'.format( fun, exc.__class__.__name__, exc, ) - data['success'] = False - self.event.fire_event(data, tagify([jid, 'ret'], 'wheel')) - return {'tag': tag, - 'data': data} + data[u'success'] = False + self.event.fire_event(data, tagify([jid, u'ret'], u'wheel')) + return {u'tag': tag, + u'data': data} def mk_token(self, clear_load): ''' @@ -1839,81 +1796,75 @@ class ClearFuncs(object): ''' token = self.loadauth.mk_token(clear_load) if not token: - log.warning('Authentication failure of type "eauth" occurred.') - return '' + log.warning(u'Authentication failure of type "eauth" occurred.') + return u'' return token def get_token(self, clear_load): ''' Return the name associated with a token or False if the token is invalid ''' - if 'token' not in clear_load: + if u'token' not in clear_load: return False - return self.loadauth.get_tok(clear_load['token']) + return self.loadauth.get_tok(clear_load[u'token']) def publish(self, clear_load): ''' This method sends out publications to the minions, it can only be used by the LocalClient. ''' - extra = clear_load.get('kwargs', {}) + extra = clear_load.get(u'kwargs', {}) - publisher_acl = salt.acl.PublisherACL(self.opts['publisher_acl_blacklist']) + publisher_acl = salt.acl.PublisherACL(self.opts[u'publisher_acl_blacklist']) - if publisher_acl.user_is_blacklisted(clear_load['user']) or \ - publisher_acl.cmd_is_blacklisted(clear_load['fun']): + if publisher_acl.user_is_blacklisted(clear_load[u'user']) or \ + publisher_acl.cmd_is_blacklisted(clear_load[u'fun']): log.error( - '{user} does not have permissions to run {function}. Please ' - 'contact your local administrator if you believe this is in ' - 'error.\n'.format( - user=clear_load['user'], - function=clear_load['fun'] - ) + u'%s does not have permissions to run %s. Please contact ' + u'your local administrator if you believe this is in ' + u'error.\n', clear_load[u'user'], clear_load[u'fun'] ) - return '' + return u'' # Retrieve the minions list - delimiter = clear_load.get('kwargs', {}).get('delimiter', DEFAULT_TARGET_DELIM) - minions = self.ckminions.check_minions( - clear_load['tgt'], - clear_load.get('tgt_type', 'glob'), + delimiter = clear_load.get(u'kwargs', {}).get(u'delimiter', DEFAULT_TARGET_DELIM) + _res = self.ckminions.check_minions( + clear_load[u'tgt'], + clear_load.get(u'tgt_type', u'glob'), delimiter ) + minions = _res.get('minions', list()) + missing = _res.get('missing', list()) # Check for external auth calls - if extra.get('token', False): + if extra.get(u'token', False): # Authenticate. token = self.loadauth.authenticate_token(extra) if not token: - return '' + return u'' # Get acl - if self.opts['keep_acl_in_token'] and 'auth_list' in token: - auth_list = token['auth_list'] - else: - extra['eauth'] = token['eauth'] - extra['username'] = token['name'] - auth_list = self.loadauth.get_auth_list(extra) + auth_list = self.loadauth.get_auth_list(extra, token) # Authorize the request if not self.ckminions.auth_check( auth_list, - clear_load['fun'], - clear_load['arg'], - clear_load['tgt'], - clear_load.get('tgt_type', 'glob'), + clear_load[u'fun'], + clear_load[u'arg'], + clear_load[u'tgt'], + clear_load.get(u'tgt_type', u'glob'), minions=minions, # always accept find_job - whitelist=['saltutil.find_job'], + whitelist=[u'saltutil.find_job'], ): - log.warning('Authentication failure of type "token" occurred.') - return '' - clear_load['user'] = token['name'] - log.debug('Minion tokenized user = "{0}"'.format(clear_load['user'])) - elif 'eauth' in extra: + log.warning(u'Authentication failure of type "token" occurred.') + return u'' + clear_load[u'user'] = token[u'name'] + log.debug(u'Minion tokenized user = "%s"', clear_load[u'user']) + elif u'eauth' in extra: # Authenticate. if not self.loadauth.authenticate_eauth(extra): - return '' + return u'' # Get acl from eauth module. auth_list = self.loadauth.get_auth_list(extra) @@ -1921,95 +1872,111 @@ class ClearFuncs(object): # Authorize the request if not self.ckminions.auth_check( auth_list, - clear_load['fun'], - clear_load['arg'], - clear_load['tgt'], - clear_load.get('tgt_type', 'glob'), + clear_load[u'fun'], + clear_load[u'arg'], + clear_load[u'tgt'], + clear_load.get(u'tgt_type', u'glob'), minions=minions, # always accept find_job - whitelist=['saltutil.find_job'], + whitelist=[u'saltutil.find_job'], ): - log.warning('Authentication failure of type "eauth" occurred.') - return '' - clear_load['user'] = self.loadauth.load_name(extra) # The username we are attempting to auth with + log.warning(u'Authentication failure of type "eauth" occurred.') + return u'' + clear_load[u'user'] = self.loadauth.load_name(extra) # The username we are attempting to auth with # Verify that the caller has root on master else: auth_ret = self.loadauth.authenticate_key(clear_load, self.key) if auth_ret is False: - return '' + return u'' if auth_ret is not True: - if salt.auth.AuthUser(clear_load['user']).is_sudo(): - if not self.opts['sudo_acl'] or not self.opts['publisher_acl']: + if salt.auth.AuthUser(clear_load[u'user']).is_sudo(): + if not self.opts[u'sudo_acl'] or not self.opts[u'publisher_acl']: auth_ret = True if auth_ret is not True: auth_list = salt.utils.get_values_of_matching_keys( - self.opts['publisher_acl'], + self.opts[u'publisher_acl'], auth_ret) if not auth_list: log.warning( - 'Authentication failure of type "user" occurred.' + u'Authentication failure of type "user" occurred.' ) - return '' + return u'' if not self.ckminions.auth_check( auth_list, - clear_load['fun'], - clear_load['arg'], - clear_load['tgt'], - clear_load.get('tgt_type', 'glob'), + clear_load[u'fun'], + clear_load[u'arg'], + clear_load[u'tgt'], + clear_load.get(u'tgt_type', u'glob'), minions=minions, # always accept find_job - whitelist=['saltutil.find_job'], + whitelist=[u'saltutil.find_job'], ): - log.warning('Authentication failure of type "user" occurred.') - return '' + log.warning(u'Authentication failure of type "user" occurred.') + return u'' # If we order masters (via a syndic), don't short circuit if no minions # are found - if not self.opts.get('order_masters'): + if not self.opts.get(u'order_masters'): # Check for no minions if not minions: return { - 'enc': 'clear', - 'load': { - 'jid': None, - 'minions': minions, - 'error': 'Master could not resolve minions for target {0}'.format(clear_load['tgt']) + u'enc': u'clear', + u'load': { + u'jid': None, + u'minions': minions, + u'error': u'Master could not resolve minions for target {0}'.format(clear_load[u'tgt']) } } jid = self._prep_jid(clear_load, extra) if jid is None: - return {'enc': 'clear', - 'load': { - 'error': 'Master failed to assign jid', - } - } - payload = self._prep_pub(minions, jid, clear_load, extra) + return {u'enc': u'clear', + u'load': {u'error': u'Master failed to assign jid'}} + payload = self._prep_pub(minions, jid, clear_load, extra, missing) # Send it! self._send_pub(payload) return { - 'enc': 'clear', - 'load': { - 'jid': clear_load['jid'], - 'minions': minions + u'enc': u'clear', + u'load': { + u'jid': clear_load[u'jid'], + u'minions': minions, + u'missing': missing } } + def _prep_auth_info(self, clear_load): + sensitive_load_keys = [] + key = None + if u'token' in clear_load: + auth_type = u'token' + err_name = u'TokenAuthenticationError' + sensitive_load_keys = [u'token'] + elif u'eauth' in clear_load: + auth_type = u'eauth' + err_name = u'EauthAuthenticationError' + sensitive_load_keys = [u'username', u'password'] + else: + auth_type = u'user' + err_name = u'UserAuthenticationError' + key = self.key + + return auth_type, err_name, key, sensitive_load_keys + def _prep_jid(self, clear_load, extra): ''' Return a jid for this publication ''' # the jid in clear_load can be None, '', or something else. this is an # attempt to clean up the value before passing to plugins - passed_jid = clear_load['jid'] if clear_load.get('jid') else None - nocache = extra.get('nocache', False) + passed_jid = clear_load[u'jid'] if clear_load.get(u'jid') else None + nocache = extra.get(u'nocache', False) # Retrieve the jid - fstr = '{0}.prep_jid'.format(self.opts['master_job_cache']) + fstr = u'{0}.prep_jid'.format(self.opts[u'master_job_cache']) try: # Retrieve the jid jid = self.mminion.returners[fstr](nocache=nocache, @@ -2017,11 +1984,11 @@ class ClearFuncs(object): except (KeyError, TypeError): # The returner is not present msg = ( - 'Failed to allocate a jid. The requested returner \'{0}\' ' - 'could not be loaded.'.format(fstr.split('.')[0]) + u'Failed to allocate a jid. The requested returner \'{0}\' ' + u'could not be loaded.'.format(fstr.split(u'.')[0]) ) log.error(msg) - return {'error': msg} + return {u'error': msg} return jid def _send_pub(self, load): @@ -2032,7 +1999,7 @@ class ClearFuncs(object): chan = salt.transport.server.PubServerChannel.factory(opts) chan.publish(load) - def _prep_pub(self, minions, jid, clear_load, extra): + def _prep_pub(self, minions, jid, clear_load, extra, missing): ''' Take a given load and perform the necessary steps to prepare a publication. @@ -2040,26 +2007,27 @@ class ClearFuncs(object): TODO: This is really only bound by temporal cohesion and thus should be refactored even further. ''' - clear_load['jid'] = jid - delimiter = clear_load.get('kwargs', {}).get('delimiter', DEFAULT_TARGET_DELIM) + clear_load[u'jid'] = jid + delimiter = clear_load.get(u'kwargs', {}).get(u'delimiter', DEFAULT_TARGET_DELIM) # TODO Error reporting over the master event bus - self.event.fire_event({'minions': minions}, clear_load['jid']) + self.event.fire_event({u'minions': minions}, clear_load[u'jid']) new_job_load = { - 'jid': clear_load['jid'], - 'tgt_type': clear_load['tgt_type'], - 'tgt': clear_load['tgt'], - 'user': clear_load['user'], - 'fun': clear_load['fun'], - 'arg': clear_load['arg'], - 'minions': minions, + u'jid': clear_load[u'jid'], + u'tgt_type': clear_load[u'tgt_type'], + u'tgt': clear_load[u'tgt'], + u'user': clear_load[u'user'], + u'fun': clear_load[u'fun'], + u'arg': clear_load[u'arg'], + u'minions': minions, + u'missing': missing, } # Announce the job on the event bus - self.event.fire_event(new_job_load, tagify([clear_load['jid'], 'new'], 'job')) + self.event.fire_event(new_job_load, tagify([clear_load[u'jid'], u'new'], u'job')) - if self.opts['ext_job_cache']: - fstr = '{0}.save_load'.format(self.opts['ext_job_cache']) + if self.opts[u'ext_job_cache']: + fstr = u'{0}.save_load'.format(self.opts[u'ext_job_cache']) save_load_func = True # Get the returner's save_load arg_spec. @@ -2068,50 +2036,46 @@ class ClearFuncs(object): # Check if 'minions' is included in returner's save_load arg_spec. # This may be missing in custom returners, which we should warn about. - if 'minions' not in arg_spec.args: + if u'minions' not in arg_spec.args: log.critical( - 'The specified returner used for the external job cache ' - '\'{0}\' does not have a \'minions\' kwarg in the returner\'s ' - 'save_load function.'.format( - self.opts['ext_job_cache'] - ) + u'The specified returner used for the external job cache ' + u'\'%s\' does not have a \'minions\' kwarg in the returner\'s ' + u'save_load function.', self.opts[u'ext_job_cache'] ) except (AttributeError, KeyError): save_load_func = False log.critical( - 'The specified returner used for the external job cache ' - '"{0}" does not have a save_load function!'.format( - self.opts['ext_job_cache'] - ) + u'The specified returner used for the external job cache ' + u'"%s" does not have a save_load function!', + self.opts[u'ext_job_cache'] ) if save_load_func: try: - self.mminion.returners[fstr](clear_load['jid'], clear_load, minions=minions) + self.mminion.returners[fstr](clear_load[u'jid'], clear_load, minions=minions) except Exception: log.critical( - 'The specified returner threw a stack trace:\n', + u'The specified returner threw a stack trace:\n', exc_info=True ) # always write out to the master job caches try: - fstr = '{0}.save_load'.format(self.opts['master_job_cache']) - self.mminion.returners[fstr](clear_load['jid'], clear_load, minions) + fstr = u'{0}.save_load'.format(self.opts[u'master_job_cache']) + self.mminion.returners[fstr](clear_load[u'jid'], clear_load, minions) except KeyError: log.critical( - 'The specified returner used for the master job cache ' - '"{0}" does not have a save_load function!'.format( - self.opts['master_job_cache'] - ) + u'The specified returner used for the master job cache ' + u'"%s" does not have a save_load function!', + self.opts[u'master_job_cache'] ) except Exception: log.critical( - 'The specified returner threw a stack trace:\n', + u'The specified returner threw a stack trace:\n', exc_info=True ) # Set up the payload - payload = {'enc': 'aes'} + payload = {u'enc': u'aes'} # Altering the contents of the publish load is serious!! Changes here # break compatibility with minion/master versions and even tiny # additions can have serious implications on the performance of the @@ -2121,59 +2085,57 @@ class ClearFuncs(object): # touching this stuff, we can probably do what you want to do another # way that won't have a negative impact. load = { - 'fun': clear_load['fun'], - 'arg': clear_load['arg'], - 'tgt': clear_load['tgt'], - 'jid': clear_load['jid'], - 'ret': clear_load['ret'], + u'fun': clear_load[u'fun'], + u'arg': clear_load[u'arg'], + u'tgt': clear_load[u'tgt'], + u'jid': clear_load[u'jid'], + u'ret': clear_load[u'ret'], } # if you specified a master id, lets put that in the load - if 'master_id' in self.opts: - load['master_id'] = self.opts['master_id'] + if u'master_id' in self.opts: + load[u'master_id'] = self.opts[u'master_id'] # if someone passed us one, use that - if 'master_id' in extra: - load['master_id'] = extra['master_id'] + if u'master_id' in extra: + load[u'master_id'] = extra[u'master_id'] # Only add the delimiter to the pub data if it is non-default if delimiter != DEFAULT_TARGET_DELIM: - load['delimiter'] = delimiter + load[u'delimiter'] = delimiter - if 'id' in extra: - load['id'] = extra['id'] - if 'tgt_type' in clear_load: - load['tgt_type'] = clear_load['tgt_type'] - if 'to' in clear_load: - load['to'] = clear_load['to'] + if u'id' in extra: + load[u'id'] = extra[u'id'] + if u'tgt_type' in clear_load: + load[u'tgt_type'] = clear_load[u'tgt_type'] + if u'to' in clear_load: + load[u'to'] = clear_load[u'to'] - if 'kwargs' in clear_load: - if 'ret_config' in clear_load['kwargs']: - load['ret_config'] = clear_load['kwargs'].get('ret_config') + if u'kwargs' in clear_load: + if u'ret_config' in clear_load[u'kwargs']: + load[u'ret_config'] = clear_load[u'kwargs'].get(u'ret_config') - if 'metadata' in clear_load['kwargs']: - load['metadata'] = clear_load['kwargs'].get('metadata') + if u'metadata' in clear_load[u'kwargs']: + load[u'metadata'] = clear_load[u'kwargs'].get(u'metadata') - if 'module_executors' in clear_load['kwargs']: - load['module_executors'] = clear_load['kwargs'].get('module_executors') + if u'module_executors' in clear_load[u'kwargs']: + load[u'module_executors'] = clear_load[u'kwargs'].get(u'module_executors') - if 'executor_opts' in clear_load['kwargs']: - load['executor_opts'] = clear_load['kwargs'].get('executor_opts') + if u'executor_opts' in clear_load[u'kwargs']: + load[u'executor_opts'] = clear_load[u'kwargs'].get(u'executor_opts') - if 'ret_kwargs' in clear_load['kwargs']: - load['ret_kwargs'] = clear_load['kwargs'].get('ret_kwargs') + if u'ret_kwargs' in clear_load[u'kwargs']: + load[u'ret_kwargs'] = clear_load[u'kwargs'].get(u'ret_kwargs') - if 'user' in clear_load: + if u'user' in clear_load: log.info( - 'User {user} Published command {fun} with jid {jid}'.format( - **clear_load - ) + u'User %s Published command %s with jid %s', + clear_load[u'user'], clear_load[u'fun'], clear_load[u'jid'] ) - load['user'] = clear_load['user'] + load[u'user'] = clear_load[u'user'] else: log.info( - 'Published command {fun} with jid {jid}'.format( - **clear_load - ) + u'Published command %s with jid %s', + clear_load[u'fun'], clear_load[u'jid'] ) - log.debug('Published command details {0}'.format(load)) + log.debug(u'Published command details %s', load) return load def ping(self, clear_load): @@ -2205,15 +2167,15 @@ class FloMWorker(MWorker): self.aes_funcs = salt.master.AESFuncs(self.opts) self.context = zmq.Context(1) self.socket = self.context.socket(zmq.REP) - if self.opts.get('ipc_mode', '') == 'tcp': - self.w_uri = 'tcp://127.0.0.1:{0}'.format( - self.opts.get('tcp_master_workers', 4515) + if self.opts.get(u'ipc_mode', u'') == u'tcp': + self.w_uri = u'tcp://127.0.0.1:{0}'.format( + self.opts.get(u'tcp_master_workers', 4515) ) else: - self.w_uri = 'ipc://{0}'.format( - os.path.join(self.opts['sock_dir'], 'workers.ipc') + self.w_uri = u'ipc://{0}'.format( + os.path.join(self.opts[u'sock_dir'], u'workers.ipc') ) - log.info('ZMQ Worker binding to socket {0}'.format(self.w_uri)) + log.info(u'ZMQ Worker binding to socket %s', self.w_uri) self.poller = zmq.Poller() self.poller.register(self.socket, zmq.POLLIN) self.socket.connect(self.w_uri) diff --git a/salt/minion.py b/salt/minion.py index 444b3d1471b..d51445be28a 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -21,10 +21,11 @@ import multiprocessing from random import randint, shuffle from stat import S_IMODE import salt.serializers.msgpack +from binascii import crc32 # Import Salt Libs # pylint: disable=import-error,no-name-in-module,redefined-builtin -import salt.ext.six as six +from salt.ext import six if six.PY3: import ipaddress else: @@ -38,7 +39,7 @@ try: # TODO: cleanup import zmq.eventloop.ioloop # support pyzmq 13.0.x, TODO: remove once we force people to 14.0.x - if not hasattr(zmq.eventloop.ioloop, 'ZMQIOLoop'): + if not hasattr(zmq.eventloop.ioloop, u'ZMQIOLoop'): zmq.eventloop.ioloop.ZMQIOLoop = zmq.eventloop.ioloop.IOLoop LOOP_CLASS = zmq.eventloop.ioloop.ZMQIOLoop HAS_ZMQ = True @@ -83,23 +84,26 @@ import salt.loader import salt.beacons import salt.engines import salt.payload +import salt.pillar import salt.syspaths import salt.utils -import salt.utils.context -import salt.utils.jid -import salt.pillar import salt.utils.args +import salt.utils.context +import salt.utils.error import salt.utils.event -import salt.utils.network +import salt.utils.files +import salt.utils.jid import salt.utils.minion import salt.utils.minions +import salt.utils.network +import salt.utils.platform import salt.utils.schedule -import salt.utils.error import salt.utils.zeromq import salt.defaults.exitcodes import salt.cli.daemons import salt.log.setup +import salt.utils.dictupdate from salt.config import DEFAULT_MINION_OPTS from salt.defaults import DEFAULT_TARGET_DELIM from salt.utils.debug import enable_sigusr1_handler @@ -140,64 +144,66 @@ def resolve_dns(opts, fallback=True): ''' ret = {} check_dns = True - if (opts.get('file_client', 'remote') == 'local' and - not opts.get('use_master_when_local', False)): + if (opts.get(u'file_client', u'remote') == u'local' and + not opts.get(u'use_master_when_local', False)): check_dns = False if check_dns is True: # Because I import salt.log below I need to re-import salt.utils here import salt.utils try: - if opts['master'] == '': + if opts[u'master'] == u'': raise SaltSystemExit - ret['master_ip'] = \ - salt.utils.dns_check(opts['master'], int(opts['master_port']), True, opts['ipv6']) + ret[u'master_ip'] = \ + salt.utils.dns_check(opts[u'master'], int(opts[u'master_port']), True, opts[u'ipv6']) except SaltClientError: - if opts['retry_dns']: + if opts[u'retry_dns']: while True: import salt.log - msg = ('Master hostname: \'{0}\' not found or not responsive. ' - 'Retrying in {1} seconds').format(opts['master'], opts['retry_dns']) + msg = (u'Master hostname: \'{0}\' not found or not responsive. ' + u'Retrying in {1} seconds').format(opts[u'master'], opts[u'retry_dns']) if salt.log.setup.is_console_configured(): log.error(msg) else: - print('WARNING: {0}'.format(msg)) - time.sleep(opts['retry_dns']) + print(u'WARNING: {0}'.format(msg)) + time.sleep(opts[u'retry_dns']) try: - ret['master_ip'] = salt.utils.dns_check( - opts['master'], int(opts['master_port']), True, opts['ipv6'] + ret[u'master_ip'] = salt.utils.dns_check( + opts[u'master'], int(opts[u'master_port']), True, opts[u'ipv6'] ) break except SaltClientError: pass else: if fallback: - ret['master_ip'] = '127.0.0.1' + ret[u'master_ip'] = u'127.0.0.1' else: raise except SaltSystemExit: - unknown_str = 'unknown address' - master = opts.get('master', unknown_str) - if master == '': + unknown_str = u'unknown address' + master = opts.get(u'master', unknown_str) + if master == u'': master = unknown_str - if opts.get('__role') == 'syndic': - err = 'Master address: \'{0}\' could not be resolved. Invalid or unresolveable address. ' \ - 'Set \'syndic_master\' value in minion config.'.format(master) + if opts.get(u'__role') == u'syndic': + err = u'Master address: \'{0}\' could not be resolved. Invalid or unresolveable address. ' \ + u'Set \'syndic_master\' value in minion config.'.format(master) else: - err = 'Master address: \'{0}\' could not be resolved. Invalid or unresolveable address. ' \ - 'Set \'master\' value in minion config.'.format(master) + err = u'Master address: \'{0}\' could not be resolved. Invalid or unresolveable address. ' \ + u'Set \'master\' value in minion config.'.format(master) log.error(err) raise SaltSystemExit(code=42, msg=err) else: - ret['master_ip'] = '127.0.0.1' + ret[u'master_ip'] = u'127.0.0.1' - if 'master_ip' in ret and 'master_ip' in opts: - if ret['master_ip'] != opts['master_ip']: - log.warning('Master ip address changed from {0} to {1}'.format(opts['master_ip'], - ret['master_ip']) + if u'master_ip' in ret and u'master_ip' in opts: + if ret[u'master_ip'] != opts[u'master_ip']: + log.warning( + u'Master ip address changed from %s to %s', + opts[u'master_ip'], ret[u'master_ip'] ) - ret['master_uri'] = 'tcp://{ip}:{port}'.format(ip=ret['master_ip'], - port=opts['master_port']) + ret[u'master_uri'] = u'tcp://{ip}:{port}'.format( + ip=ret[u'master_ip'], port=opts[u'master_port']) + return ret @@ -206,23 +212,23 @@ def prep_ip_port(opts): # Use given master IP if "ip_only" is set or if master_ip is an ipv6 address without # a port specified. The is_ipv6 check returns False if brackets are used in the IP # definition such as master: '[::1]:1234'. - if opts['master_uri_format'] == 'ip_only' or salt.utils.network.is_ipv6(opts['master']): - ret['master'] = opts['master'] + if opts[u'master_uri_format'] == u'ip_only' or salt.utils.network.is_ipv6(opts[u'master']): + ret[u'master'] = opts[u'master'] else: - ip_port = opts['master'].rsplit(":", 1) + ip_port = opts[u'master'].rsplit(u':', 1) if len(ip_port) == 1: # e.g. master: mysaltmaster - ret['master'] = ip_port[0] + ret[u'master'] = ip_port[0] else: # e.g. master: localhost:1234 # e.g. master: 127.0.0.1:1234 # e.g. master: [::1]:1234 # Strip off brackets for ipv6 support - ret['master'] = ip_port[0].strip('[]') + ret[u'master'] = ip_port[0].strip(u'[]') # Cast port back to an int! Otherwise a TypeError is thrown # on some of the socket calls elsewhere in the minion and utils code. - ret['master_port'] = int(ip_port[1]) + ret[u'master_port'] = int(ip_port[1]) return ret @@ -242,13 +248,13 @@ def get_proc_dir(cachedir, **kwargs): made. Same applies if the directory is already owned by this gid. Must be int. Works only on unix/unix like systems. ''' - fn_ = os.path.join(cachedir, 'proc') - mode = kwargs.pop('mode', None) + fn_ = os.path.join(cachedir, u'proc') + mode = kwargs.pop(u'mode', None) if mode is None: mode = {} else: - mode = {'mode': mode} + mode = {u'mode': mode} if not os.path.isdir(fn_): # proc_dir is not present, create it with mode settings @@ -260,13 +266,13 @@ def get_proc_dir(cachedir, **kwargs): # dir mode. So lets check if mode needs to be changed. if mode: mode_part = S_IMODE(d_stat.st_mode) - if mode_part != mode['mode']: - os.chmod(fn_, (d_stat.st_mode ^ mode_part) | mode['mode']) + if mode_part != mode[u'mode']: + os.chmod(fn_, (d_stat.st_mode ^ mode_part) | mode[u'mode']) - if hasattr(os, 'chown'): + if hasattr(os, u'chown'): # only on unix/unix like systems - uid = kwargs.pop('uid', -1) - gid = kwargs.pop('gid', -1) + uid = kwargs.pop(u'uid', -1) + gid = kwargs.pop(u'gid', -1) # if uid and gid are both -1 then go ahead with # no changes at all @@ -288,7 +294,7 @@ def load_args_and_kwargs(func, args, data=None, ignore_invalid=False): invalid_kwargs = [] for arg in args: - if isinstance(arg, dict) and arg.pop('__kwarg__', False) is True: + if isinstance(arg, dict) and arg.pop(u'__kwarg__', False) is True: # if the arg is a dict with __kwarg__ == True, then its a kwarg for key, val in six.iteritems(arg): if argspec.keywords or key in argspec.args: @@ -299,30 +305,32 @@ def load_args_and_kwargs(func, args, data=None, ignore_invalid=False): # **kwargs not in argspec and parsed argument name not in # list of positional arguments. This keyword argument is # invalid. - invalid_kwargs.append('{0}={1}'.format(key, val)) + invalid_kwargs.append(u'{0}={1}'.format(key, val)) continue else: string_kwarg = salt.utils.args.parse_input([arg], condition=False)[1] # pylint: disable=W0632 if string_kwarg: - log.critical( - 'String kwarg(s) %s passed to ' - 'salt.minion.load_args_and_kwargs(). This is no longer ' - 'supported, so the kwarg(s) will be ignored. Arguments ' - 'passed to salt.minion.load_args_and_kwargs() should be ' - 'passed to salt.utils.args.parse_input() first to load ' - 'and condition them properly.', string_kwarg - ) + if argspec.keywords or next(six.iterkeys(string_kwarg)) in argspec.args: + # Function supports **kwargs or is a positional argument to + # the function. + _kwargs.update(string_kwarg) + else: + # **kwargs not in argspec and parsed argument name not in + # list of positional arguments. This keyword argument is + # invalid. + for key, val in six.iteritems(string_kwarg): + invalid_kwargs.append(u'{0}={1}'.format(key, val)) else: _args.append(arg) if invalid_kwargs and not ignore_invalid: - salt.utils.invalid_kwargs(invalid_kwargs) + salt.utils.args.invalid_kwargs(invalid_kwargs) if argspec.keywords and isinstance(data, dict): # this function accepts **kwargs, pack in the publish data for key, val in six.iteritems(data): - _kwargs['__pub_{0}'.format(key)] = val + _kwargs[u'__pub_{0}'.format(key)] = val return _args, _kwargs @@ -332,40 +340,40 @@ def eval_master_func(opts): Evaluate master function if master type is 'func' and save it result in opts['master'] ''' - if '__master_func_evaluated' not in opts: + if u'__master_func_evaluated' not in opts: # split module and function and try loading the module - mod_fun = opts['master'] - mod, fun = mod_fun.split('.') + mod_fun = opts[u'master'] + mod, fun = mod_fun.split(u'.') try: master_mod = salt.loader.raw_mod(opts, mod, fun) if not master_mod: raise KeyError # we take whatever the module returns as master address - opts['master'] = master_mod[mod_fun]() + opts[u'master'] = master_mod[mod_fun]() # Check for valid types - if not isinstance(opts['master'], (str, list)): + if not isinstance(opts[u'master'], (six.string_types, list)): raise TypeError - opts['__master_func_evaluated'] = True + opts[u'__master_func_evaluated'] = True except KeyError: - log.error('Failed to load module {0}'.format(mod_fun)) + log.error(u'Failed to load module %s', mod_fun) sys.exit(salt.defaults.exitcodes.EX_GENERIC) except TypeError: - log.error('{0} returned from {1} is not a string'.format(opts['master'], mod_fun)) + log.error(u'%s returned from %s is not a string', opts[u'master'], mod_fun) sys.exit(salt.defaults.exitcodes.EX_GENERIC) - log.info('Evaluated master from module: {0}'.format(mod_fun)) + log.info(u'Evaluated master from module: %s', mod_fun) def master_event(type, master=None): ''' Centralized master event function which will return event type based on event_map ''' - event_map = {'connected': '__master_connected', - 'disconnected': '__master_disconnected', - 'failback': '__master_failback', - 'alive': '__master_alive'} + event_map = {u'connected': u'__master_connected', + u'disconnected': u'__master_disconnected', + u'failback': u'__master_failback', + u'alive': u'__master_alive'} - if type == 'alive' and master is not None: - return '{0}_{1}'.format(event_map.get(type), master) + if type == u'alive' and master is not None: + return u'{0}_{1}'.format(event_map.get(type), master) return event_map.get(type, None) @@ -377,22 +385,20 @@ class MinionBase(object): @staticmethod def process_schedule(minion, loop_interval): try: - if hasattr(minion, 'schedule'): + if hasattr(minion, u'schedule'): minion.schedule.eval() else: - log.error('Minion scheduler not initialized. Scheduled jobs will not be run.') + log.error(u'Minion scheduler not initialized. Scheduled jobs will not be run.') return # Check if scheduler requires lower loop interval than # the loop_interval setting if minion.schedule.loop_interval < loop_interval: loop_interval = minion.schedule.loop_interval log.debug( - 'Overriding loop_interval because of scheduled jobs.' + u'Overriding loop_interval because of scheduled jobs.' ) except Exception as exc: - log.error( - 'Exception {0} occurred in scheduled job'.format(exc) - ) + log.error(u'Exception %s occurred in scheduled job', exc) return loop_interval def process_beacons(self, functions): @@ -400,10 +406,10 @@ class MinionBase(object): Evaluate all of the configured beacons, grab the config again in case the pillar or grains changed ''' - if 'config.merge' in functions: - b_conf = functions['config.merge']('beacons', self.opts['beacons'], omit_opts=True) + if u'config.merge' in functions: + b_conf = functions[u'config.merge'](u'beacons', self.opts[u'beacons'], omit_opts=True) if b_conf: - return self.beacons.process(b_conf, self.opts['grains']) # pylint: disable=no-member + return self.beacons.process(b_conf, self.opts[u'grains']) # pylint: disable=no-member return [] @tornado.gen.coroutine @@ -429,38 +435,57 @@ class MinionBase(object): (possibly failed) master will then be removed from the list of masters. ''' # return early if we are not connecting to a master - if opts['master_type'] == 'disable': - log.warning('Master is set to disable, skipping connection') + if opts[u'master_type'] == u'disable': + log.warning(u'Master is set to disable, skipping connection') self.connected = False raise tornado.gen.Return((None, None)) # check if master_type was altered from its default - elif opts['master_type'] != 'str' and opts['__role'] != 'syndic': + elif opts[u'master_type'] != u'str' and opts[u'__role'] != u'syndic': # check for a valid keyword - if opts['master_type'] == 'func': + if opts[u'master_type'] == u'func': eval_master_func(opts) - # if failover is set, master has to be of type list - elif opts['master_type'] == 'failover': - if isinstance(opts['master'], list): - log.info('Got list of available master addresses:' - ' {0}'.format(opts['master'])) - if opts['master_shuffle']: - if opts['master_failback']: - secondary_masters = opts['master'][1:] - shuffle(secondary_masters) - opts['master'][1:] = secondary_masters + # if failover or distributed is set, master has to be of type list + elif opts[u'master_type'] in (u'failover', u'distributed'): + if isinstance(opts[u'master'], list): + log.info( + u'Got list of available master addresses: %s', + opts[u'master'] + ) + + if opts[u'master_type'] == u'distributed': + master_len = len(opts[u'master']) + if master_len > 1: + secondary_masters = opts[u'master'][1:] + master_idx = crc32(opts[u'id']) % master_len + try: + preferred_masters = opts[u'master'] + preferred_masters[0] = opts[u'master'][master_idx] + preferred_masters[1:] = [m for m in opts[u'master'] if m != preferred_masters[0]] + opts[u'master'] = preferred_masters + log.info(u'Distributed to the master at \'{0}\'.'.format(opts[u'master'][0])) + except (KeyError, AttributeError, TypeError): + log.warning(u'Failed to distribute to a specific master.') else: - shuffle(opts['master']) - opts['auth_tries'] = 0 - if opts['master_failback'] and opts['master_failback_interval'] == 0: - opts['master_failback_interval'] = opts['master_alive_interval'] + log.warning(u'master_type = distributed needs more than 1 master.') + + if opts[u'master_shuffle']: + if opts[u'master_failback']: + secondary_masters = opts[u'master'][1:] + shuffle(secondary_masters) + opts[u'master'][1:] = secondary_masters + else: + shuffle(opts[u'master']) + opts[u'auth_tries'] = 0 + if opts[u'master_failback'] and opts[u'master_failback_interval'] == 0: + opts[u'master_failback_interval'] = opts[u'master_alive_interval'] # if opts['master'] is a str and we have never created opts['master_list'] - elif isinstance(opts['master'], str) and ('master_list' not in opts): + elif isinstance(opts[u'master'], six.string_types) and (u'master_list' not in opts): # We have a string, but a list was what was intended. Convert. # See issue 23611 for details - opts['master'] = [opts['master']] - elif opts['__role'] == 'syndic': - log.info('Syndic setting master_syndic to \'{0}\''.format(opts['master'])) + opts[u'master'] = [opts[u'master']] + elif opts[u'__role'] == u'syndic': + log.info(u'Syndic setting master_syndic to \'%s\'', opts[u'master']) # if failed=True, the minion was previously connected # we're probably called from the minions main-event-loop @@ -469,90 +494,94 @@ class MinionBase(object): elif failed: if failback: # failback list of masters to original config - opts['master'] = opts['master_list'] + opts[u'master'] = opts[u'master_list'] else: - log.info('Moving possibly failed master {0} to the end of' - ' the list of masters'.format(opts['master'])) - if opts['master'] in opts['local_masters']: + log.info( + u'Moving possibly failed master %s to the end of ' + u'the list of masters', opts[u'master'] + ) + if opts[u'master'] in opts[u'local_masters']: # create new list of master with the possibly failed # one moved to the end - failed_master = opts['master'] - opts['master'] = [x for x in opts['local_masters'] if opts['master'] != x] - opts['master'].append(failed_master) + failed_master = opts[u'master'] + opts[u'master'] = [x for x in opts[u'local_masters'] if opts[u'master'] != x] + opts[u'master'].append(failed_master) else: - opts['master'] = opts['master_list'] + opts[u'master'] = opts[u'master_list'] else: - msg = ('master_type set to \'failover\' but \'master\' ' - 'is not of type list but of type ' - '{0}'.format(type(opts['master']))) + msg = (u'master_type set to \'failover\' but \'master\' ' + u'is not of type list but of type ' + u'{0}'.format(type(opts[u'master']))) log.error(msg) sys.exit(salt.defaults.exitcodes.EX_GENERIC) # If failover is set, minion have to failover on DNS errors instead of retry DNS resolve. # See issue 21082 for details - if opts['retry_dns']: - msg = ('\'master_type\' set to \'failover\' but \'retry_dns\' is not 0. ' - 'Setting \'retry_dns\' to 0 to failover to the next master on DNS errors.') + if opts[u'retry_dns'] and opts[u'master_type'] == u'failover': + msg = (u'\'master_type\' set to \'failover\' but \'retry_dns\' is not 0. ' + u'Setting \'retry_dns\' to 0 to failover to the next master on DNS errors.') log.critical(msg) - opts['retry_dns'] = 0 + opts[u'retry_dns'] = 0 else: - msg = ('Invalid keyword \'{0}\' for variable ' - '\'master_type\''.format(opts['master_type'])) + msg = (u'Invalid keyword \'{0}\' for variable ' + u'\'master_type\''.format(opts[u'master_type'])) log.error(msg) sys.exit(salt.defaults.exitcodes.EX_GENERIC) # FIXME: if SMinion don't define io_loop, it can't switch master see #29088 # Specify kwargs for the channel factory so that SMinion doesn't need to define an io_loop # (The channel factories will set a default if the kwarg isn't passed) - factory_kwargs = {'timeout': timeout, 'safe': safe} - if getattr(self, 'io_loop', None): - factory_kwargs['io_loop'] = self.io_loop # pylint: disable=no-member + factory_kwargs = {u'timeout': timeout, u'safe': safe} + if getattr(self, u'io_loop', None): + factory_kwargs[u'io_loop'] = self.io_loop # pylint: disable=no-member - tries = opts.get('master_tries', 1) + tries = opts.get(u'master_tries', 1) attempts = 0 # if we have a list of masters, loop through them and be # happy with the first one that allows us to connect - if isinstance(opts['master'], list): + if isinstance(opts[u'master'], list): conn = False # shuffle the masters and then loop through them - opts['local_masters'] = copy.copy(opts['master']) - if opts['random_master']: - shuffle(opts['local_masters']) + opts[u'local_masters'] = copy.copy(opts[u'master']) + if opts[u'random_master']: + shuffle(opts[u'local_masters']) last_exc = None - opts['master_uri_list'] = list() + opts[u'master_uri_list'] = list() # This sits outside of the connection loop below because it needs to set # up a list of master URIs regardless of which masters are available # to connect _to_. This is primarily used for masterless mode, when # we need a list of master URIs to fire calls back to. - for master in opts['local_masters']: - opts['master'] = master + for master in opts[u'local_masters']: + opts[u'master'] = master opts.update(prep_ip_port(opts)) - opts['master_uri_list'].append(resolve_dns(opts)['master_uri']) + opts[u'master_uri_list'].append(resolve_dns(opts)[u'master_uri']) while True: if attempts != 0: # Give up a little time between connection attempts # to allow the IOLoop to run any other scheduled tasks. - yield tornado.gen.sleep(opts['acceptance_wait_time']) + yield tornado.gen.sleep(opts[u'acceptance_wait_time']) attempts += 1 if tries > 0: - log.debug('Connecting to master. Attempt {0} ' - 'of {1}'.format(attempts, tries) + log.debug( + u'Connecting to master. Attempt %s of %s', + attempts, tries ) else: - log.debug('Connecting to master. Attempt {0} ' - '(infinite attempts)'.format(attempts) + log.debug( + u'Connecting to master. Attempt %s (infinite attempts)', + attempts ) - for master in opts['local_masters']: - opts['master'] = master + for master in opts[u'local_masters']: + opts[u'master'] = master opts.update(prep_ip_port(opts)) opts.update(resolve_dns(opts)) # on first run, update self.opts with the whole master list # to enable a minion to re-use old masters if they get fixed - if 'master_list' not in opts: - opts['master_list'] = copy.copy(opts['local_masters']) + if u'master_list' not in opts: + opts[u'master_list'] = copy.copy(opts[u'local_masters']) self.opts = opts @@ -563,66 +592,70 @@ class MinionBase(object): break except SaltClientError as exc: last_exc = exc - msg = ('Master {0} could not be reached, trying ' - 'next master (if any)'.format(opts['master'])) - log.info(msg) + log.info( + u'Master %s could not be reached, trying next ' + u'next master (if any)', opts[u'master'] + ) continue if not conn: if attempts == tries: # Exhausted all attempts. Return exception. self.connected = False - self.opts['master'] = copy.copy(self.opts['local_masters']) - msg = ('No master could be reached or all masters ' - 'denied the minions connection attempt.') - log.error(msg) + self.opts[u'master'] = copy.copy(self.opts[u'local_masters']) + log.error( + u'No master could be reached or all masters ' + u'denied the minion\'s connection attempt.' + ) # If the code reaches this point, 'last_exc' # should already be set. raise last_exc # pylint: disable=E0702 else: - self.tok = pub_channel.auth.gen_token('salt') + self.tok = pub_channel.auth.gen_token(u'salt') self.connected = True - raise tornado.gen.Return((opts['master'], pub_channel)) + raise tornado.gen.Return((opts[u'master'], pub_channel)) # single master sign in else: - if opts['random_master']: - log.warning('random_master is True but there is only one master specified. Ignoring.') + if opts[u'random_master']: + log.warning(u'random_master is True but there is only one master specified. Ignoring.') while True: if attempts != 0: # Give up a little time between connection attempts # to allow the IOLoop to run any other scheduled tasks. - yield tornado.gen.sleep(opts['acceptance_wait_time']) + yield tornado.gen.sleep(opts[u'acceptance_wait_time']) attempts += 1 if tries > 0: - log.debug('Connecting to master. Attempt {0} ' - 'of {1}'.format(attempts, tries) + log.debug( + u'Connecting to master. Attempt %s of %s', + attempts, tries ) else: - log.debug('Connecting to master. Attempt {0} ' - '(infinite attempts)'.format(attempts) + log.debug( + u'Connecting to master. Attempt %s (infinite attempts)', + attempts ) opts.update(prep_ip_port(opts)) opts.update(resolve_dns(opts)) try: - if self.opts['transport'] == 'detect': - self.opts['detect_mode'] = True - for trans in ('zeromq', 'tcp'): - if trans == 'zeromq' and not HAS_ZMQ: + if self.opts[u'transport'] == u'detect': + self.opts[u'detect_mode'] = True + for trans in (u'zeromq', u'tcp'): + if trans == u'zeromq' and not HAS_ZMQ: continue - self.opts['transport'] = trans + self.opts[u'transport'] = trans pub_channel = salt.transport.client.AsyncPubChannel.factory(self.opts, **factory_kwargs) yield pub_channel.connect() if not pub_channel.auth.authenticated: continue - del self.opts['detect_mode'] + del self.opts[u'detect_mode'] break else: pub_channel = salt.transport.client.AsyncPubChannel.factory(self.opts, **factory_kwargs) yield pub_channel.connect() - self.tok = pub_channel.auth.gen_token('salt') + self.tok = pub_channel.auth.gen_token(u'salt') self.connected = True - raise tornado.gen.Return((opts['master'], pub_channel)) + raise tornado.gen.Return((opts[u'master'], pub_channel)) except SaltClientError as exc: if attempts == tries: # Exhausted all attempts. Return exception. @@ -639,13 +672,13 @@ class SMinion(MinionBase): ''' def __init__(self, opts): # Late setup of the opts grains, so we can log from the grains module - opts['grains'] = salt.loader.grains(opts) + opts[u'grains'] = salt.loader.grains(opts) super(SMinion, self).__init__(opts) # Clean out the proc directory (default /var/cache/salt/minion/proc) - if (self.opts.get('file_client', 'remote') == 'remote' - or self.opts.get('use_master_when_local', False)): - if self.opts['transport'] == 'zeromq' and HAS_ZMQ: + if (self.opts.get(u'file_client', u'remote') == u'remote' + or self.opts.get(u'use_master_when_local', False)): + if self.opts[u'transport'] == u'zeromq' and HAS_ZMQ: io_loop = zmq.eventloop.ioloop.ZMQIOLoop() else: io_loop = LOOP_CLASS.current() @@ -655,19 +688,19 @@ class SMinion(MinionBase): self.gen_modules(initial_load=True) # If configured, cache pillar data on the minion - if self.opts['file_client'] == 'remote' and self.opts.get('minion_pillar_cache', False): + if self.opts[u'file_client'] == u'remote' and self.opts.get(u'minion_pillar_cache', False): import yaml from salt.utils.yamldumper import SafeOrderedDumper - pdir = os.path.join(self.opts['cachedir'], 'pillar') + pdir = os.path.join(self.opts[u'cachedir'], u'pillar') if not os.path.isdir(pdir): os.makedirs(pdir, 0o700) - ptop = os.path.join(pdir, 'top.sls') - if self.opts['environment'] is not None: - penv = self.opts['environment'] + ptop = os.path.join(pdir, u'top.sls') + if self.opts[u'environment'] is not None: + penv = self.opts[u'environment'] else: - penv = 'base' - cache_top = {penv: {self.opts['id']: ['cache']}} - with salt.utils.fopen(ptop, 'wb') as fp_: + penv = u'base' + cache_top = {penv: {self.opts[u'id']: [u'cache']}} + with salt.utils.files.fopen(ptop, u'wb') as fp_: fp_.write( yaml.dump( cache_top, @@ -675,11 +708,11 @@ class SMinion(MinionBase): ) ) os.chmod(ptop, 0o600) - cache_sls = os.path.join(pdir, 'cache.sls') - with salt.utils.fopen(cache_sls, 'wb') as fp_: + cache_sls = os.path.join(pdir, u'cache.sls') + with salt.utils.files.fopen(cache_sls, u'wb') as fp_: fp_.write( yaml.dump( - self.opts['pillar'], + self.opts[u'pillar'], Dumper=SafeOrderedDumper ) ) @@ -695,12 +728,12 @@ class SMinion(MinionBase): salt '*' sys.reload_modules ''' - self.opts['pillar'] = salt.pillar.get_pillar( + self.opts[u'pillar'] = salt.pillar.get_pillar( self.opts, - self.opts['grains'], - self.opts['id'], - self.opts['environment'], - pillarenv=self.opts.get('pillarenv'), + self.opts[u'grains'], + self.opts[u'id'], + self.opts[u'environment'], + pillarenv=self.opts.get(u'pillarenv'), ).compile_pillar() self.utils = salt.loader.utils(self.opts) @@ -716,7 +749,7 @@ class SMinion(MinionBase): self.serializers) self.rend = salt.loader.render(self.opts, self.functions) self.matcher = Matcher(self.opts, self.functions) - self.functions['sys.reload_modules'] = self.gen_modules + self.functions[u'sys.reload_modules'] = self.gen_modules self.executors = salt.loader.executors(self.opts) @@ -735,11 +768,11 @@ class MasterMinion(object): matcher=True, whitelist=None, ignore_config_errors=True): - self.opts = salt.config.minion_config(opts['conf_file'], ignore_config_errors=ignore_config_errors) + self.opts = salt.config.minion_config(opts[u'conf_file'], ignore_config_errors=ignore_config_errors) self.opts.update(opts) self.whitelist = whitelist - self.opts['grains'] = salt.loader.grains(opts) - self.opts['pillar'] = {} + self.opts[u'grains'] = salt.loader.grains(opts) + self.opts[u'pillar'] = {} self.mk_returners = returners self.mk_states = states self.mk_rend = rend @@ -774,7 +807,7 @@ class MasterMinion(object): self.rend = salt.loader.render(self.opts, self.functions) if self.mk_matcher: self.matcher = Matcher(self.opts, self.functions) - self.functions['sys.reload_modules'] = self.gen_modules + self.functions[u'sys.reload_modules'] = self.gen_modules class MinionManager(MinionBase): @@ -785,15 +818,15 @@ class MinionManager(MinionBase): ''' def __init__(self, opts): super(MinionManager, self).__init__(opts) - self.auth_wait = self.opts['acceptance_wait_time'] - self.max_auth_wait = self.opts['acceptance_wait_time_max'] + self.auth_wait = self.opts[u'acceptance_wait_time'] + self.max_auth_wait = self.opts[u'acceptance_wait_time_max'] self.minions = [] self.jid_queue = [] if HAS_ZMQ: zmq.eventloop.ioloop.install() self.io_loop = LOOP_CLASS.current() - self.process_manager = ProcessManager(name='MultiMinionProcessManager') + self.process_manager = ProcessManager(name=u'MultiMinionProcessManager') self.io_loop.spawn_callback(self.process_manager.run, async=True) def __del__(self): @@ -805,8 +838,8 @@ class MinionManager(MinionBase): self.opts, io_loop=self.io_loop, ) - self.event = salt.utils.event.get_event('minion', opts=self.opts, io_loop=self.io_loop) - self.event.subscribe('') + self.event = salt.utils.event.get_event(u'minion', opts=self.opts, io_loop=self.io_loop) + self.event.subscribe(u'') self.event.set_event_handler(self.handle_event) @tornado.gen.coroutine @@ -830,19 +863,19 @@ class MinionManager(MinionBase): ''' Spawn all the coroutines which will sign in to masters ''' - masters = self.opts['master'] - if self.opts['master_type'] == 'failover' or not isinstance(self.opts['master'], list): + masters = self.opts[u'master'] + if (self.opts[u'master_type'] in (u'failover', u'distributed')) or not isinstance(self.opts[u'master'], list): masters = [masters] for master in masters: s_opts = copy.deepcopy(self.opts) - s_opts['master'] = master - s_opts['multimaster'] = True + s_opts[u'master'] = master + s_opts[u'multimaster'] = True minion = self._create_minion_object(s_opts, - s_opts['auth_timeout'], + s_opts[u'auth_timeout'], False, io_loop=self.io_loop, - loaded_base_name='salt.loader.{0}'.format(s_opts['master']), + loaded_base_name=u'salt.loader.{0}'.format(s_opts[u'master']), jid_queue=self.jid_queue, ) self.minions.append(minion) @@ -854,7 +887,7 @@ class MinionManager(MinionBase): Create a minion, and asynchronously connect it to a master ''' last = 0 # never have we signed in - auth_wait = minion.opts['acceptance_wait_time'] + auth_wait = minion.opts[u'acceptance_wait_time'] failed = False while True: try: @@ -863,14 +896,20 @@ class MinionManager(MinionBase): break except SaltClientError as exc: failed = True - log.error('Error while bringing up minion for multi-master. Is master at {0} responding?'.format(minion.opts['master'])) + log.error( + u'Error while bringing up minion for multi-master. Is ' + u'master at %s responding?', minion.opts[u'master'] + ) last = time.time() if auth_wait < self.max_auth_wait: auth_wait += self.auth_wait yield tornado.gen.sleep(auth_wait) # TODO: log? except Exception as e: failed = True - log.critical('Unexpected error while connecting to {0}'.format(minion.opts['master']), exc_info=True) + log.critical( + u'Unexpected error while connecting to %s', + minion.opts[u'master'], exc_info=True + ) # Multi Master Tune In def tune_in(self): @@ -948,36 +987,38 @@ class Minion(MinionBase): # PyZMQ <= 2.1.9 does not have zmq_version_info, fall back to # using zmq.zmq_version() and build a version info tuple. zmq_version_info = tuple( - [int(x) for x in zmq.zmq_version().split('.')] # pylint: disable=no-member + [int(x) for x in zmq.zmq_version().split(u'.')] # pylint: disable=no-member ) if zmq_version_info < (3, 2): log.warning( - 'You have a version of ZMQ less than ZMQ 3.2! There are ' - 'known connection keep-alive issues with ZMQ < 3.2 which ' - 'may result in loss of contact with minions. Please ' - 'upgrade your ZMQ!' + u'You have a version of ZMQ less than ZMQ 3.2! There are ' + u'known connection keep-alive issues with ZMQ < 3.2 which ' + u'may result in loss of contact with minions. Please ' + u'upgrade your ZMQ!' ) # Late setup the of the opts grains, so we can log from the grains # module. If this is a proxy, however, we need to init the proxymodule # before we can get the grains. We do this for proxies in the # post_master_init - if not salt.utils.is_proxy(): - self.opts['grains'] = salt.loader.grains(opts) + if not salt.utils.platform.is_proxy(): + self.opts[u'grains'] = salt.loader.grains(opts) - log.info('Creating minion process manager') + log.info(u'Creating minion process manager') - if self.opts['random_startup_delay']: - sleep_time = random.randint(0, self.opts['random_startup_delay']) - log.info('Minion sleeping for {0} seconds due to configured ' - 'startup_delay between 0 and {1} seconds'.format(sleep_time, - self.opts['random_startup_delay'])) + if self.opts[u'random_startup_delay']: + sleep_time = random.randint(0, self.opts[u'random_startup_delay']) + log.info( + u'Minion sleeping for %s seconds due to configured ' + u'startup_delay between 0 and %s seconds', + sleep_time, self.opts[u'random_startup_delay'] + ) time.sleep(sleep_time) - self.process_manager = ProcessManager(name='MinionProcessManager') + self.process_manager = ProcessManager(name=u'MinionProcessManager') self.io_loop.spawn_callback(self.process_manager.run, async=True) # We don't have the proxy setup yet, so we can't start engines # Engines need to be able to access __proxy__ - if not salt.utils.is_proxy(): + if not salt.utils.platform.is_proxy(): self.io_loop.spawn_callback(salt.engines.start_engines, self.opts, self.process_manager) @@ -1005,7 +1046,7 @@ class Minion(MinionBase): Block until we are connected to a master ''' self._sync_connect_master_success = False - log.debug("sync_connect_master") + log.debug(u"sync_connect_master") def on_connect_master_future_done(future): self._sync_connect_master_success = True @@ -1028,7 +1069,7 @@ class Minion(MinionBase): # This needs to be re-raised to preserve restart_on_error behavior. raise six.reraise(*future_exception) if timeout and self._sync_connect_master_success is False: - raise SaltDaemonNotRunning('Failed to connect to the salt-master') + raise SaltDaemonNotRunning(u'Failed to connect to the salt-master') @tornado.gen.coroutine def connect_master(self, failed=False): @@ -1055,15 +1096,15 @@ class Minion(MinionBase): functions. ''' if self.connected: - self.opts['master'] = master + self.opts[u'master'] = master # Initialize pillar before loader to make pillar accessible in modules - self.opts['pillar'] = yield salt.pillar.get_async_pillar( + self.opts[u'pillar'] = yield salt.pillar.get_async_pillar( self.opts, - self.opts['grains'], - self.opts['id'], - self.opts['environment'], - pillarenv=self.opts.get('pillarenv') + self.opts[u'grains'], + self.opts[u'id'], + self.opts[u'environment'], + pillarenv=self.opts.get(u'pillarenv') ).compile_pillar() self.functions, self.returners, self.function_errors, self.executors = self._load_modules() @@ -1071,68 +1112,68 @@ class Minion(MinionBase): self.mod_opts = self._prep_mod_opts() self.matcher = Matcher(self.opts, self.functions) self.beacons = salt.beacons.Beacon(self.opts, self.functions) - uid = salt.utils.get_uid(user=self.opts.get('user', None)) - self.proc_dir = get_proc_dir(self.opts['cachedir'], uid=uid) + uid = salt.utils.get_uid(user=self.opts.get(u'user', None)) + self.proc_dir = get_proc_dir(self.opts[u'cachedir'], uid=uid) self.schedule = salt.utils.schedule.Schedule( self.opts, self.functions, self.returners, - cleanup=[master_event(type='alive')]) + cleanup=[master_event(type=u'alive')]) # add default scheduling jobs to the minions scheduler - if self.opts['mine_enabled'] and 'mine.update' in self.functions: + if self.opts[u'mine_enabled'] and u'mine.update' in self.functions: self.schedule.add_job({ - '__mine_interval': + u'__mine_interval': { - 'function': 'mine.update', - 'minutes': self.opts['mine_interval'], - 'jid_include': True, - 'maxrunning': 2, - 'return_job': self.opts.get('mine_return_job', False) + u'function': u'mine.update', + u'minutes': self.opts[u'mine_interval'], + u'jid_include': True, + u'maxrunning': 2, + u'return_job': self.opts.get(u'mine_return_job', False) } }, persist=True) - log.info('Added mine.update to scheduler') + log.info(u'Added mine.update to scheduler') else: - self.schedule.delete_job('__mine_interval', persist=True) + self.schedule.delete_job(u'__mine_interval', persist=True) # add master_alive job if enabled - if (self.opts['transport'] != 'tcp' and - self.opts['master_alive_interval'] > 0 and + if (self.opts[u'transport'] != u'tcp' and + self.opts[u'master_alive_interval'] > 0 and self.connected): self.schedule.add_job({ - master_event(type='alive', master=self.opts['master']): + master_event(type=u'alive', master=self.opts[u'master']): { - 'function': 'status.master', - 'seconds': self.opts['master_alive_interval'], - 'jid_include': True, - 'maxrunning': 1, - 'return_job': False, - 'kwargs': {'master': self.opts['master'], - 'connected': True} + u'function': u'status.master', + u'seconds': self.opts[u'master_alive_interval'], + u'jid_include': True, + u'maxrunning': 1, + u'return_job': False, + u'kwargs': {u'master': self.opts[u'master'], + u'connected': True} } }, persist=True) - if self.opts['master_failback'] and \ - 'master_list' in self.opts and \ - self.opts['master'] != self.opts['master_list'][0]: + if self.opts[u'master_failback'] and \ + u'master_list' in self.opts and \ + self.opts[u'master'] != self.opts[u'master_list'][0]: self.schedule.add_job({ - master_event(type='failback'): + master_event(type=u'failback'): { - 'function': 'status.ping_master', - 'seconds': self.opts['master_failback_interval'], - 'jid_include': True, - 'maxrunning': 1, - 'return_job': False, - 'kwargs': {'master': self.opts['master_list'][0]} + u'function': u'status.ping_master', + u'seconds': self.opts[u'master_failback_interval'], + u'jid_include': True, + u'maxrunning': 1, + u'return_job': False, + u'kwargs': {u'master': self.opts[u'master_list'][0]} } }, persist=True) else: - self.schedule.delete_job(master_event(type='failback'), persist=True) + self.schedule.delete_job(master_event(type=u'failback'), persist=True) else: - self.schedule.delete_job(master_event(type='alive', master=self.opts['master']), persist=True) - self.schedule.delete_job(master_event(type='failback'), persist=True) + self.schedule.delete_job(master_event(type=u'alive', master=self.opts[u'master']), persist=True) + self.schedule.delete_job(master_event(type=u'failback'), persist=True) - self.grains_cache = self.opts['grains'] + self.grains_cache = self.opts[u'grains'] self.ready = True def _return_retry_timer(self): @@ -1140,26 +1181,28 @@ class Minion(MinionBase): Based on the minion configuration, either return a randomized timer or just return the value of the return_retry_timer. ''' - msg = 'Minion return retry timer set to {0} seconds' - if self.opts.get('return_retry_timer_max'): + msg = u'Minion return retry timer set to {0} seconds' + # future lint: disable=str-format-in-logging + if self.opts.get(u'return_retry_timer_max'): try: - random_retry = randint(self.opts['return_retry_timer'], self.opts['return_retry_timer_max']) - log.debug(msg.format(random_retry) + ' (randomized)') + random_retry = randint(self.opts[u'return_retry_timer'], self.opts[u'return_retry_timer_max']) + log.debug(msg.format(random_retry) + u' (randomized)') return random_retry except ValueError: # Catch wiseguys using negative integers here log.error( - 'Invalid value (return_retry_timer: {0} or return_retry_timer_max: {1})' - 'both must be a positive integers'.format( - self.opts['return_retry_timer'], - self.opts['return_retry_timer_max'], - ) + u'Invalid value (return_retry_timer: %s or ' + u'return_retry_timer_max: %s). Both must be positive ' + u'integers.', + self.opts[u'return_retry_timer'], + self.opts[u'return_retry_timer_max'], ) - log.debug(msg.format(DEFAULT_MINION_OPTS['return_retry_timer'])) - return DEFAULT_MINION_OPTS['return_retry_timer'] + log.debug(msg.format(DEFAULT_MINION_OPTS[u'return_retry_timer'])) + return DEFAULT_MINION_OPTS[u'return_retry_timer'] else: - log.debug(msg.format(self.opts.get('return_retry_timer'))) - return self.opts.get('return_retry_timer') + log.debug(msg.format(self.opts.get(u'return_retry_timer'))) + return self.opts.get(u'return_retry_timer') + # future lint: enable=str-format-in-logging def _prep_mod_opts(self): ''' @@ -1167,7 +1210,7 @@ class Minion(MinionBase): ''' mod_opts = {} for key, val in six.iteritems(self.opts): - if key == 'logger': + if key == u'logger': continue mod_opts[key] = val return mod_opts @@ -1181,30 +1224,33 @@ class Minion(MinionBase): # a memory limit on module imports # this feature ONLY works on *nix like OSs (resource module doesn't work on windows) modules_max_memory = False - if self.opts.get('modules_max_memory', -1) > 0 and HAS_PSUTIL and HAS_RESOURCE: - log.debug('modules_max_memory set, enforcing a maximum of {0}'.format(self.opts['modules_max_memory'])) + if self.opts.get(u'modules_max_memory', -1) > 0 and HAS_PSUTIL and HAS_RESOURCE: + log.debug( + u'modules_max_memory set, enforcing a maximum of %s', + self.opts[u'modules_max_memory'] + ) modules_max_memory = True old_mem_limit = resource.getrlimit(resource.RLIMIT_AS) rss, vms = psutil.Process(os.getpid()).memory_info() - mem_limit = rss + vms + self.opts['modules_max_memory'] + mem_limit = rss + vms + self.opts[u'modules_max_memory'] resource.setrlimit(resource.RLIMIT_AS, (mem_limit, mem_limit)) - elif self.opts.get('modules_max_memory', -1) > 0: + elif self.opts.get(u'modules_max_memory', -1) > 0: if not HAS_PSUTIL: - log.error('Unable to enforce modules_max_memory because psutil is missing') + log.error(u'Unable to enforce modules_max_memory because psutil is missing') if not HAS_RESOURCE: - log.error('Unable to enforce modules_max_memory because resource is missing') + log.error(u'Unable to enforce modules_max_memory because resource is missing') # This might be a proxy minion - if hasattr(self, 'proxy'): + if hasattr(self, u'proxy'): proxy = self.proxy else: proxy = None if grains is None: - self.opts['grains'] = salt.loader.grains(self.opts, force_refresh, proxy=proxy) + self.opts[u'grains'] = salt.loader.grains(self.opts, force_refresh, proxy=proxy) self.utils = salt.loader.utils(self.opts, proxy=proxy) - if self.opts.get('multimaster', False): + if self.opts.get(u'multimaster', False): s_opts = copy.deepcopy(self.opts) functions = salt.loader.minion_mods(s_opts, utils=self.utils, proxy=proxy, loaded_base_name=self.loaded_base_name, notify=notify) @@ -1212,9 +1258,9 @@ class Minion(MinionBase): functions = salt.loader.minion_mods(self.opts, utils=self.utils, notify=notify, proxy=proxy) returners = salt.loader.returners(self.opts, functions, proxy=proxy) errors = {} - if '_errors' in functions: - errors = functions['_errors'] - functions.pop('_errors') + if u'_errors' in functions: + errors = functions[u'_errors'] + functions.pop(u'_errors') # we're done, reset the limits! if modules_max_memory is True: @@ -1226,11 +1272,11 @@ class Minion(MinionBase): def _send_req_sync(self, load, timeout): - if self.opts['minion_sign_messages']: - log.trace('Signing event to be published onto the bus.') - minion_privkey_path = os.path.join(self.opts['pki_dir'], 'minion.pem') + if self.opts[u'minion_sign_messages']: + log.trace(u'Signing event to be published onto the bus.') + minion_privkey_path = os.path.join(self.opts[u'pki_dir'], u'minion.pem') sig = salt.crypt.sign_message(minion_privkey_path, salt.serializers.msgpack.serialize(load)) - load['sig'] = sig + load[u'sig'] = sig channel = salt.transport.Channel.factory(self.opts) return channel.send(load, timeout=timeout) @@ -1238,49 +1284,51 @@ class Minion(MinionBase): @tornado.gen.coroutine def _send_req_async(self, load, timeout): - if self.opts['minion_sign_messages']: - log.trace('Signing event to be published onto the bus.') - minion_privkey_path = os.path.join(self.opts['pki_dir'], 'minion.pem') + if self.opts[u'minion_sign_messages']: + log.trace(u'Signing event to be published onto the bus.') + minion_privkey_path = os.path.join(self.opts[u'pki_dir'], u'minion.pem') sig = salt.crypt.sign_message(minion_privkey_path, salt.serializers.msgpack.serialize(load)) - load['sig'] = sig + load[u'sig'] = sig channel = salt.transport.client.AsyncReqChannel.factory(self.opts) ret = yield channel.send(load, timeout=timeout) raise tornado.gen.Return(ret) - def _fire_master(self, data=None, tag=None, events=None, pretag=None, timeout=60, sync=True): + def _fire_master(self, data=None, tag=None, events=None, pretag=None, timeout=60, sync=True, timeout_handler=None): ''' Fire an event on the master, or drop message if unable to send. ''' - load = {'id': self.opts['id'], - 'cmd': '_minion_event', - 'pretag': pretag, - 'tok': self.tok} + load = {u'id': self.opts[u'id'], + u'cmd': u'_minion_event', + u'pretag': pretag, + u'tok': self.tok} if events: - load['events'] = events + load[u'events'] = events elif data and tag: - load['data'] = data - load['tag'] = tag + load[u'data'] = data + load[u'tag'] = tag elif not data and tag: - load['data'] = {} - load['tag'] = tag + load[u'data'] = {} + load[u'tag'] = tag else: return - def timeout_handler(*_): - log.info('fire_master failed: master could not be contacted. Request timed out.') - return True - if sync: try: self._send_req_sync(load, timeout) except salt.exceptions.SaltReqTimeoutError: - log.info('fire_master failed: master could not be contacted. Request timed out.') + log.info(u'fire_master failed: master could not be contacted. Request timed out.') return False except Exception: - log.info('fire_master failed: {0}'.format(traceback.format_exc())) + log.info(u'fire_master failed: %s', traceback.format_exc()) return False else: + if timeout_handler is None: + def handle_timeout(*_): + log.info(u'fire_master failed: master could not be contacted. Request timed out.') + return True + timeout_handler = handle_timeout + with tornado.stack_context.ExceptionStackContext(timeout_handler): self._send_req_async(load, timeout, callback=lambda f: None) # pylint: disable=unexpected-keyword-arg return True @@ -1290,29 +1338,30 @@ class Minion(MinionBase): Override this method if you wish to handle the decoded data differently. ''' - if 'user' in data: + if u'user' in data: log.info( - 'User {0[user]} Executing command {0[fun]} with jid ' - '{0[jid]}'.format(data) + u'User %s Executing command %s with jid %s', + data[u'user'], data[u'fun'], data[u'jid'] ) else: log.info( - 'Executing command {0[fun]} with jid {0[jid]}'.format(data) + u'Executing command %s with jid %s', + data[u'fun'], data[u'jid'] ) - log.debug('Command details {0}'.format(data)) + log.debug(u'Command details %s', data) # Don't duplicate jobs - log.trace('Started JIDs: {0}'.format(self.jid_queue)) + log.trace(u'Started JIDs: %s', self.jid_queue) if self.jid_queue is not None: - if data['jid'] in self.jid_queue: + if data[u'jid'] in self.jid_queue: return else: - self.jid_queue.append(data['jid']) - if len(self.jid_queue) > self.opts['minion_jid_queue_hwm']: + self.jid_queue.append(data[u'jid']) + if len(self.jid_queue) > self.opts[u'minion_jid_queue_hwm']: self.jid_queue.pop(0) - if isinstance(data['fun'], six.string_types): - if data['fun'] == 'sys.reload_modules': + if isinstance(data[u'fun'], six.string_types): + if data[u'fun'] == u'sys.reload_modules': self.functions, self.returners, self.function_errors, self.executors = self._load_modules() self.schedule.functions = self.functions self.schedule.returners = self.returners @@ -1321,9 +1370,9 @@ class Minion(MinionBase): # python needs to be able to reconstruct the reference on the other # side. instance = self - multiprocessing_enabled = self.opts.get('multiprocessing', True) + multiprocessing_enabled = self.opts.get(u'multiprocessing', True) if multiprocessing_enabled: - if sys.platform.startswith('win'): + if sys.platform.startswith(u'win'): # let python reconstruct the minion on the other side if we're # running on windows instance = None @@ -1335,7 +1384,7 @@ class Minion(MinionBase): process = threading.Thread( target=self._target, args=(instance, self.opts, data, self.connected), - name=data['jid'] + name=data[u'jid'] ) if multiprocessing_enabled: @@ -1347,14 +1396,15 @@ class Minion(MinionBase): process.start() # TODO: remove the windows specific check? - if multiprocessing_enabled and not salt.utils.is_windows(): + if multiprocessing_enabled and not salt.utils.platform.is_windows(): # we only want to join() immediately if we are daemonizing a process process.join() else: self.win_proc.append(process) def ctx(self): - '''Return a single context manager for the minion's data + ''' + Return a single context manager for the minion's data ''' if six.PY2: return contextlib.nested( @@ -1374,24 +1424,24 @@ class Minion(MinionBase): if not minion_instance: minion_instance = cls(opts) minion_instance.connected = connected - if not hasattr(minion_instance, 'functions'): + if not hasattr(minion_instance, u'functions'): functions, returners, function_errors, executors = ( - minion_instance._load_modules(grains=opts['grains']) + minion_instance._load_modules(grains=opts[u'grains']) ) minion_instance.functions = functions minion_instance.returners = returners minion_instance.function_errors = function_errors minion_instance.executors = executors - if not hasattr(minion_instance, 'serial'): + if not hasattr(minion_instance, u'serial'): minion_instance.serial = salt.payload.Serial(opts) - if not hasattr(minion_instance, 'proc_dir'): - uid = salt.utils.get_uid(user=opts.get('user', None)) + if not hasattr(minion_instance, u'proc_dir'): + uid = salt.utils.get_uid(user=opts.get(u'user', None)) minion_instance.proc_dir = ( - get_proc_dir(opts['cachedir'], uid=uid) + get_proc_dir(opts[u'cachedir'], uid=uid) ) with tornado.stack_context.StackContext(minion_instance.ctx): - if isinstance(data['fun'], tuple) or isinstance(data['fun'], list): + if isinstance(data[u'fun'], tuple) or isinstance(data[u'fun'], list): Minion._thread_multi_return(minion_instance, opts, data) else: Minion._thread_return(minion_instance, opts, data) @@ -1402,9 +1452,9 @@ class Minion(MinionBase): This method should be used as a threading target, start the actual minion side execution. ''' - fn_ = os.path.join(minion_instance.proc_dir, data['jid']) + fn_ = os.path.join(minion_instance.proc_dir, data[u'jid']) - if opts['multiprocessing'] and not salt.utils.is_windows(): + if opts[u'multiprocessing'] and not salt.utils.platform.is_windows(): # Shutdown the multiprocessing before daemonizing salt.log.setup.shutdown_multiprocessing_logging() @@ -1413,45 +1463,53 @@ class Minion(MinionBase): # Reconfigure multiprocessing logging after daemonizing salt.log.setup.setup_multiprocessing_logging() - salt.utils.appendproctitle('{0}._thread_return {1}'.format(cls.__name__, data['jid'])) + salt.utils.appendproctitle(u'{0}._thread_return {1}'.format(cls.__name__, data[u'jid'])) - sdata = {'pid': os.getpid()} + sdata = {u'pid': os.getpid()} sdata.update(data) - log.info('Starting a new job with PID {0}'.format(sdata['pid'])) - with salt.utils.fopen(fn_, 'w+b') as fp_: + log.info(u'Starting a new job with PID %s', sdata[u'pid']) + with salt.utils.files.fopen(fn_, u'w+b') as fp_: fp_.write(minion_instance.serial.dumps(sdata)) - ret = {'success': False} - function_name = data['fun'] + ret = {u'success': False} + function_name = data[u'fun'] if function_name in minion_instance.functions: try: - if minion_instance.connected and minion_instance.opts['pillar'].get('minion_blackout', False): - # this minion is blacked out. Only allow saltutil.refresh_pillar - if function_name != 'saltutil.refresh_pillar' and \ - function_name not in minion_instance.opts['pillar'].get('minion_blackout_whitelist', []): - raise SaltInvocationError('Minion in blackout mode. Set \'minion_blackout\' ' - 'to False in pillar to resume operations. Only ' - 'saltutil.refresh_pillar allowed in blackout mode.') + minion_blackout_violation = False + if minion_instance.connected and minion_instance.opts[u'pillar'].get(u'minion_blackout', False): + whitelist = minion_instance.opts[u'pillar'].get(u'minion_blackout_whitelist', []) + # this minion is blacked out. Only allow saltutil.refresh_pillar and the whitelist + if function_name != u'saltutil.refresh_pillar' and function_name not in whitelist: + minion_blackout_violation = True + elif minion_instance.opts[u'grains'].get(u'minion_blackout', False): + whitelist = minion_instance.opts[u'grains'].get(u'minion_blackout_whitelist', []) + if function_name != u'saltutil.refresh_pillar' and function_name not in whitelist: + minion_blackout_violation = True + if minion_blackout_violation: + raise SaltInvocationError(u'Minion in blackout mode. Set \'minion_blackout\' ' + u'to False in pillar or grains to resume operations. Only ' + u'saltutil.refresh_pillar allowed in blackout mode.') + func = minion_instance.functions[function_name] args, kwargs = load_args_and_kwargs( func, - data['arg'], + data[u'arg'], data) - minion_instance.functions.pack['__context__']['retcode'] = 0 + minion_instance.functions.pack[u'__context__'][u'retcode'] = 0 - executors = data.get('module_executors') or opts.get('module_executors', ['direct_call']) + executors = data.get(u'module_executors') or opts.get(u'module_executors', [u'direct_call']) if isinstance(executors, six.string_types): executors = [executors] elif not isinstance(executors, list) or not executors: - raise SaltInvocationError("Wrong executors specification: {0}. String or non-empty list expected". + raise SaltInvocationError(u"Wrong executors specification: {0}. String or non-empty list expected". format(executors)) - if opts.get('sudo_user', '') and executors[-1] != 'sudo': - executors[-1] = 'sudo' # replace the last one with sudo - log.trace('Executors list {0}'.format(executors)) # pylint: disable=no-member + if opts.get(u'sudo_user', u'') and executors[-1] != u'sudo': + executors[-1] = u'sudo' # replace the last one with sudo + log.trace(u'Executors list %s', executors) # pylint: disable=no-member for name in executors: - fname = '{0}.execute'.format(name) + fname = u'{0}.execute'.format(name) if fname not in minion_instance.executors: - raise SaltInvocationError("Executor '{0}' is not available".format(name)) + raise SaltInvocationError(u"Executor '{0}' is not available".format(name)) return_data = minion_instance.executors[fname](opts, data, func, args, kwargs) if return_data is not None: break @@ -1466,79 +1524,80 @@ class Minion(MinionBase): if not iret: iret = [] iret.append(single) - tag = tagify([data['jid'], 'prog', opts['id'], str(ind)], 'job') - event_data = {'return': single} + tag = tagify([data[u'jid'], u'prog', opts[u'id'], str(ind)], u'job') + event_data = {u'return': single} minion_instance._fire_master(event_data, tag) ind += 1 - ret['return'] = iret + ret[u'return'] = iret else: - ret['return'] = return_data - ret['retcode'] = minion_instance.functions.pack['__context__'].get( - 'retcode', + ret[u'return'] = return_data + ret[u'retcode'] = minion_instance.functions.pack[u'__context__'].get( + u'retcode', 0 ) - ret['success'] = True + ret[u'success'] = True except CommandNotFoundError as exc: - msg = 'Command required for \'{0}\' not found'.format( + msg = u'Command required for \'{0}\' not found'.format( function_name ) log.debug(msg, exc_info=True) - ret['return'] = '{0}: {1}'.format(msg, exc) - ret['out'] = 'nested' + ret[u'return'] = u'{0}: {1}'.format(msg, exc) + ret[u'out'] = u'nested' except CommandExecutionError as exc: log.error( - 'A command in \'{0}\' had a problem: {1}'.format( - function_name, - exc - ), + u'A command in \'%s\' had a problem: %s', + function_name, exc, exc_info_on_loglevel=logging.DEBUG ) - ret['return'] = 'ERROR: {0}'.format(exc) - ret['out'] = 'nested' + ret[u'return'] = u'ERROR: {0}'.format(exc) + ret[u'out'] = u'nested' except SaltInvocationError as exc: log.error( - 'Problem executing \'{0}\': {1}'.format( - function_name, - exc - ), + u'Problem executing \'%s\': %s', + function_name, exc, exc_info_on_loglevel=logging.DEBUG ) - ret['return'] = 'ERROR executing \'{0}\': {1}'.format( + ret[u'return'] = u'ERROR executing \'{0}\': {1}'.format( function_name, exc ) - ret['out'] = 'nested' + ret[u'out'] = u'nested' except TypeError as exc: - msg = 'Passed invalid arguments to {0}: {1}\n{2}'.format(function_name, exc, func.__doc__, ) + msg = u'Passed invalid arguments to {0}: {1}\n{2}'.format(function_name, exc, func.__doc__) log.warning(msg, exc_info_on_loglevel=logging.DEBUG) - ret['return'] = msg - ret['out'] = 'nested' + ret[u'return'] = msg + ret[u'out'] = u'nested' except Exception: - msg = 'The minion function caused an exception' + msg = u'The minion function caused an exception' log.warning(msg, exc_info_on_loglevel=True) salt.utils.error.fire_exception(salt.exceptions.MinionError(msg), opts, job=data) - ret['return'] = '{0}: {1}'.format(msg, traceback.format_exc()) - ret['out'] = 'nested' + ret[u'return'] = u'{0}: {1}'.format(msg, traceback.format_exc()) + ret[u'out'] = u'nested' else: - ret['return'] = minion_instance.functions.missing_fun_string(function_name) - mod_name = function_name.split('.')[0] - if mod_name in minion_instance.function_errors: - ret['return'] += ' Possible reasons: \'{0}\''.format( - minion_instance.function_errors[mod_name] - ) - ret['success'] = False - ret['retcode'] = 254 - ret['out'] = 'nested' - - ret['jid'] = data['jid'] - ret['fun'] = data['fun'] - ret['fun_args'] = data['arg'] - if 'master_id' in data: - ret['master_id'] = data['master_id'] - if 'metadata' in data: - if isinstance(data['metadata'], dict): - ret['metadata'] = data['metadata'] + docs = minion_instance.functions[u'sys.doc'](u'{0}*'.format(function_name)) + if docs: + docs[function_name] = minion_instance.functions.missing_fun_string(function_name) + ret[u'return'] = docs else: - log.warning('The metadata parameter must be a dictionary. Ignoring.') + ret[u'return'] = minion_instance.functions.missing_fun_string(function_name) + mod_name = function_name.split('.')[0] + if mod_name in minion_instance.function_errors: + ret[u'return'] += u' Possible reasons: \'{0}\''.format( + minion_instance.function_errors[mod_name] + ) + ret[u'success'] = False + ret[u'retcode'] = 254 + ret[u'out'] = u'nested' + + ret[u'jid'] = data[u'jid'] + ret[u'fun'] = data[u'fun'] + ret[u'fun_args'] = data[u'arg'] + if u'master_id' in data: + ret[u'master_id'] = data[u'master_id'] + if u'metadata' in data: + if isinstance(data[u'metadata'], dict): + ret[u'metadata'] = data[u'metadata'] + else: + log.warning(u'The metadata parameter must be a dictionary. Ignoring.') if minion_instance.connected: minion_instance._return_pub( ret, @@ -1547,37 +1606,35 @@ class Minion(MinionBase): # Add default returners from minion config # Should have been coverted to comma-delimited string already - if isinstance(opts.get('return'), six.string_types): - if data['ret']: - data['ret'] = ','.join((data['ret'], opts['return'])) + if isinstance(opts.get(u'return'), six.string_types): + if data[u'ret']: + data[u'ret'] = u','.join((data[u'ret'], opts[u'return'])) else: - data['ret'] = opts['return'] + data[u'ret'] = opts[u'return'] - log.debug('minion return: %s', ret) + log.debug(u'minion return: %s', ret) # TODO: make a list? Seems odd to split it this late :/ - if data['ret'] and isinstance(data['ret'], six.string_types): - if 'ret_config' in data: - ret['ret_config'] = data['ret_config'] - if 'ret_kwargs' in data: - ret['ret_kwargs'] = data['ret_kwargs'] - ret['id'] = opts['id'] - for returner in set(data['ret'].split(',')): + if data[u'ret'] and isinstance(data[u'ret'], six.string_types): + if u'ret_config' in data: + ret[u'ret_config'] = data[u'ret_config'] + if u'ret_kwargs' in data: + ret[u'ret_kwargs'] = data[u'ret_kwargs'] + ret[u'id'] = opts[u'id'] + for returner in set(data[u'ret'].split(u',')): try: - returner_str = '{0}.returner'.format(returner) + returner_str = u'{0}.returner'.format(returner) if returner_str in minion_instance.returners: minion_instance.returners[returner_str](ret) else: returner_err = minion_instance.returners.missing_fun_string(returner_str) - log.error('Returner {0} could not be loaded: {1}'.format( - returner_str, returner_err)) - except Exception as exc: - log.error( - 'The return failed for job {0} {1}'.format( - data['jid'], - exc + log.error( + u'Returner %s could not be loaded: %s', + returner_str, returner_err ) + except Exception as exc: + log.exception( + u'The return failed for job %s: %s', data[u'jid'], exc ) - log.error(traceback.format_exc()) @classmethod def _thread_multi_return(cls, minion_instance, opts, data): @@ -1585,78 +1642,81 @@ class Minion(MinionBase): This method should be used as a threading target, start the actual minion side execution. ''' - salt.utils.appendproctitle('{0}._thread_multi_return {1}'.format(cls.__name__, data['jid'])) + salt.utils.appendproctitle(u'{0}._thread_multi_return {1}'.format(cls.__name__, data[u'jid'])) ret = { - 'return': {}, - 'retcode': {}, - 'success': {} + u'return': {}, + u'retcode': {}, + u'success': {} } - for ind in range(0, len(data['fun'])): - ret['success'][data['fun'][ind]] = False + for ind in range(0, len(data[u'fun'])): + ret[u'success'][data[u'fun'][ind]] = False try: - if minion_instance.connected and minion_instance.opts['pillar'].get('minion_blackout', False): - # this minion is blacked out. Only allow saltutil.refresh_pillar - if data['fun'][ind] != 'saltutil.refresh_pillar' and \ - data['fun'][ind] not in minion_instance.opts['pillar'].get('minion_blackout_whitelist', []): - raise SaltInvocationError('Minion in blackout mode. Set \'minion_blackout\' ' - 'to False in pillar to resume operations. Only ' - 'saltutil.refresh_pillar allowed in blackout mode.') - func = minion_instance.functions[data['fun'][ind]] + minion_blackout_violation = False + if minion_instance.connected and minion_instance.opts[u'pillar'].get(u'minion_blackout', False): + whitelist = minion_instance.opts[u'pillar'].get(u'minion_blackout_whitelist', []) + # this minion is blacked out. Only allow saltutil.refresh_pillar and the whitelist + if data[u'fun'][ind] != u'saltutil.refresh_pillar' and data[u'fun'][ind] not in whitelist: + minion_blackout_violation = True + elif minion_instance.opts[u'grains'].get(u'minion_blackout', False): + whitelist = minion_instance.opts[u'grains'].get(u'minion_blackout_whitelist', []) + if data[u'fun'][ind] != u'saltutil.refresh_pillar' and data[u'fun'][ind] not in whitelist: + minion_blackout_violation = True + if minion_blackout_violation: + raise SaltInvocationError(u'Minion in blackout mode. Set \'minion_blackout\' ' + u'to False in pillar or grains to resume operations. Only ' + u'saltutil.refresh_pillar allowed in blackout mode.') + + func = minion_instance.functions[data[u'fun'][ind]] + args, kwargs = load_args_and_kwargs( func, - data['arg'][ind], + data[u'arg'][ind], data) - minion_instance.functions.pack['__context__']['retcode'] = 0 - ret['return'][data['fun'][ind]] = func(*args, **kwargs) - ret['retcode'][data['fun'][ind]] = minion_instance.functions.pack['__context__'].get( - 'retcode', + minion_instance.functions.pack[u'__context__'][u'retcode'] = 0 + ret[u'return'][data[u'fun'][ind]] = func(*args, **kwargs) + ret[u'retcode'][data[u'fun'][ind]] = minion_instance.functions.pack[u'__context__'].get( + u'retcode', 0 ) - ret['success'][data['fun'][ind]] = True + ret[u'success'][data[u'fun'][ind]] = True except Exception as exc: trb = traceback.format_exc() - log.warning( - 'The minion function caused an exception: {0}'.format( - exc - ) - ) - ret['return'][data['fun'][ind]] = trb - ret['jid'] = data['jid'] - ret['fun'] = data['fun'] - ret['fun_args'] = data['arg'] - if 'metadata' in data: - ret['metadata'] = data['metadata'] + log.warning(u'The minion function caused an exception: %s', exc) + ret[u'return'][data[u'fun'][ind]] = trb + ret[u'jid'] = data[u'jid'] + ret[u'fun'] = data[u'fun'] + ret[u'fun_args'] = data[u'arg'] + if u'metadata' in data: + ret[u'metadata'] = data[u'metadata'] if minion_instance.connected: minion_instance._return_pub( ret, timeout=minion_instance._return_retry_timer() ) - if data['ret']: - if 'ret_config' in data: - ret['ret_config'] = data['ret_config'] - if 'ret_kwargs' in data: - ret['ret_kwargs'] = data['ret_kwargs'] - for returner in set(data['ret'].split(',')): - ret['id'] = opts['id'] + if data[u'ret']: + if u'ret_config' in data: + ret[u'ret_config'] = data[u'ret_config'] + if u'ret_kwargs' in data: + ret[u'ret_kwargs'] = data[u'ret_kwargs'] + for returner in set(data[u'ret'].split(u',')): + ret[u'id'] = opts[u'id'] try: - minion_instance.returners['{0}.returner'.format( + minion_instance.returners[u'{0}.returner'.format( returner )](ret) except Exception as exc: log.error( - 'The return failed for job {0} {1}'.format( - data['jid'], - exc - ) + u'The return failed for job %s: %s', + data[u'jid'], exc ) - def _return_pub(self, ret, ret_cmd='_return', timeout=60, sync=True): + def _return_pub(self, ret, ret_cmd=u'_return', timeout=60, sync=True): ''' Return the data from the executed command to the master server ''' - jid = ret.get('jid', ret.get('__jid__')) - fun = ret.get('fun', ret.get('__fun__')) - if self.opts['multiprocessing']: + jid = ret.get(u'jid', ret.get(u'__jid__')) + fun = ret.get(u'fun', ret.get(u'__fun__')) + if self.opts[u'multiprocessing']: fn_ = os.path.join(self.proc_dir, jid) if os.path.isfile(fn_): try: @@ -1664,35 +1724,37 @@ class Minion(MinionBase): except (OSError, IOError): # The file is gone already pass - log.info('Returning information for job: {0}'.format(jid)) - if ret_cmd == '_syndic_return': - load = {'cmd': ret_cmd, - 'id': self.opts['id'], - 'jid': jid, - 'fun': fun, - 'arg': ret.get('arg'), - 'tgt': ret.get('tgt'), - 'tgt_type': ret.get('tgt_type'), - 'load': ret.get('__load__')} - if '__master_id__' in ret: - load['master_id'] = ret['__master_id__'] - load['return'] = {} + log.info(u'Returning information for job: %s', jid) + if ret_cmd == u'_syndic_return': + load = {u'cmd': ret_cmd, + u'id': self.opts[u'uid'], + u'jid': jid, + u'fun': fun, + u'arg': ret.get(u'arg'), + u'tgt': ret.get(u'tgt'), + u'tgt_type': ret.get(u'tgt_type'), + u'load': ret.get(u'__load__')} + if u'__master_id__' in ret: + load[u'master_id'] = ret[u'__master_id__'] + load[u'return'] = {} for key, value in six.iteritems(ret): - if key.startswith('__'): + if key.startswith(u'__'): continue - load['return'][key] = value + load[u'return'][key] = value else: - load = {'cmd': ret_cmd, - 'id': self.opts['id']} + load = {u'cmd': ret_cmd, + u'id': self.opts[u'id']} for key, value in six.iteritems(ret): load[key] = value - if 'out' in ret: - if isinstance(ret['out'], six.string_types): - load['out'] = ret['out'] + if u'out' in ret: + if isinstance(ret[u'out'], six.string_types): + load[u'out'] = ret[u'out'] else: - log.error('Invalid outputter {0}. This is likely a bug.' - .format(ret['out'])) + log.error( + u'Invalid outputter %s. This is likely a bug.', + ret[u'out'] + ) else: try: oput = self.functions[fun].__outputter__ @@ -1700,20 +1762,21 @@ class Minion(MinionBase): pass else: if isinstance(oput, six.string_types): - load['out'] = oput - if self.opts['cache_jobs']: + load[u'out'] = oput + if self.opts[u'cache_jobs']: # Local job cache has been enabled - salt.utils.minion.cache_jobs(self.opts, load['jid'], ret) + salt.utils.minion.cache_jobs(self.opts, load[u'jid'], ret) - if not self.opts['pub_ret']: - return '' + if not self.opts[u'pub_ret']: + return u'' def timeout_handler(*_): - msg = ('The minion failed to return the job information for job ' - '{0}. This is often due to the master being shut down or ' - 'overloaded. If the master is running consider increasing ' - 'the worker_threads value.').format(jid) - log.warning(msg) + log.warning( + u'The minion failed to return the job information for job %s. ' + u'This is often due to the master being shut down or ' + u'overloaded. If the master is running, consider increasing ' + u'the worker_threads value.', jid + ) return True if sync: @@ -1721,35 +1784,37 @@ class Minion(MinionBase): ret_val = self._send_req_sync(load, timeout=timeout) except SaltReqTimeoutError: timeout_handler() - return '' + return u'' else: with tornado.stack_context.ExceptionStackContext(timeout_handler): ret_val = self._send_req_async(load, timeout=timeout, callback=lambda f: None) # pylint: disable=unexpected-keyword-arg - log.trace('ret_val = {0}'.format(ret_val)) # pylint: disable=no-member + log.trace(u'ret_val = %s', ret_val) # pylint: disable=no-member return ret_val def _state_run(self): ''' Execute a state run based on information set in the minion config file ''' - if self.opts['startup_states']: - if self.opts.get('master_type', 'str') == 'disable' and \ - self.opts.get('file_client', 'remote') == 'remote': - log.warning('Cannot run startup_states when \'master_type\' is ' - 'set to \'disable\' and \'file_client\' is set to ' - '\'remote\'. Skipping.') + if self.opts[u'startup_states']: + if self.opts.get(u'master_type', u'str') == u'disable' and \ + self.opts.get(u'file_client', u'remote') == u'remote': + log.warning( + u'Cannot run startup_states when \'master_type\' is set ' + u'to \'disable\' and \'file_client\' is set to ' + u'\'remote\'. Skipping.' + ) else: - data = {'jid': 'req', 'ret': self.opts.get('ext_job_cache', '')} - if self.opts['startup_states'] == 'sls': - data['fun'] = 'state.sls' - data['arg'] = [self.opts['sls_list']] - elif self.opts['startup_states'] == 'top': - data['fun'] = 'state.top' - data['arg'] = [self.opts['top_file']] + data = {u'jid': u'req', u'ret': self.opts.get(u'ext_job_cache', u'')} + if self.opts[u'startup_states'] == u'sls': + data[u'fun'] = u'state.sls' + data[u'arg'] = [self.opts[u'sls_list']] + elif self.opts[u'startup_states'] == u'top': + data[u'fun'] = u'state.top' + data[u'arg'] = [self.opts[u'top_file']] else: - data['fun'] = 'state.highstate' - data['arg'] = [] + data[u'fun'] = u'state.highstate' + data[u'arg'] = [] self._handle_decoded_payload(data) def _refresh_grains_watcher(self, refresh_interval_in_minutes): @@ -1758,41 +1823,41 @@ class Minion(MinionBase): :param refresh_interval_in_minutes: :return: None ''' - if '__update_grains' not in self.opts.get('schedule', {}): - if 'schedule' not in self.opts: - self.opts['schedule'] = {} - self.opts['schedule'].update({ - '__update_grains': + if u'__update_grains' not in self.opts.get(u'schedule', {}): + if u'schedule' not in self.opts: + self.opts[u'schedule'] = {} + self.opts[u'schedule'].update({ + u'__update_grains': { - 'function': 'event.fire', - 'args': [{}, 'grains_refresh'], - 'minutes': refresh_interval_in_minutes + u'function': u'event.fire', + u'args': [{}, u'grains_refresh'], + u'minutes': refresh_interval_in_minutes } }) def _fire_master_minion_start(self): # Send an event to the master that the minion is live self._fire_master( - 'Minion {0} started at {1}'.format( - self.opts['id'], + u'Minion {0} started at {1}'.format( + self.opts[u'id'], time.asctime() ), - 'minion_start' + u'minion_start' ) # dup name spaced event self._fire_master( - 'Minion {0} started at {1}'.format( - self.opts['id'], + u'Minion {0} started at {1}'.format( + self.opts[u'id'], time.asctime() ), - tagify([self.opts['id'], 'start'], 'minion'), + tagify([self.opts[u'id'], u'start'], u'minion'), ) def module_refresh(self, force_refresh=False, notify=False): ''' Refresh the functions and returners. ''' - log.debug('Refreshing modules. Notify={0}'.format(notify)) + log.debug(u'Refreshing modules. Notify=%s', notify) self.functions, self.returners, _, self.executors = self._load_modules(force_refresh, notify=notify) self.schedule.functions = self.functions @@ -1802,7 +1867,7 @@ class Minion(MinionBase): ''' Refresh the functions and returners. ''' - log.debug('Refreshing beacons.') + log.debug(u'Refreshing beacons.') self.beacons = salt.beacons.Beacon(self.opts, self.functions) # TODO: only allow one future in flight at a time? @@ -1812,89 +1877,93 @@ class Minion(MinionBase): Refresh the pillar ''' if self.connected: - log.debug('Refreshing pillar') + log.debug(u'Refreshing pillar') try: - self.opts['pillar'] = yield salt.pillar.get_async_pillar( + self.opts[u'pillar'] = yield salt.pillar.get_async_pillar( self.opts, - self.opts['grains'], - self.opts['id'], - self.opts['environment'], - pillarenv=self.opts.get('pillarenv'), + self.opts[u'grains'], + self.opts[u'id'], + self.opts[u'environment'], + pillarenv=self.opts.get(u'pillarenv'), ).compile_pillar() except SaltClientError: # Do not exit if a pillar refresh fails. - log.error('Pillar data could not be refreshed. ' - 'One or more masters may be down!') + log.error(u'Pillar data could not be refreshed. ' + u'One or more masters may be down!') self.module_refresh(force_refresh) def manage_schedule(self, tag, data): ''' Refresh the functions and returners. ''' - func = data.get('func', None) - name = data.get('name', None) - schedule = data.get('schedule', None) - where = data.get('where', None) - persist = data.get('persist', None) + func = data.get(u'func', None) + name = data.get(u'name', None) + schedule = data.get(u'schedule', None) + where = data.get(u'where', None) + persist = data.get(u'persist', None) - if func == 'delete': + if func == u'delete': self.schedule.delete_job(name, persist) - elif func == 'add': + elif func == u'add': self.schedule.add_job(schedule, persist) - elif func == 'modify': + elif func == u'modify': self.schedule.modify_job(name, schedule, persist) - elif func == 'enable': + elif func == u'enable': self.schedule.enable_schedule() - elif func == 'disable': + elif func == u'disable': self.schedule.disable_schedule() - elif func == 'enable_job': + elif func == u'enable_job': self.schedule.enable_job(name, persist) - elif func == 'run_job': + elif func == u'run_job': self.schedule.run_job(name) - elif func == 'disable_job': + elif func == u'disable_job': self.schedule.disable_job(name, persist) - elif func == 'reload': + elif func == u'reload': self.schedule.reload(schedule) - elif func == 'list': + elif func == u'list': self.schedule.list(where) - elif func == 'save_schedule': + elif func == u'save_schedule': self.schedule.save_schedule() def manage_beacons(self, tag, data): ''' Manage Beacons ''' - func = data.get('func', None) - name = data.get('name', None) - beacon_data = data.get('beacon_data', None) + func = data.get(u'func', None) + name = data.get(u'name', None) + beacon_data = data.get(u'beacon_data', None) - if func == 'add': + if func == u'add': self.beacons.add_beacon(name, beacon_data) - elif func == 'modify': + elif func == u'modify': self.beacons.modify_beacon(name, beacon_data) - elif func == 'delete': + elif func == u'delete': self.beacons.delete_beacon(name) - elif func == 'enable': + elif func == u'enable': self.beacons.enable_beacons() - elif func == 'disable': + elif func == u'disable': self.beacons.disable_beacons() - elif func == 'enable_beacon': + elif func == u'enable_beacon': self.beacons.enable_beacon(name) - elif func == 'disable_beacon': + elif func == u'disable_beacon': self.beacons.disable_beacon(name) - elif func == 'list': + elif func == u'list': self.beacons.list_beacons() + elif func == u'list_available': + self.beacons.list_available_beacons() + elif func == u'validate_beacon': + self.beacons.validate_beacon(name, beacon_data) def environ_setenv(self, tag, data): ''' Set the salt-minion main process environment according to the data contained in the minion event data ''' - environ = data.get('environ', None) + environ = data.get(u'environ', None) if environ is None: return False - false_unsets = data.get('false_unsets', False) - clear_all = data.get('clear_all', False) + false_unsets = data.get(u'false_unsets', False) + clear_all = data.get(u'clear_all', False) import salt.modules.environ as mod_environ return mod_environ.setenv(environ, false_unsets, clear_all) @@ -1907,32 +1976,29 @@ class Minion(MinionBase): self._running = True elif self._running is False: log.error( - 'This {0} was scheduled to stop. Not running ' - '{0}.tune_in()'.format(self.__class__.__name__) + u'This %s was scheduled to stop. Not running %s.tune_in()', + self.__class__.__name__, self.__class__.__name__ ) return elif self._running is True: log.error( - 'This {0} is already running. Not running ' - '{0}.tune_in()'.format(self.__class__.__name__) + u'This %s is already running. Not running %s.tune_in()', + self.__class__.__name__, self.__class__.__name__ ) return try: log.info( - '{0} is starting as user \'{1}\''.format( - self.__class__.__name__, - salt.utils.get_user() - ) + u'%s is starting as user \'%s\'', + self.__class__.__name__, salt.utils.get_user() ) except Exception as err: # Only windows is allowed to fail here. See #3189. Log as debug in # that case. Else, error. log.log( - salt.utils.is_windows() and logging.DEBUG or logging.ERROR, - 'Failed to get the user who is starting {0}'.format( - self.__class__.__name__ - ), + salt.utils.platform.is_windows() and logging.DEBUG or logging.ERROR, + u'Failed to get the user who is starting %s', + self.__class__.__name__, exc_info=err ) @@ -1941,12 +2007,12 @@ class Minion(MinionBase): Send mine data to the master ''' channel = salt.transport.Channel.factory(self.opts) - data['tok'] = self.tok + data[u'tok'] = self.tok try: ret = channel.send(data) return ret except SaltReqTimeoutError: - log.warning('Unable to send mine data to master.') + log.warning(u'Unable to send mine data to master.') return None @tornado.gen.coroutine @@ -1957,79 +2023,83 @@ class Minion(MinionBase): if not self.ready: raise tornado.gen.Return() tag, data = salt.utils.event.SaltEvent.unpack(package) - log.debug('Minion of "{0}" is handling event tag \'{1}\''.format(self.opts['master'], tag)) - if tag.startswith('module_refresh'): + log.debug( + u'Minion of \'%s\' is handling event tag \'%s\'', + self.opts[u'master'], tag + ) + if tag.startswith(u'module_refresh'): self.module_refresh( - force_refresh=data.get('force_refresh', False), - notify=data.get('notify', False) + force_refresh=data.get(u'force_refresh', False), + notify=data.get(u'notify', False) ) - elif tag.startswith('pillar_refresh'): + elif tag.startswith(u'pillar_refresh'): yield self.pillar_refresh( - force_refresh=data.get('force_refresh', False) + force_refresh=data.get(u'force_refresh', False) ) - elif tag.startswith('beacons_refresh'): + elif tag.startswith(u'beacons_refresh'): self.beacons_refresh() - elif tag.startswith('manage_schedule'): + elif tag.startswith(u'manage_schedule'): self.manage_schedule(tag, data) - elif tag.startswith('manage_beacons'): + elif tag.startswith(u'manage_beacons'): self.manage_beacons(tag, data) - elif tag.startswith('grains_refresh'): - if (data.get('force_refresh', False) or - self.grains_cache != self.opts['grains']): + elif tag.startswith(u'grains_refresh'): + if (data.get(u'force_refresh', False) or + self.grains_cache != self.opts[u'grains']): self.pillar_refresh(force_refresh=True) - self.grains_cache = self.opts['grains'] - elif tag.startswith('environ_setenv'): + self.grains_cache = self.opts[u'grains'] + elif tag.startswith(u'environ_setenv'): self.environ_setenv(tag, data) - elif tag.startswith('_minion_mine'): + elif tag.startswith(u'_minion_mine'): self._mine_send(tag, data) - elif tag.startswith('fire_master'): - log.debug('Forwarding master event tag={tag}'.format(tag=data['tag'])) - self._fire_master(data['data'], data['tag'], data['events'], data['pretag']) - elif tag.startswith(master_event(type='disconnected')) or tag.startswith(master_event(type='failback')): + elif tag.startswith(u'fire_master'): + if self.connected: + log.debug(u'Forwarding master event tag=%s', data[u'tag']) + self._fire_master(data[u'data'], data[u'tag'], data[u'events'], data[u'pretag']) + elif tag.startswith(master_event(type=u'disconnected')) or tag.startswith(master_event(type=u'failback')): # if the master disconnect event is for a different master, raise an exception - if tag.startswith(master_event(type='disconnected')) and data['master'] != self.opts['master']: + if tag.startswith(master_event(type=u'disconnected')) and data[u'master'] != self.opts[u'master']: # not mine master, ignore return - if tag.startswith(master_event(type='failback')): + if tag.startswith(master_event(type=u'failback')): # if the master failback event is not for the top master, raise an exception - if data['master'] != self.opts['master_list'][0]: - raise SaltException('Bad master \'{0}\' when mine failback is \'{1}\''.format( - data['master'], self.opts['master'])) + if data[u'master'] != self.opts[u'master_list'][0]: + raise SaltException(u'Bad master \'{0}\' when mine failback is \'{1}\''.format( + data[u'master'], self.opts[u'master'])) # if the master failback event is for the current master, raise an exception - elif data['master'] == self.opts['master'][0]: - raise SaltException('Already connected to \'{0}\''.format(data['master'])) + elif data[u'master'] == self.opts[u'master'][0]: + raise SaltException(u'Already connected to \'{0}\''.format(data[u'master'])) if self.connected: # we are not connected anymore self.connected = False - log.info('Connection to master {0} lost'.format(self.opts['master'])) + log.info(u'Connection to master %s lost', self.opts[u'master']) - if self.opts['master_type'] != 'failover': + if self.opts[u'master_type'] != u'failover': # modify the scheduled job to fire on reconnect - if self.opts['transport'] != 'tcp': + if self.opts[u'transport'] != u'tcp': schedule = { - 'function': 'status.master', - 'seconds': self.opts['master_alive_interval'], - 'jid_include': True, - 'maxrunning': 1, - 'return_job': False, - 'kwargs': {'master': self.opts['master'], - 'connected': False} + u'function': u'status.master', + u'seconds': self.opts[u'master_alive_interval'], + u'jid_include': True, + u'maxrunning': 1, + u'return_job': False, + u'kwargs': {u'master': self.opts[u'master'], + u'connected': False} } - self.schedule.modify_job(name=master_event(type='alive', master=self.opts['master']), + self.schedule.modify_job(name=master_event(type=u'alive', master=self.opts[u'master']), schedule=schedule) else: # delete the scheduled job to don't interfere with the failover process - if self.opts['transport'] != 'tcp': - self.schedule.delete_job(name=master_event(type='alive')) + if self.opts[u'transport'] != u'tcp': + self.schedule.delete_job(name=master_event(type=u'alive')) - log.info('Trying to tune in to next master from master-list') + log.info(u'Trying to tune in to next master from master-list') - if hasattr(self, 'pub_channel'): + if hasattr(self, u'pub_channel'): self.pub_channel.on_recv(None) - if hasattr(self.pub_channel, 'auth'): + if hasattr(self.pub_channel, u'auth'): self.pub_channel.auth.invalidate() - if hasattr(self.pub_channel, 'close'): + if hasattr(self.pub_channel, u'close'): self.pub_channel.close() del self.pub_channel @@ -2039,95 +2109,102 @@ class Minion(MinionBase): master, self.pub_channel = yield self.eval_master( opts=self.opts, failed=True, - failback=tag.startswith(master_event(type='failback'))) + failback=tag.startswith(master_event(type=u'failback'))) except SaltClientError: pass if self.connected: - self.opts['master'] = master + self.opts[u'master'] = master # re-init the subsystems to work with the new master - log.info('Re-initialising subsystems for new ' - 'master {0}'.format(self.opts['master'])) + log.info( + u'Re-initialising subsystems for new master %s', + self.opts[u'master'] + ) # put the current schedule into the new loaders - self.opts['schedule'] = self.schedule.option('schedule') + self.opts[u'schedule'] = self.schedule.option(u'schedule') self.functions, self.returners, self.function_errors, self.executors = self._load_modules() # make the schedule to use the new 'functions' loader self.schedule.functions = self.functions self.pub_channel.on_recv(self._handle_payload) self._fire_master_minion_start() - log.info('Minion is ready to receive requests!') + log.info(u'Minion is ready to receive requests!') # update scheduled job to run with the new master addr - if self.opts['transport'] != 'tcp': + if self.opts[u'transport'] != u'tcp': schedule = { - 'function': 'status.master', - 'seconds': self.opts['master_alive_interval'], - 'jid_include': True, - 'maxrunning': 1, - 'return_job': False, - 'kwargs': {'master': self.opts['master'], - 'connected': True} + u'function': u'status.master', + u'seconds': self.opts[u'master_alive_interval'], + u'jid_include': True, + u'maxrunning': 1, + u'return_job': False, + u'kwargs': {u'master': self.opts[u'master'], + u'connected': True} } - self.schedule.modify_job(name=master_event(type='alive', master=self.opts['master']), + self.schedule.modify_job(name=master_event(type=u'alive', master=self.opts[u'master']), schedule=schedule) - if self.opts['master_failback'] and 'master_list' in self.opts: - if self.opts['master'] != self.opts['master_list'][0]: + if self.opts[u'master_failback'] and u'master_list' in self.opts: + if self.opts[u'master'] != self.opts[u'master_list'][0]: schedule = { - 'function': 'status.ping_master', - 'seconds': self.opts['master_failback_interval'], - 'jid_include': True, - 'maxrunning': 1, - 'return_job': False, - 'kwargs': {'master': self.opts['master_list'][0]} + u'function': u'status.ping_master', + u'seconds': self.opts[u'master_failback_interval'], + u'jid_include': True, + u'maxrunning': 1, + u'return_job': False, + u'kwargs': {u'master': self.opts[u'master_list'][0]} } - self.schedule.modify_job(name=master_event(type='failback'), + self.schedule.modify_job(name=master_event(type=u'failback'), schedule=schedule) else: - self.schedule.delete_job(name=master_event(type='failback'), persist=True) + self.schedule.delete_job(name=master_event(type=u'failback'), persist=True) else: self.restart = True self.io_loop.stop() - elif tag.startswith(master_event(type='connected')): + elif tag.startswith(master_event(type=u'connected')): # handle this event only once. otherwise it will pollute the log # also if master type is failover all the reconnection work is done # by `disconnected` event handler and this event must never happen, # anyway check it to be sure - if not self.connected and self.opts['master_type'] != 'failover': - log.info('Connection to master {0} re-established'.format(self.opts['master'])) + if not self.connected and self.opts[u'master_type'] != u'failover': + log.info(u'Connection to master %s re-established', self.opts[u'master']) self.connected = True # modify the __master_alive job to only fire, # if the connection is lost again - if self.opts['transport'] != 'tcp': + if self.opts[u'transport'] != u'tcp': schedule = { - 'function': 'status.master', - 'seconds': self.opts['master_alive_interval'], - 'jid_include': True, - 'maxrunning': 1, - 'return_job': False, - 'kwargs': {'master': self.opts['master'], - 'connected': True} + u'function': u'status.master', + u'seconds': self.opts[u'master_alive_interval'], + u'jid_include': True, + u'maxrunning': 1, + u'return_job': False, + u'kwargs': {u'master': self.opts[u'master'], + u'connected': True} } - self.schedule.modify_job(name=master_event(type='alive', master=self.opts['master']), + self.schedule.modify_job(name=master_event(type=u'alive', master=self.opts[u'master']), schedule=schedule) - elif tag.startswith('__schedule_return'): + elif tag.startswith(u'__schedule_return'): # reporting current connection with master - if data['schedule'].startswith(master_event(type='alive', master='')): - if data['return']: - log.debug('Connected to master {0}'.format(data['schedule'].split(master_event(type='alive', master=''))[1])) - self._return_pub(data, ret_cmd='_return', sync=False) - elif tag.startswith('_salt_error'): + if data[u'schedule'].startswith(master_event(type=u'alive', master=u'')): + if data[u'return']: + log.debug( + u'Connected to master %s', + data[u'schedule'].split(master_event(type=u'alive', master=u''))[1] + ) + self._return_pub(data, ret_cmd=u'_return', sync=False) + elif tag.startswith(u'_salt_error'): if self.connected: - log.debug('Forwarding salt error event tag={tag}'.format(tag=tag)) + log.debug(u'Forwarding salt error event tag=%s', tag) self._fire_master(data, tag) - elif tag.startswith('salt/auth/creds'): - key = tuple(data['key']) - log.debug('Updating auth data for {0}: {1} -> {2}'.format( - key, salt.crypt.AsyncAuth.creds_map.get(key), data['creds'])) - salt.crypt.AsyncAuth.creds_map[tuple(data['key'])] = data['creds'] + elif tag.startswith(u'salt/auth/creds'): + key = tuple(data[u'key']) + log.debug( + u'Updating auth data for %s: %s -> %s', + key, salt.crypt.AsyncAuth.creds_map.get(key), data[u'creds'] + ) + salt.crypt.AsyncAuth.creds_map[tuple(data[u'key'])] = data[u'creds'] def _fallback_cleanups(self): ''' @@ -2137,7 +2214,7 @@ class Minion(MinionBase): multiprocessing.active_children() # Cleanup Windows threads - if not salt.utils.is_windows(): + if not salt.utils.platform.is_windows(): return for thread in self.win_proc: if not thread.is_alive(): @@ -2156,13 +2233,13 @@ class Minion(MinionBase): ''' self._pre_tune() - log.debug('Minion \'{0}\' trying to tune in'.format(self.opts['id'])) + log.debug(u'Minion \'%s\' trying to tune in', self.opts[u'id']) if start: self.sync_connect_master() if self.connected: self._fire_master_minion_start() - log.info('Minion is ready to receive requests!') + log.info(u'Minion is ready to receive requests!') # Make sure to gracefully handle SIGUSR1 enable_sigusr1_handler() @@ -2173,48 +2250,45 @@ class Minion(MinionBase): # On first startup execute a state run if configured to do so self._state_run() - loop_interval = self.opts['loop_interval'] + loop_interval = self.opts[u'loop_interval'] try: - if self.opts['grains_refresh_every']: # If exists and is not zero. In minutes, not seconds! - if self.opts['grains_refresh_every'] > 1: - log.debug( - 'Enabling the grains refresher. Will run every {0} minutes.'.format( - self.opts['grains_refresh_every']) - ) - else: # Clean up minute vs. minutes in log message - log.debug( - 'Enabling the grains refresher. Will run every {0} minute.'.format( - self.opts['grains_refresh_every']) - - ) + if self.opts[u'grains_refresh_every']: # If exists and is not zero. In minutes, not seconds! + log.debug( + u'Enabling the grains refresher. Will run every %s ' + u'minute%s.', + self.opts[u'grains_refresh_every'], + u's' if self.opts[u'grains_refresh_every'] > 1 else u'' + ) self._refresh_grains_watcher( - abs(self.opts['grains_refresh_every']) + abs(self.opts[u'grains_refresh_every']) ) except Exception as exc: log.error( - 'Exception occurred in attempt to initialize grain refresh routine during minion tune-in: {0}'.format( - exc) + u'Exception occurred in attempt to initialize grain refresh ' + u'routine during minion tune-in: %s', exc ) self.periodic_callbacks = {} # schedule the stuff that runs every interval - ping_interval = self.opts.get('ping_interval', 0) * 60 + ping_interval = self.opts.get(u'ping_interval', 0) * 60 if ping_interval > 0 and self.connected: def ping_master(): try: - if not self._fire_master('ping', 'minion_ping'): - if not self.opts.get('auth_safemode', True): - log.error('** Master Ping failed. Attempting to restart minion**') - delay = self.opts.get('random_reauth_delay', 5) - log.info('delaying random_reauth_delay {0}s'.format(delay)) + def ping_timeout_handler(*_): + if not self.opts.get(u'auth_safemode', True): + log.error(u'** Master Ping failed. Attempting to restart minion**') + delay = self.opts.get(u'random_reauth_delay', 5) + log.info(u'delaying random_reauth_delay %ss', delay) # regular sys.exit raises an exception -- which isn't sufficient in a thread os._exit(salt.defaults.exitcodes.SALT_KEEPALIVE) - except Exception: - log.warning('Attempt to ping master failed.', exc_on_loglevel=logging.DEBUG) - self.periodic_callbacks['ping'] = tornado.ioloop.PeriodicCallback(ping_master, ping_interval * 1000, io_loop=self.io_loop) - self.periodic_callbacks['cleanup'] = tornado.ioloop.PeriodicCallback(self._fallback_cleanups, loop_interval * 1000, io_loop=self.io_loop) + self._fire_master('ping', 'minion_ping', sync=False, timeout_handler=ping_timeout_handler) + except Exception: + log.warning(u'Attempt to ping master failed.', exc_on_loglevel=logging.DEBUG) + self.periodic_callbacks[u'ping'] = tornado.ioloop.PeriodicCallback(ping_master, ping_interval * 1000, io_loop=self.io_loop) + + self.periodic_callbacks[u'cleanup'] = tornado.ioloop.PeriodicCallback(self._fallback_cleanups, loop_interval * 1000, io_loop=self.io_loop) def handle_beacons(): # Process Beacons @@ -2222,27 +2296,27 @@ class Minion(MinionBase): try: beacons = self.process_beacons(self.functions) except Exception: - log.critical('The beacon errored: ', exc_info=True) + log.critical(u'The beacon errored: ', exc_info=True) if beacons and self.connected: - self._fire_master(events=beacons) + self._fire_master(events=beacons, sync=False) - self.periodic_callbacks['beacons'] = tornado.ioloop.PeriodicCallback(handle_beacons, loop_interval * 1000, io_loop=self.io_loop) + self.periodic_callbacks[u'beacons'] = tornado.ioloop.PeriodicCallback(handle_beacons, loop_interval * 1000, io_loop=self.io_loop) # TODO: actually listen to the return and change period def handle_schedule(): self.process_schedule(self, loop_interval) - if hasattr(self, 'schedule'): - self.periodic_callbacks['schedule'] = tornado.ioloop.PeriodicCallback(handle_schedule, 1000, io_loop=self.io_loop) + if hasattr(self, u'schedule'): + self.periodic_callbacks[u'schedule'] = tornado.ioloop.PeriodicCallback(handle_schedule, 1000, io_loop=self.io_loop) # start all the other callbacks for periodic_cb in six.itervalues(self.periodic_callbacks): periodic_cb.start() # add handler to subscriber - if hasattr(self, 'pub_channel') and self.pub_channel is not None: + if hasattr(self, u'pub_channel') and self.pub_channel is not None: self.pub_channel.on_recv(self._handle_payload) - elif self.opts.get('master_type') != 'disable': - log.error('No connection to master found. Scheduled jobs will not run.') + elif self.opts.get(u'master_type') != u'disable': + log.error(u'No connection to master found. Scheduled jobs will not run.') if start: try: @@ -2253,20 +2327,23 @@ class Minion(MinionBase): self.destroy() def _handle_payload(self, payload): - if payload is not None and payload['enc'] == 'aes': - if self._target_load(payload['load']): - self._handle_decoded_payload(payload['load']) - elif self.opts['zmq_filtering']: + if payload is not None and payload[u'enc'] == u'aes': + if self._target_load(payload[u'load']): + self._handle_decoded_payload(payload[u'load']) + elif self.opts[u'zmq_filtering']: # In the filtering enabled case, we'd like to know when minion sees something it shouldnt - log.trace('Broadcast message received not for this minion, Load: {0}'.format(payload['load'])) + log.trace( + u'Broadcast message received not for this minion, Load: %s', + payload[u'load'] + ) # If it's not AES, and thus has not been verified, we do nothing. # In the future, we could add support for some clearfuncs, but # the minion currently has no need. def _target_load(self, load): # Verify that the publication is valid - if 'tgt' not in load or 'jid' not in load or 'fun' not in load \ - or 'arg' not in load: + if u'tgt' not in load or u'jid' not in load or u'fun' not in load \ + or u'arg' not in load: return False # Verify that the publication applies to this minion @@ -2276,19 +2353,19 @@ class Minion(MinionBase): # pre-processing on the master and this minion should not see the # publication if the master does not determine that it should. - if 'tgt_type' in load: + if u'tgt_type' in load: match_func = getattr(self.matcher, - '{0}_match'.format(load['tgt_type']), None) + u'{0}_match'.format(load[u'tgt_type']), None) if match_func is None: return False - if load['tgt_type'] in ('grain', 'grain_pcre', 'pillar'): - delimiter = load.get('delimiter', DEFAULT_TARGET_DELIM) - if not match_func(load['tgt'], delimiter=delimiter): + if load[u'tgt_type'] in (u'grain', u'grain_pcre', u'pillar'): + delimiter = load.get(u'delimiter', DEFAULT_TARGET_DELIM) + if not match_func(load[u'tgt'], delimiter=delimiter): return False - elif not match_func(load['tgt']): + elif not match_func(load[u'tgt']): return False else: - if not self.matcher.glob_match(load['tgt']): + if not self.matcher.glob_match(load[u'tgt']): return False return True @@ -2298,14 +2375,14 @@ class Minion(MinionBase): Tear down the minion ''' self._running = False - if hasattr(self, 'schedule'): + if hasattr(self, u'schedule'): del self.schedule - if hasattr(self, 'pub_channel') and self.pub_channel is not None: + if hasattr(self, u'pub_channel') and self.pub_channel is not None: self.pub_channel.on_recv(None) - if hasattr(self.pub_channel, 'close'): + if hasattr(self.pub_channel, u'close'): self.pub_channel.close() del self.pub_channel - if hasattr(self, 'periodic_callbacks'): + if hasattr(self, u'periodic_callbacks'): for cb in six.itervalues(self.periodic_callbacks): cb.stop() @@ -2319,11 +2396,11 @@ class Syndic(Minion): master to authenticate with a higher level master. ''' def __init__(self, opts, **kwargs): - self._syndic_interface = opts.get('interface') + self._syndic_interface = opts.get(u'interface') self._syndic = True # force auth_safemode True because Syndic don't support autorestart - opts['auth_safemode'] = True - opts['loop_interval'] = 1 + opts[u'auth_safemode'] = True + opts[u'loop_interval'] = 1 super(Syndic, self).__init__(opts, **kwargs) self.mminion = salt.minion.MasterMinion(opts) self.jid_forward_cache = set() @@ -2337,9 +2414,9 @@ class Syndic(Minion): differently. ''' # TODO: even do this?? - data['to'] = int(data.get('to', self.opts['timeout'])) - 1 + data[u'to'] = int(data.get(u'to', self.opts[u'timeout'])) - 1 # Only forward the command if it didn't originate from ourselves - if data.get('master_id', 0) != self.opts.get('master_id', 1): + if data.get(u'master_id', 0) != self.opts.get(u'master_id', 1): self.syndic_cmd(data) def syndic_cmd(self, data): @@ -2347,29 +2424,29 @@ class Syndic(Minion): Take the now clear load and forward it on to the client cmd ''' # Set up default tgt_type - if 'tgt_type' not in data: - data['tgt_type'] = 'glob' + if u'tgt_type' not in data: + data[u'tgt_type'] = u'glob' kwargs = {} # optionally add a few fields to the publish data - for field in ('master_id', # which master the job came from - 'user', # which user ran the job + for field in (u'master_id', # which master the job came from + u'user', # which user ran the job ): if field in data: kwargs[field] = data[field] def timeout_handler(*args): - log.warning('Unable to forward pub data: {0}'.format(args[1])) + log.warning(u'Unable to forward pub data: %s', args[1]) return True with tornado.stack_context.ExceptionStackContext(timeout_handler): - self.local.pub_async(data['tgt'], - data['fun'], - data['arg'], - data['tgt_type'], - data['ret'], - data['jid'], - data['to'], + self.local.pub_async(data[u'tgt'], + data[u'fun'], + data[u'arg'], + data[u'tgt_type'], + data[u'ret'], + data[u'jid'], + data[u'to'], io_loop=self.io_loop, callback=lambda _: None, **kwargs) @@ -2377,19 +2454,19 @@ class Syndic(Minion): def fire_master_syndic_start(self): # Send an event to the master that the minion is live self._fire_master( - 'Syndic {0} started at {1}'.format( - self.opts['id'], + u'Syndic {0} started at {1}'.format( + self.opts[u'id'], time.asctime() ), - 'syndic_start', + u'syndic_start', sync=False, ) self._fire_master( - 'Syndic {0} started at {1}'.format( - self.opts['id'], + u'Syndic {0} started at {1}'.format( + self.opts[u'id'], time.asctime() ), - tagify([self.opts['id'], 'start'], 'syndic'), + tagify([self.opts[u'id'], u'start'], u'syndic'), sync=False, ) @@ -2402,15 +2479,15 @@ class Syndic(Minion): ''' # Instantiate the local client self.local = salt.client.get_local_client( - self.opts['_minion_conf_file'], io_loop=self.io_loop) + self.opts[u'_minion_conf_file'], io_loop=self.io_loop) # add handler to subscriber self.pub_channel.on_recv(self._process_cmd_socket) def _process_cmd_socket(self, payload): - if payload is not None and payload['enc'] == 'aes': - log.trace('Handling payload') - self._handle_decoded_payload(payload['load']) + if payload is not None and payload[u'enc'] == u'aes': + log.trace(u'Handling payload') + self._handle_decoded_payload(payload[u'load']) # If it's not AES, and thus has not been verified, we do nothing. # In the future, we could add support for some clearfuncs, but # the syndic currently has no need. @@ -2419,15 +2496,15 @@ class Syndic(Minion): def _return_pub_multi(self, values): for value in values: yield self._return_pub(value, - '_syndic_return', + u'_syndic_return', timeout=self._return_retry_timer(), sync=False) @tornado.gen.coroutine def reconnect(self): - if hasattr(self, 'pub_channel'): + if hasattr(self, u'pub_channel'): self.pub_channel.on_recv(None) - if hasattr(self.pub_channel, 'close'): + if hasattr(self.pub_channel, u'close'): self.pub_channel.close() del self.pub_channel @@ -2436,9 +2513,9 @@ class Syndic(Minion): master, self.pub_channel = yield self.eval_master(opts=self.opts) if self.connected: - self.opts['master'] = master + self.opts[u'master'] = master self.pub_channel.on_recv(self._process_cmd_socket) - log.info('Minion is ready to receive requests!') + log.info(u'Minion is ready to receive requests!') raise tornado.gen.Return(self) @@ -2449,10 +2526,10 @@ class Syndic(Minion): # We borrowed the local clients poller so give it back before # it's destroyed. Reset the local poller reference. super(Syndic, self).destroy() - if hasattr(self, 'local'): + if hasattr(self, u'local'): del self.local - if hasattr(self, 'forward_events'): + if hasattr(self, u'forward_events'): self.forward_events.stop() @@ -2481,15 +2558,15 @@ class SyndicManager(MinionBase): SYNDIC_EVENT_TIMEOUT = 5 def __init__(self, opts, io_loop=None): - opts['loop_interval'] = 1 + opts[u'loop_interval'] = 1 super(SyndicManager, self).__init__(opts) self.mminion = salt.minion.MasterMinion(opts) # sync (old behavior), cluster (only returns and publishes) - self.syndic_mode = self.opts.get('syndic_mode', 'sync') - self.syndic_failover = self.opts.get('syndic_failover', 'random') + self.syndic_mode = self.opts.get(u'syndic_mode', u'sync') + self.syndic_failover = self.opts.get(u'syndic_failover', u'random') - self.auth_wait = self.opts['acceptance_wait_time'] - self.max_auth_wait = self.opts['acceptance_wait_time_max'] + self.auth_wait = self.opts[u'acceptance_wait_time'] + self.max_auth_wait = self.opts[u'acceptance_wait_time_max'] self._has_master = threading.Event() self.jid_forward_cache = set() @@ -2516,12 +2593,12 @@ class SyndicManager(MinionBase): Spawn all the coroutines which will sign in the syndics ''' self._syndics = OrderedDict() # mapping of opts['master'] -> syndic - masters = self.opts['master'] + masters = self.opts[u'master'] if not isinstance(masters, list): masters = [masters] for master in masters: s_opts = copy.copy(self.opts) - s_opts['master'] = master + s_opts[u'master'] = master self._syndics[master] = self._connect_syndic(s_opts) @tornado.gen.coroutine @@ -2530,10 +2607,13 @@ class SyndicManager(MinionBase): Create a syndic, and asynchronously connect it to a master ''' last = 0 # never have we signed in - auth_wait = opts['acceptance_wait_time'] + auth_wait = opts[u'acceptance_wait_time'] failed = False while True: - log.debug('Syndic attempting to connect to {0}'.format(opts['master'])) + log.debug( + u'Syndic attempting to connect to %s', + opts[u'master'] + ) try: syndic = Syndic(opts, timeout=self.SYNDIC_CONNECT_TIMEOUT, @@ -2547,11 +2627,17 @@ class SyndicManager(MinionBase): # Send an event to the master that the minion is live syndic.fire_master_syndic_start() - log.info('Syndic successfully connected to {0}'.format(opts['master'])) + log.info( + u'Syndic successfully connected to %s', + opts[u'master'] + ) break except SaltClientError as exc: failed = True - log.error('Error while bringing up syndic for multi-syndic. Is master at {0} responding?'.format(opts['master'])) + log.error( + u'Error while bringing up syndic for multi-syndic. Is the ' + u'master at %s responding?', opts[u'master'] + ) last = time.time() if auth_wait < self.max_auth_wait: auth_wait += self.auth_wait @@ -2560,7 +2646,10 @@ class SyndicManager(MinionBase): raise except: # pylint: disable=W0702 failed = True - log.critical('Unexpected error while connecting to {0}'.format(opts['master']), exc_info=True) + log.critical( + u'Unexpected error while connecting to %s', + opts[u'master'], exc_info=True + ) raise tornado.gen.Return(syndic) @@ -2573,7 +2662,11 @@ class SyndicManager(MinionBase): syndic = self._syndics[master].result() # pylint: disable=no-member self._syndics[master] = syndic.reconnect() else: - log.info('Attempting to mark {0} as dead, although it is already marked dead'.format(master)) # TODO: debug? + # TODO: debug? + log.info( + u'Attempting to mark %s as dead, although it is already ' + u'marked dead', master + ) def _call_syndic(self, func, args=(), kwargs=None, master_id=None): ''' @@ -2583,26 +2676,35 @@ class SyndicManager(MinionBase): kwargs = {} for master, syndic_future in self.iter_master_options(master_id): if not syndic_future.done() or syndic_future.exception(): - log.error('Unable to call {0} on {1}, that syndic is not connected'.format(func, master)) + log.error( + u'Unable to call %s on %s, that syndic is not connected', + func, master + ) continue try: getattr(syndic_future.result(), func)(*args, **kwargs) return except SaltClientError: - log.error('Unable to call {0} on {1}, trying another...'.format(func, master)) + log.error( + u'Unable to call %s on %s, trying another...', + func, master + ) self._mark_master_dead(master) continue - log.critical('Unable to call {0} on any masters!'.format(func)) + log.critical(u'Unable to call %s on any masters!', func) def _return_pub_syndic(self, values, master_id=None): ''' Wrapper to call the '_return_pub_multi' a syndic, best effort to get the one you asked for ''' - func = '_return_pub_multi' + func = u'_return_pub_multi' for master, syndic_future in self.iter_master_options(master_id): if not syndic_future.done() or syndic_future.exception(): - log.error('Unable to call {0} on {1}, that syndic is not connected'.format(func, master)) + log.error( + u'Unable to call %s on %s, that syndic is not connected', + func, master + ) continue future, data = self.pub_futures.get(master, (None, None)) @@ -2616,7 +2718,10 @@ class SyndicManager(MinionBase): continue elif future.exception(): # Previous execution on this master returned an error - log.error('Unable to call {0} on {1}, trying another...'.format(func, master)) + log.error( + u'Unable to call %s on %s, trying another...', + func, master + ) self._mark_master_dead(master) del self.pub_futures[master] # Add not sent data to the delayed list and try the next master @@ -2633,7 +2738,7 @@ class SyndicManager(MinionBase): Iterate (in order) over your options for master ''' masters = list(self._syndics.keys()) - if self.opts['syndic_failover'] == 'random': + if self.opts[u'syndic_failover'] == u'random': shuffle(masters) if master_id not in self._syndics: master_id = masters.pop(0) @@ -2662,10 +2767,10 @@ class SyndicManager(MinionBase): self._spawn_syndics() # Instantiate the local client self.local = salt.client.get_local_client( - self.opts['_minion_conf_file'], io_loop=self.io_loop) - self.local.event.subscribe('') + self.opts[u'_minion_conf_file'], io_loop=self.io_loop) + self.local.event.subscribe(u'') - log.debug('SyndicManager \'{0}\' trying to tune in'.format(self.opts['id'])) + log.debug(u'SyndicManager \'%s\' trying to tune in', self.opts[u'id']) # register the event sub to the poller self.job_rets = {} @@ -2676,7 +2781,7 @@ class SyndicManager(MinionBase): # forward events every syndic_event_forward_timeout self.forward_events = tornado.ioloop.PeriodicCallback(self._forward_events, - self.opts['syndic_event_forward_timeout'] * 1000, + self.opts[u'syndic_event_forward_timeout'] * 1000, io_loop=self.io_loop) self.forward_events.start() @@ -2688,65 +2793,65 @@ class SyndicManager(MinionBase): def _process_event(self, raw): # TODO: cleanup: Move down into event class mtag, data = self.local.event.unpack(raw, self.local.event.serial) - log.trace('Got event {0}'.format(mtag)) # pylint: disable=no-member + log.trace(u'Got event %s', mtag) # pylint: disable=no-member - tag_parts = mtag.split('/') - if len(tag_parts) >= 4 and tag_parts[1] == 'job' and \ - salt.utils.jid.is_jid(tag_parts[2]) and tag_parts[3] == 'ret' and \ - 'return' in data: - if 'jid' not in data: + tag_parts = mtag.split(u'/') + if len(tag_parts) >= 4 and tag_parts[1] == u'job' and \ + salt.utils.jid.is_jid(tag_parts[2]) and tag_parts[3] == u'ret' and \ + u'return' in data: + if u'jid' not in data: # Not a job return return - if self.syndic_mode == 'cluster' and data.get('master_id', 0) == self.opts.get('master_id', 1): - log.debug('Return received with matching master_id, not forwarding') + if self.syndic_mode == u'cluster' and data.get(u'master_id', 0) == self.opts.get(u'master_id', 1): + log.debug(u'Return received with matching master_id, not forwarding') return - master = data.get('master_id') + master = data.get(u'master_id') jdict = self.job_rets.setdefault(master, {}).setdefault(mtag, {}) if not jdict: - jdict['__fun__'] = data.get('fun') - jdict['__jid__'] = data['jid'] - jdict['__load__'] = {} - fstr = '{0}.get_load'.format(self.opts['master_job_cache']) + jdict[u'__fun__'] = data.get(u'fun') + jdict[u'__jid__'] = data[u'jid'] + jdict[u'__load__'] = {} + fstr = u'{0}.get_load'.format(self.opts[u'master_job_cache']) # Only need to forward each load once. Don't hit the disk # for every minion return! - if data['jid'] not in self.jid_forward_cache: - jdict['__load__'].update( - self.mminion.returners[fstr](data['jid']) + if data[u'jid'] not in self.jid_forward_cache: + jdict[u'__load__'].update( + self.mminion.returners[fstr](data[u'jid']) ) - self.jid_forward_cache.add(data['jid']) - if len(self.jid_forward_cache) > self.opts['syndic_jid_forward_cache_hwm']: + self.jid_forward_cache.add(data[u'jid']) + if len(self.jid_forward_cache) > self.opts[u'syndic_jid_forward_cache_hwm']: # Pop the oldest jid from the cache tmp = sorted(list(self.jid_forward_cache)) tmp.pop(0) self.jid_forward_cache = set(tmp) if master is not None: # __'s to make sure it doesn't print out on the master cli - jdict['__master_id__'] = master + jdict[u'__master_id__'] = master ret = {} - for key in 'return', 'retcode', 'success': + for key in u'return', u'retcode', u'success': if key in data: ret[key] = data[key] - jdict[data['id']] = ret + jdict[data[u'id']] = ret else: # TODO: config to forward these? If so we'll have to keep track of who # has seen them # if we are the top level masters-- don't forward all the minion events - if self.syndic_mode == 'sync': + if self.syndic_mode == u'sync': # Add generic event aggregation here - if 'retcode' not in data: - self.raw_events.append({'data': data, 'tag': mtag}) + if u'retcode' not in data: + self.raw_events.append({u'data': data, u'tag': mtag}) def _forward_events(self): - log.trace('Forwarding events') # pylint: disable=no-member + log.trace(u'Forwarding events') # pylint: disable=no-member if self.raw_events: events = self.raw_events self.raw_events = [] - self._call_syndic('_fire_master', - kwargs={'events': events, - 'pretag': tagify(self.opts['id'], base='syndic'), - 'timeout': self.SYNDIC_EVENT_TIMEOUT, - 'sync': False, + self._call_syndic(u'_fire_master', + kwargs={u'events': events, + u'pretag': tagify(self.opts[u'id'], base=u'syndic'), + u'timeout': self.SYNDIC_EVENT_TIMEOUT, + u'sync': False, }, ) if self.delayed: @@ -2773,24 +2878,22 @@ class Matcher(object): Takes the data passed to a top file environment and determines if the data matches this minion ''' - matcher = 'compound' + matcher = u'compound' if not data: - log.error('Received bad data when setting the match from the top ' - 'file') + log.error(u'Received bad data when setting the match from the top ' + u'file') return False for item in data: if isinstance(item, dict): - if 'match' in item: - matcher = item['match'] - if hasattr(self, matcher + '_match'): - funcname = '{0}_match'.format(matcher) - if matcher == 'nodegroup': + if u'match' in item: + matcher = item[u'match'] + if hasattr(self, matcher + u'_match'): + funcname = u'{0}_match'.format(matcher) + if matcher == u'nodegroup': return getattr(self, funcname)(match, nodegroups) return getattr(self, funcname)(match) else: - log.error('Attempting to match with unknown matcher: {0}'.format( - matcher - )) + log.error(u'Attempting to match with unknown matcher: %s', matcher) return False def glob_match(self, tgt): @@ -2800,45 +2903,45 @@ class Matcher(object): if not isinstance(tgt, six.string_types): return False - return fnmatch.fnmatch(self.opts['id'], tgt) + return fnmatch.fnmatch(self.opts[u'id'], tgt) def pcre_match(self, tgt): ''' Returns true if the passed pcre regex matches ''' - return bool(re.match(tgt, self.opts['id'])) + return bool(re.match(tgt, self.opts[u'id'])) def list_match(self, tgt): ''' Determines if this host is on the list ''' if isinstance(tgt, six.string_types): - tgt = tgt.split(',') - return bool(self.opts['id'] in tgt) + tgt = tgt.split(u',') + return bool(self.opts[u'id'] in tgt) def grain_match(self, tgt, delimiter=DEFAULT_TARGET_DELIM): ''' Reads in the grains glob match ''' - log.debug('grains target: {0}'.format(tgt)) + log.debug(u'grains target: %s', tgt) if delimiter not in tgt: - log.error('Got insufficient arguments for grains match ' - 'statement from master') + log.error(u'Got insufficient arguments for grains match ' + u'statement from master') return False return salt.utils.subdict_match( - self.opts['grains'], tgt, delimiter=delimiter + self.opts[u'grains'], tgt, delimiter=delimiter ) def grain_pcre_match(self, tgt, delimiter=DEFAULT_TARGET_DELIM): ''' Matches a grain based on regex ''' - log.debug('grains pcre target: {0}'.format(tgt)) + log.debug(u'grains pcre target: %s', tgt) if delimiter not in tgt: - log.error('Got insufficient arguments for grains pcre match ' - 'statement from master') + log.error(u'Got insufficient arguments for grains pcre match ' + u'statement from master') return False - return salt.utils.subdict_match(self.opts['grains'], tgt, + return salt.utils.subdict_match(self.opts[u'grains'], tgt, delimiter=delimiter, regex_match=True) def data_match(self, tgt): @@ -2848,10 +2951,10 @@ class Matcher(object): if self.functions is None: utils = salt.loader.utils(self.opts) self.functions = salt.loader.minion_mods(self.opts, utils=utils) - comps = tgt.split(':') + comps = tgt.split(u':') if len(comps) < 2: return False - val = self.functions['data.getval'](comps[0]) + val = self.functions[u'data.getval'](comps[0]) if val is None: # The value is not defined return False @@ -2874,38 +2977,38 @@ class Matcher(object): ''' Reads in the pillar glob match ''' - log.debug('pillar target: {0}'.format(tgt)) + log.debug(u'pillar target: %s', tgt) if delimiter not in tgt: - log.error('Got insufficient arguments for pillar match ' - 'statement from master') + log.error(u'Got insufficient arguments for pillar match ' + u'statement from master') return False return salt.utils.subdict_match( - self.opts['pillar'], tgt, delimiter=delimiter + self.opts[u'pillar'], tgt, delimiter=delimiter ) def pillar_pcre_match(self, tgt, delimiter=DEFAULT_TARGET_DELIM): ''' Reads in the pillar pcre match ''' - log.debug('pillar PCRE target: {0}'.format(tgt)) + log.debug(u'pillar PCRE target: %s', tgt) if delimiter not in tgt: - log.error('Got insufficient arguments for pillar PCRE match ' - 'statement from master') + log.error(u'Got insufficient arguments for pillar PCRE match ' + u'statement from master') return False return salt.utils.subdict_match( - self.opts['pillar'], tgt, delimiter=delimiter, regex_match=True + self.opts[u'pillar'], tgt, delimiter=delimiter, regex_match=True ) - def pillar_exact_match(self, tgt, delimiter=':'): + def pillar_exact_match(self, tgt, delimiter=u':'): ''' Reads in the pillar match, no globbing, no PCRE ''' - log.debug('pillar target: {0}'.format(tgt)) + log.debug(u'pillar target: %s', tgt) if delimiter not in tgt: - log.error('Got insufficient arguments for pillar match ' - 'statement from master') + log.error(u'Got insufficient arguments for pillar match ' + u'statement from master') return False - return salt.utils.subdict_match(self.opts['pillar'], + return salt.utils.subdict_match(self.opts[u'pillar'], tgt, delimiter=delimiter, exact_match=True) @@ -2922,11 +3025,11 @@ class Matcher(object): # Target is a network? tgt = ipaddress.ip_network(tgt) except: # pylint: disable=bare-except - log.error('Invalid IP/CIDR target: {0}'.format(tgt)) + log.error(u'Invalid IP/CIDR target: %s', tgt) return [] - proto = 'ipv{0}'.format(tgt.version) + proto = u'ipv{0}'.format(tgt.version) - grains = self.opts['grains'] + grains = self.opts[u'grains'] if proto not in grains: match = False @@ -2942,11 +3045,11 @@ class Matcher(object): Matches based on range cluster ''' if HAS_RANGE: - range_ = seco.range.Range(self.opts['range_server']) + range_ = seco.range.Range(self.opts[u'range_server']) try: - return self.opts['grains']['fqdn'] in range_.expand(tgt) + return self.opts[u'grains'][u'fqdn'] in range_.expand(tgt) except seco.range.RangeException as exc: - log.debug('Range exception in compound match: {0}'.format(exc)) + log.debug(u'Range exception in compound match: %s', exc) return False return False @@ -2955,22 +3058,22 @@ class Matcher(object): Runs the compound target check ''' if not isinstance(tgt, six.string_types) and not isinstance(tgt, (list, tuple)): - log.error('Compound target received that is neither string, list nor tuple') + log.error(u'Compound target received that is neither string, list nor tuple') return False - log.debug('compound_match: {0} ? {1}'.format(self.opts['id'], tgt)) - ref = {'G': 'grain', - 'P': 'grain_pcre', - 'I': 'pillar', - 'J': 'pillar_pcre', - 'L': 'list', - 'N': None, # Nodegroups should already be expanded - 'S': 'ipcidr', - 'E': 'pcre'} + log.debug(u'compound_match: %s ? %s', self.opts[u'id'], tgt) + ref = {u'G': u'grain', + u'P': u'grain_pcre', + u'I': u'pillar', + u'J': u'pillar_pcre', + u'L': u'list', + u'N': None, # Nodegroups should already be expanded + u'S': u'ipcidr', + u'E': u'pcre'} if HAS_RANGE: - ref['R'] = 'range' + ref[u'R'] = u'range' results = [] - opers = ['and', 'or', 'not', '(', ')'] + opers = [u'and', u'or', u'not', u'(', u')'] if isinstance(tgt, six.string_types): words = tgt.split() @@ -2983,55 +3086,55 @@ class Matcher(object): # Easy check first if word in opers: if results: - if results[-1] == '(' and word in ('and', 'or'): - log.error('Invalid beginning operator after "(": {0}'.format(word)) + if results[-1] == u'(' and word in (u'and', u'or'): + log.error(u'Invalid beginning operator after "(": %s', word) return False - if word == 'not': - if not results[-1] in ('and', 'or', '('): - results.append('and') + if word == u'not': + if not results[-1] in (u'and', u'or', u'('): + results.append(u'and') results.append(word) else: # seq start with binary oper, fail - if word not in ['(', 'not']: - log.error('Invalid beginning operator: {0}'.format(word)) + if word not in [u'(', u'not']: + log.error(u'Invalid beginning operator: %s', word) return False results.append(word) - elif target_info and target_info['engine']: - if 'N' == target_info['engine']: + elif target_info and target_info[u'engine']: + if u'N' == target_info[u'engine']: # Nodegroups should already be expanded/resolved to other engines - log.error('Detected nodegroup expansion failure of "{0}"'.format(word)) + log.error( + u'Detected nodegroup expansion failure of "%s"', word) return False - engine = ref.get(target_info['engine']) + engine = ref.get(target_info[u'engine']) if not engine: # If an unknown engine is called at any time, fail out - log.error('Unrecognized target engine "{0}" for' - ' target expression "{1}"'.format( - target_info['engine'], - word, - ) - ) + log.error( + u'Unrecognized target engine "%s" for target ' + u'expression "%s"', target_info[u'engine'], word + ) return False - engine_args = [target_info['pattern']] + engine_args = [target_info[u'pattern']] engine_kwargs = {} - if target_info['delimiter']: - engine_kwargs['delimiter'] = target_info['delimiter'] + if target_info[u'delimiter']: + engine_kwargs[u'delimiter'] = target_info[u'delimiter'] results.append( - str(getattr(self, '{0}_match'.format(engine))(*engine_args, **engine_kwargs)) + str(getattr(self, u'{0}_match'.format(engine))(*engine_args, **engine_kwargs)) ) else: # The match is not explicitly defined, evaluate it as a glob results.append(str(self.glob_match(word))) - results = ' '.join(results) - log.debug('compound_match {0} ? "{1}" => "{2}"'.format(self.opts['id'], tgt, results)) + results = u' '.join(results) + log.debug(u'compound_match %s ? "%s" => "%s"', self.opts[u'id'], tgt, results) try: return eval(results) # pylint: disable=W0123 except Exception: - log.error('Invalid compound target: {0} for results: {1}'.format(tgt, results)) + log.error( + u'Invalid compound target: %s for results: %s', tgt, results) return False return False @@ -3068,7 +3171,7 @@ class ProxyMinionManager(MinionManager): class ProxyMinion(Minion): ''' - This class instantiates a 'proxy' minion--a minion that does not manipulate + This class instantiates a u'proxy' minion--a minion that does not manipulate the host it runs on, but instead manipulates a device that cannot run a minion. ''' @@ -3088,30 +3191,50 @@ class ProxyMinion(Minion): which is why the differences are not factored out into separate helper functions. ''' - log.debug("subclassed _post_master_init") + log.debug(u"subclassed _post_master_init") if self.connected: - self.opts['master'] = master + self.opts[u'master'] = master - self.opts['pillar'] = yield salt.pillar.get_async_pillar( + self.opts[u'pillar'] = yield salt.pillar.get_async_pillar( self.opts, - self.opts['grains'], - self.opts['id'], - saltenv=self.opts['environment'], - pillarenv=self.opts.get('pillarenv'), + self.opts[u'grains'], + self.opts[u'id'], + saltenv=self.opts[u'environment'], + pillarenv=self.opts.get(u'pillarenv'), ).compile_pillar() - if 'proxy' not in self.opts['pillar'] and 'proxy' not in self.opts: - errmsg = 'No proxy key found in pillar or opts for id '+self.opts['id']+'. '+\ - 'Check your pillar/opts configuration and contents. Salt-proxy aborted.' + if u'proxy' not in self.opts[u'pillar'] and u'proxy' not in self.opts: + errmsg = u'No proxy key found in pillar or opts for id ' + self.opts[u'id'] + u'. ' + \ + u'Check your pillar/opts configuration and contents. Salt-proxy aborted.' log.error(errmsg) self._running = False raise SaltSystemExit(code=-1, msg=errmsg) - if 'proxy' not in self.opts: - self.opts['proxy'] = self.opts['pillar']['proxy'] + if u'proxy' not in self.opts: + self.opts[u'proxy'] = self.opts[u'pillar'][u'proxy'] - fq_proxyname = self.opts['proxy']['proxytype'] + if self.opts.get(u'proxy_merge_pillar_in_opts'): + # Override proxy opts with pillar data when the user required. + self.opts = salt.utils.dictupdate.merge(self.opts, + self.opts[u'pillar'], + strategy=self.opts.get(u'proxy_merge_pillar_in_opts_strategy'), + merge_lists=self.opts.get(u'proxy_deep_merge_pillar_in_opts', False)) + elif self.opts.get(u'proxy_mines_pillar'): + # Even when not required, some details such as mine configuration + # should be merged anyway whenever possible. + if u'mine_interval' in self.opts[u'pillar']: + self.opts[u'mine_interval'] = self.opts[u'pillar'][u'mine_interval'] + if u'mine_functions' in self.opts[u'pillar']: + general_proxy_mines = self.opts.get(u'mine_functions', []) + specific_proxy_mines = self.opts[u'pillar'][u'mine_functions'] + try: + self.opts[u'mine_functions'] = general_proxy_mines + specific_proxy_mines + except TypeError as terr: + log.error(u'Unable to merge mine functions from the pillar in the opts, for proxy {}'.format( + self.opts[u'id'])) + + fq_proxyname = self.opts[u'proxy'][u'proxytype'] # Need to load the modules so they get all the dunder variables self.functions, self.returners, self.function_errors, self.executors = self._load_modules() @@ -3119,7 +3242,7 @@ class ProxyMinion(Minion): # we can then sync any proxymodules down from the master # we do a sync_all here in case proxy code was installed by # SPM or was manually placed in /srv/salt/_modules etc. - self.functions['saltutil.sync_all'](saltenv=self.opts['environment']) + self.functions[u'saltutil.sync_all'](saltenv=self.opts[u'environment']) # Pull in the utils self.utils = salt.loader.utils(self.opts) @@ -3129,14 +3252,14 @@ class ProxyMinion(Minion): # And re-load the modules so the __proxy__ variable gets injected self.functions, self.returners, self.function_errors, self.executors = self._load_modules() - self.functions.pack['__proxy__'] = self.proxy - self.proxy.pack['__salt__'] = self.functions - self.proxy.pack['__ret__'] = self.returners - self.proxy.pack['__pillar__'] = self.opts['pillar'] + self.functions.pack[u'__proxy__'] = self.proxy + self.proxy.pack[u'__salt__'] = self.functions + self.proxy.pack[u'__ret__'] = self.returners + self.proxy.pack[u'__pillar__'] = self.opts[u'pillar'] # Reload utils as well (chicken and egg, __utils__ needs __proxy__ and __proxy__ needs __utils__ self.utils = salt.loader.utils(self.opts, proxy=self.proxy) - self.proxy.pack['__utils__'] = self.utils + self.proxy.pack[u'__utils__'] = self.utils # Reload all modules so all dunder variables are injected self.proxy.reload_modules() @@ -3147,119 +3270,119 @@ class ProxyMinion(Minion): self.io_loop.spawn_callback(salt.engines.start_engines, self.opts, self.process_manager, proxy=self.proxy) - if ('{0}.init'.format(fq_proxyname) not in self.proxy - or '{0}.shutdown'.format(fq_proxyname) not in self.proxy): - errmsg = 'Proxymodule {0} is missing an init() or a shutdown() or both. '.format(fq_proxyname)+\ - 'Check your proxymodule. Salt-proxy aborted.' + if (u'{0}.init'.format(fq_proxyname) not in self.proxy + or u'{0}.shutdown'.format(fq_proxyname) not in self.proxy): + errmsg = u'Proxymodule {0} is missing an init() or a shutdown() or both. '.format(fq_proxyname) + \ + u'Check your proxymodule. Salt-proxy aborted.' log.error(errmsg) self._running = False raise SaltSystemExit(code=-1, msg=errmsg) - proxy_init_fn = self.proxy[fq_proxyname+'.init'] + proxy_init_fn = self.proxy[fq_proxyname + u'.init'] proxy_init_fn(self.opts) - self.opts['grains'] = salt.loader.grains(self.opts, proxy=self.proxy) + self.opts[u'grains'] = salt.loader.grains(self.opts, proxy=self.proxy) self.serial = salt.payload.Serial(self.opts) self.mod_opts = self._prep_mod_opts() self.matcher = Matcher(self.opts, self.functions) self.beacons = salt.beacons.Beacon(self.opts, self.functions) - uid = salt.utils.get_uid(user=self.opts.get('user', None)) - self.proc_dir = get_proc_dir(self.opts['cachedir'], uid=uid) + uid = salt.utils.get_uid(user=self.opts.get(u'user', None)) + self.proc_dir = get_proc_dir(self.opts[u'cachedir'], uid=uid) - if self.connected and self.opts['pillar']: + if self.connected and self.opts[u'pillar']: # The pillar has changed due to the connection to the master. # Reload the functions so that they can use the new pillar data. self.functions, self.returners, self.function_errors, self.executors = self._load_modules() - if hasattr(self, 'schedule'): + if hasattr(self, u'schedule'): self.schedule.functions = self.functions self.schedule.returners = self.returners - if not hasattr(self, 'schedule'): + if not hasattr(self, u'schedule'): self.schedule = salt.utils.schedule.Schedule( self.opts, self.functions, self.returners, - cleanup=[master_event(type='alive')], + cleanup=[master_event(type=u'alive')], proxy=self.proxy) # add default scheduling jobs to the minions scheduler - if self.opts['mine_enabled'] and 'mine.update' in self.functions: + if self.opts[u'mine_enabled'] and u'mine.update' in self.functions: self.schedule.add_job({ - '__mine_interval': + u'__mine_interval': { - 'function': 'mine.update', - 'minutes': self.opts['mine_interval'], - 'jid_include': True, - 'maxrunning': 2, - 'return_job': self.opts.get('mine_return_job', False) + u'function': u'mine.update', + u'minutes': self.opts[u'mine_interval'], + u'jid_include': True, + u'maxrunning': 2, + u'return_job': self.opts.get(u'mine_return_job', False) } }, persist=True) - log.info('Added mine.update to scheduler') + log.info(u'Added mine.update to scheduler') else: - self.schedule.delete_job('__mine_interval', persist=True) + self.schedule.delete_job(u'__mine_interval', persist=True) # add master_alive job if enabled - if (self.opts['transport'] != 'tcp' and - self.opts['master_alive_interval'] > 0): + if (self.opts[u'transport'] != u'tcp' and + self.opts[u'master_alive_interval'] > 0): self.schedule.add_job({ - master_event(type='alive', master=self.opts['master']): + master_event(type=u'alive', master=self.opts[u'master']): { - 'function': 'status.master', - 'seconds': self.opts['master_alive_interval'], - 'jid_include': True, - 'maxrunning': 1, - 'return_job': False, - 'kwargs': {'master': self.opts['master'], - 'connected': True} + u'function': u'status.master', + u'seconds': self.opts[u'master_alive_interval'], + u'jid_include': True, + u'maxrunning': 1, + u'return_job': False, + u'kwargs': {u'master': self.opts[u'master'], + u'connected': True} } }, persist=True) - if self.opts['master_failback'] and \ - 'master_list' in self.opts and \ - self.opts['master'] != self.opts['master_list'][0]: + if self.opts[u'master_failback'] and \ + u'master_list' in self.opts and \ + self.opts[u'master'] != self.opts[u'master_list'][0]: self.schedule.add_job({ - master_event(type='failback'): + master_event(type=u'failback'): { - 'function': 'status.ping_master', - 'seconds': self.opts['master_failback_interval'], - 'jid_include': True, - 'maxrunning': 1, - 'return_job': False, - 'kwargs': {'master': self.opts['master_list'][0]} + u'function': u'status.ping_master', + u'seconds': self.opts[u'master_failback_interval'], + u'jid_include': True, + u'maxrunning': 1, + u'return_job': False, + u'kwargs': {u'master': self.opts[u'master_list'][0]} } }, persist=True) else: - self.schedule.delete_job(master_event(type='failback'), persist=True) + self.schedule.delete_job(master_event(type=u'failback'), persist=True) else: - self.schedule.delete_job(master_event(type='alive', master=self.opts['master']), persist=True) - self.schedule.delete_job(master_event(type='failback'), persist=True) + self.schedule.delete_job(master_event(type=u'alive', master=self.opts[u'master']), persist=True) + self.schedule.delete_job(master_event(type=u'failback'), persist=True) # proxy keepalive - proxy_alive_fn = fq_proxyname+'.alive' + proxy_alive_fn = fq_proxyname+u'.alive' if (proxy_alive_fn in self.proxy - and 'status.proxy_reconnect' in self.functions - and self.opts.get('proxy_keep_alive', True)): + and u'status.proxy_reconnect' in self.functions + and self.opts.get(u'proxy_keep_alive', True)): # if `proxy_keep_alive` is either not specified, either set to False does not retry reconnecting self.schedule.add_job({ - '__proxy_keepalive': + u'__proxy_keepalive': { - 'function': 'status.proxy_reconnect', - 'minutes': self.opts.get('proxy_keep_alive_interval', 1), # by default, check once per minute - 'jid_include': True, - 'maxrunning': 1, - 'return_job': False, - 'kwargs': { - 'proxy_name': fq_proxyname + u'function': u'status.proxy_reconnect', + u'minutes': self.opts.get(u'proxy_keep_alive_interval', 1), # by default, check once per minute + u'jid_include': True, + u'maxrunning': 1, + u'return_job': False, + u'kwargs': { + u'proxy_name': fq_proxyname } } }, persist=True) self.schedule.enable_schedule() else: - self.schedule.delete_job('__proxy_keepalive', persist=True) + self.schedule.delete_job(u'__proxy_keepalive', persist=True) # Sync the grains here so the proxy can communicate them to the master - self.functions['saltutil.sync_grains'](saltenv='base') - self.grains_cache = self.opts['grains'] + self.functions[u'saltutil.sync_grains'](saltenv=u'base') + self.grains_cache = self.opts[u'grains'] self.ready = True @classmethod @@ -3267,10 +3390,10 @@ class ProxyMinion(Minion): if not minion_instance: minion_instance = cls(opts) minion_instance.connected = connected - if not hasattr(minion_instance, 'functions'): + if not hasattr(minion_instance, u'functions'): # Need to load the modules so they get all the dunder variables functions, returners, function_errors, executors = ( - minion_instance._load_modules(grains=opts['grains']) + minion_instance._load_modules(grains=opts[u'grains']) ) minion_instance.functions = functions minion_instance.returners = returners @@ -3285,38 +3408,38 @@ class ProxyMinion(Minion): # And re-load the modules so the __proxy__ variable gets injected functions, returners, function_errors, executors = ( - minion_instance._load_modules(grains=opts['grains']) + minion_instance._load_modules(grains=opts[u'grains']) ) minion_instance.functions = functions minion_instance.returners = returners minion_instance.function_errors = function_errors minion_instance.executors = executors - minion_instance.functions.pack['__proxy__'] = minion_instance.proxy - minion_instance.proxy.pack['__salt__'] = minion_instance.functions - minion_instance.proxy.pack['__ret__'] = minion_instance.returners - minion_instance.proxy.pack['__pillar__'] = minion_instance.opts['pillar'] + minion_instance.functions.pack[u'__proxy__'] = minion_instance.proxy + minion_instance.proxy.pack[u'__salt__'] = minion_instance.functions + minion_instance.proxy.pack[u'__ret__'] = minion_instance.returners + minion_instance.proxy.pack[u'__pillar__'] = minion_instance.opts[u'pillar'] # Reload utils as well (chicken and egg, __utils__ needs __proxy__ and __proxy__ needs __utils__ minion_instance.utils = salt.loader.utils(minion_instance.opts, proxy=minion_instance.proxy) - minion_instance.proxy.pack['__utils__'] = minion_instance.utils + minion_instance.proxy.pack[u'__utils__'] = minion_instance.utils # Reload all modules so all dunder variables are injected minion_instance.proxy.reload_modules() - fq_proxyname = opts['proxy']['proxytype'] - proxy_init_fn = minion_instance.proxy[fq_proxyname+'.init'] + fq_proxyname = opts[u'proxy'][u'proxytype'] + proxy_init_fn = minion_instance.proxy[fq_proxyname + u'.init'] proxy_init_fn(opts) - if not hasattr(minion_instance, 'serial'): + if not hasattr(minion_instance, u'serial'): minion_instance.serial = salt.payload.Serial(opts) - if not hasattr(minion_instance, 'proc_dir'): - uid = salt.utils.get_uid(user=opts.get('user', None)) + if not hasattr(minion_instance, u'proc_dir'): + uid = salt.utils.get_uid(user=opts.get(u'user', None)) minion_instance.proc_dir = ( - get_proc_dir(opts['cachedir'], uid=uid) + get_proc_dir(opts[u'cachedir'], uid=uid) ) with tornado.stack_context.StackContext(minion_instance.ctx): - if isinstance(data['fun'], tuple) or isinstance(data['fun'], list): + if isinstance(data[u'fun'], tuple) or isinstance(data[u'fun'], list): Minion._thread_multi_return(minion_instance, opts, data) else: Minion._thread_return(minion_instance, opts, data) diff --git a/salt/modules/acme.py b/salt/modules/acme.py index b07fab81971..af0ab0175f0 100644 --- a/salt/modules/acme.py +++ b/salt/modules/acme.py @@ -31,11 +31,11 @@ import datetime import os # Import salt libs -import salt.utils +import salt.utils.path log = logging.getLogger(__name__) -LEA = salt.utils.which_bin(['certbot', 'letsencrypt', +LEA = salt.utils.path.which_bin(['certbot', 'letsencrypt', 'certbot-auto', 'letsencrypt-auto', '/opt/letsencrypt/letsencrypt-auto']) LE_LIVE = '/etc/letsencrypt/live/' @@ -125,7 +125,8 @@ def cert(name, salt 'gitlab.example.com' acme.cert dev.example.com "[gitlab.example.com]" test_cert=True renew=14 webroot=/opt/gitlab/embedded/service/gitlab-rails/public ''' - cmd = [LEA, 'certonly', '--quiet'] + # cmd = [LEA, 'certonly', '--quiet'] + cmd = [LEA, 'certonly'] cert_file = _cert_file(name, 'cert') if not __salt__['file.file_exists'](cert_file): diff --git a/salt/modules/aliases.py b/salt/modules/aliases.py index 0be801837b0..b4621a123d0 100644 --- a/salt/modules/aliases.py +++ b/salt/modules/aliases.py @@ -11,12 +11,12 @@ import stat import tempfile # Import salt libs -import salt.utils -from salt.utils import which as _which +import salt.utils.files +import salt.utils.path from salt.exceptions import SaltInvocationError # Import third party libs -import salt.ext.six as six +from salt.ext import six __outputter__ = { 'rm_alias': 'txt', @@ -49,7 +49,7 @@ def __parse_aliases(): ret = [] if not os.path.isfile(afn): return ret - with salt.utils.fopen(afn, 'r') as ifile: + with salt.utils.files.fopen(afn, 'r') as ifile: for line in ifile: match = __ALIAS_RE.match(line) if match: @@ -97,7 +97,7 @@ def __write_aliases_file(lines): os.rename(out.name, afn) # Search $PATH for the newalises command - newaliases = _which('newaliases') + newaliases = salt.utils.path.which('newaliases') if newaliases is not None: __salt__['cmd.run'](newaliases) diff --git a/salt/modules/alternatives.py b/salt/modules/alternatives.py index f7a016b1506..ad21785fc39 100644 --- a/salt/modules/alternatives.py +++ b/salt/modules/alternatives.py @@ -11,10 +11,11 @@ import os import logging # Import Salt libs -import salt.utils +import salt.utils.files +import salt.utils.path # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six __outputter__ = { @@ -89,7 +90,7 @@ def show_link(name): path += 'alternatives/{0}'.format(name) try: - with salt.utils.fopen(path, 'rb') as r_file: + with salt.utils.files.fopen(path, 'rb') as r_file: contents = r_file.read() if six.PY3: contents = contents.decode(__salt_system_encoding__) @@ -241,4 +242,4 @@ def _read_link(name): Throws an OSError if the link does not exist ''' alt_link_path = '/etc/alternatives/{0}'.format(name) - return os.readlink(alt_link_path) + return salt.utils.path.readlink(alt_link_path) diff --git a/salt/modules/apache.py b/salt/modules/apache.py index ad502df5300..c236f1c74fa 100644 --- a/salt/modules/apache.py +++ b/salt/modules/apache.py @@ -16,7 +16,7 @@ import logging # Import 3rd-party libs # pylint: disable=import-error,no-name-in-module -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import cStringIO from salt.ext.six.moves.urllib.error import URLError from salt.ext.six.moves.urllib.request import ( @@ -29,7 +29,8 @@ from salt.ext.six.moves.urllib.request import ( # pylint: enable=import-error,no-name-in-module # Import salt libs -import salt.utils +import salt.utils.files +import salt.utils.path log = logging.getLogger(__name__) @@ -39,7 +40,7 @@ def __virtual__(): Only load the module if apache is installed ''' cmd = _detect_os() - if salt.utils.which(cmd): + if salt.utils.path.which(cmd): return 'apache' return (False, 'The apache execution module cannot be loaded: apache is not installed.') @@ -52,7 +53,7 @@ def _detect_os(): os_family = __grains__['os_family'] if os_family == 'RedHat': return 'apachectl' - elif os_family == 'Debian' or os_family == 'SUSE': + elif os_family == 'Debian' or os_family == 'Suse': return 'apache2ctl' else: return 'apachectl' @@ -398,7 +399,7 @@ def server_status(profile='default'): def _parse_config(conf, slot=None): ret = cStringIO() - if isinstance(conf, str): + if isinstance(conf, six.string_types): if slot: print('{0} {1}'.format(slot, conf), file=ret, end='') else: @@ -413,7 +414,7 @@ def _parse_config(conf, slot=None): ) del conf['this'] for key, value in six.iteritems(conf): - if isinstance(value, str): + if isinstance(value, six.string_types): print('{0} {1}'.format(key, value), file=ret) elif isinstance(value, list): print(_parse_config(value, key), file=ret) @@ -446,11 +447,15 @@ def config(name, config, edit=True): salt '*' apache.config /etc/httpd/conf.d/ports.conf config="[{'Listen': '22'}]" ''' + configs = [] for entry in config: key = next(six.iterkeys(entry)) - configs = _parse_config(entry[key], key) - if edit: - with salt.utils.fopen(name, 'w') as configfile: - configfile.write('# This file is managed by Salt.\n') - configfile.write(configs) - return configs + configs.append(_parse_config(entry[key], key)) + + # Python auto-correct line endings + configstext = "\n".join(configs) + if edit: + with salt.utils.files.fopen(name, 'w') as configfile: + configfile.write('# This file is managed by Salt.\n') + configfile.write(configstext) + return configstext diff --git a/salt/modules/apcups.py b/salt/modules/apcups.py index 68abb7f565e..647ae78280f 100644 --- a/salt/modules/apcups.py +++ b/salt/modules/apcups.py @@ -9,6 +9,7 @@ import logging # Import Salt libs import salt.utils +import salt.utils.path import salt.utils.decorators as decorators log = logging.getLogger(__name__) @@ -22,7 +23,7 @@ def _check_apcaccess(): ''' Looks to see if apcaccess is present on the system ''' - return salt.utils.which('apcaccess') + return salt.utils.path.which('apcaccess') def __virtual__(): diff --git a/salt/modules/apf.py b/salt/modules/apf.py index 55e8a401b23..2a007eff36e 100644 --- a/salt/modules/apf.py +++ b/salt/modules/apf.py @@ -18,15 +18,15 @@ except ImportError: # Import Salt Libs +import salt.utils.path from salt.exceptions import CommandExecutionError -import salt.utils def __virtual__(): ''' Only load if apf exists on the system ''' - if salt.utils.which('apf') is None: + if salt.utils.path.which('apf') is None: return (False, 'The apf execution module cannot be loaded: apf unavailable.') elif not IPTC_IMPORTED: @@ -40,7 +40,7 @@ def __apf_cmd(cmd): ''' Return the apf location ''' - apf_cmd = '{0} {1}'.format(salt.utils.which('apf'), cmd) + apf_cmd = '{0} {1}'.format(salt.utils.path.which('apf'), cmd) out = __salt__['cmd.run_all'](apf_cmd) if out['retcode'] != 0: diff --git a/salt/modules/aptly.py b/salt/modules/aptly.py new file mode 100644 index 00000000000..ab6acdd029e --- /dev/null +++ b/salt/modules/aptly.py @@ -0,0 +1,498 @@ +# -*- coding: utf-8 -*- +''' +Aptly Debian repository manager. + +.. versionadded:: Oxygen +''' +# Import python libs +from __future__ import absolute_import +import json +import logging +import os +import re + +# Import salt libs +from salt.exceptions import SaltInvocationError +import salt.utils.path +import salt.utils.stringutils as stringutils + +_DEFAULT_CONFIG_PATH = '/etc/aptly.conf' +_LOG = logging.getLogger(__name__) + +# Define the module's virtual name +__virtualname__ = 'aptly' + + +def __virtual__(): + ''' + Only works on systems with the aptly binary in the system path. + ''' + if salt.utils.path.which('aptly'): + return __virtualname__ + return (False, 'The aptly binaries required cannot be found or are not installed.') + + +def _cmd_run(cmd): + ''' + Run the aptly command. + + :return: The string output of the command. + :rtype: str + ''' + cmd.insert(0, 'aptly') + cmd_ret = __salt__['cmd.run_all'](cmd, ignore_retcode=True) + + if cmd_ret['retcode'] != 0: + _LOG.debug('Unable to execute command: %s\nError: %s', cmd, + cmd_ret['stderr']) + + return cmd_ret['stdout'] + + +def _format_repo_args(comment=None, component=None, distribution=None, + uploaders_file=None, saltenv='base'): + ''' + Format the common arguments for creating or editing a repository. + + :param str comment: The description of the repository. + :param str component: The default component to use when publishing. + :param str distribution: The default distribution to use when publishing. + :param str uploaders_file: The repository upload restrictions config. + :param str saltenv: The environment the file resides in. + + :return: A list of the arguments formatted as aptly arguments. + :rtype: list + ''' + ret = list() + cached_uploaders_path = None + settings = {'comment': comment, 'component': component, + 'distribution': distribution} + + if uploaders_file: + cached_uploaders_path = __salt__['cp.cache_file'](uploaders_file, saltenv) + + if not cached_uploaders_path: + _LOG.error('Unable to get cached copy of file: %s', uploaders_file) + return False + + for setting in settings: + if settings[setting] is not None: + ret.append('-{}={}'.format(setting, settings[setting])) + + if cached_uploaders_path: + ret.append('-uploaders-file={}'.format(cached_uploaders_path)) + + return ret + + +def _validate_config(config_path): + ''' + Validate that the configuration file exists and is readable. + + :param str config_path: The path to the configuration file for the aptly instance. + + :return: None + :rtype: None + ''' + _LOG.debug('Checking configuration file: %s', config_path) + + if not os.path.isfile(config_path): + message = 'Unable to get configuration file: {}'.format(config_path) + _LOG.error(message) + raise SaltInvocationError(message) + + +def get_config(config_path=_DEFAULT_CONFIG_PATH): + ''' + Get the configuration data. + + :param str config_path: The path to the configuration file for the aptly instance. + + :return: A dictionary containing the configuration data. + :rtype: dict + + CLI Example: + + .. code-block:: bash + + salt '*' aptly.get_config + ''' + _validate_config(config_path) + + cmd = ['config', 'show', '-config={}'.format(config_path)] + + cmd_ret = _cmd_run(cmd) + + return json.loads(cmd_ret) + + +def list_repos(config_path=_DEFAULT_CONFIG_PATH, with_packages=False): + ''' + List all of the repos. + + :param str config_path: The path to the configuration file for the aptly instance. + :param bool with_packages: Return a list of packages in the repo. + + :return: A dictionary of the repositories. + :rtype: dict + + CLI Example: + + .. code-block:: bash + + salt '*' aptly.list_repos + ''' + _validate_config(config_path) + + ret = dict() + cmd = ['repo', 'list', '-config={}'.format(config_path), '-raw=true'] + + cmd_ret = _cmd_run(cmd) + repos = [line.strip() for line in cmd_ret.splitlines()] + + _LOG.debug('Found repositories: %s', len(repos)) + + for name in repos: + ret[name] = get_repo(name=name, config_path=config_path, + with_packages=with_packages) + return ret + + +def get_repo(name, config_path=_DEFAULT_CONFIG_PATH, with_packages=False): + ''' + Get the details of the repository. + + :param str name: The name of the repository. + :param str config_path: The path to the configuration file for the aptly instance. + :param bool with_packages: Return a list of packages in the repo. + + :return: A dictionary containing information about the repository. + :rtype: dict + ''' + _validate_config(config_path) + + ret = dict() + cmd = ['repo', 'show', '-config={}'.format(config_path), + '-with-packages={}'.format(str(with_packages).lower()), + name] + + cmd_ret = _cmd_run(cmd) + + for line in cmd_ret.splitlines(): + try: + # Extract the settings and their values, and attempt to format + # them to match their equivalent setting names. + items = line.split(':') + key = items[0].lower().replace('default', '').strip() + key = ' '.join(key.split()).replace(' ', '_') + ret[key] = stringutils.to_none(stringutils.to_num(items[1].strip())) + except (AttributeError, IndexError): + # If the line doesn't have the separator or is otherwise invalid, skip it. + _LOG.debug('Skipping line: %s', line) + + if ret: + _LOG.debug('Found repository: %s', name) + else: + _LOG.debug('Unable to find repository: %s', name) + return ret + + +def new_repo(name, config_path=_DEFAULT_CONFIG_PATH, comment=None, component=None, + distribution=None, uploaders_file=None, from_snapshot=None, + saltenv='base'): + ''' + Create the new repository. + + :param str name: The name of the repository. + :param str config_path: The path to the configuration file for the aptly instance. + :param str comment: The description of the repository. + :param str component: The default component to use when publishing. + :param str distribution: The default distribution to use when publishing. + :param str uploaders_file: The repository upload restrictions config. + :param str from_snapshot: The snapshot to initialize the repository contents from. + :param str saltenv: The environment the file resides in. + + :return: A boolean representing whether all changes succeeded. + :rtype: bool + + CLI Example: + + .. code-block:: bash + + salt '*' aptly.new_repo name="test-repo" comment="Test main repo" component="main" distribution="trusty" + ''' + _validate_config(config_path) + + current_repo = __salt__['aptly.get_repo'](name=name) + + if current_repo: + _LOG.debug('Repository already exists: %s', name) + return True + + cmd = ['repo', 'create', '-config={}'.format(config_path)] + repo_params = _format_repo_args(comment=comment, component=component, + distribution=distribution, + uploaders_file=uploaders_file, saltenv=saltenv) + cmd.extend(repo_params) + cmd.append(name) + + if from_snapshot: + cmd.extend(['from', 'snapshot', from_snapshot]) + + _cmd_run(cmd) + repo = __salt__['aptly.get_repo'](name=name) + + if repo: + _LOG.debug('Created repo: %s', name) + return True + _LOG.error('Unable to create repo: %s', name) + return False + + +def set_repo(name, config_path=_DEFAULT_CONFIG_PATH, comment=None, component=None, + distribution=None, uploaders_file=None, saltenv='base'): + ''' + Configure the repository settings. + + :param str name: The name of the repository. + :param str config_path: The path to the configuration file for the aptly instance. + :param str comment: The description of the repository. + :param str component: The default component to use when publishing. + :param str distribution: The default distribution to use when publishing. + :param str uploaders_file: The repository upload restrictions config. + :param str from_snapshot: The snapshot to initialize the repository contents from. + :param str saltenv: The environment the file resides in. + + :return: A boolean representing whether all changes succeeded. + :rtype: bool + + CLI Example: + + .. code-block:: bash + + salt '*' aptly.set_repo name="test-repo" comment="Test universe repo" component="universe" distribution="xenial" + ''' + _validate_config(config_path) + + failed_settings = dict() + + # Only check for settings that were passed in and skip the rest. + settings = {'comment': comment, 'component': component, + 'distribution': distribution} + + for setting in list(settings): + if settings[setting] is None: + settings.pop(setting, None) + + current_settings = __salt__['aptly.get_repo'](name=name) + + if not current_settings: + _LOG.error('Unable to get repo: %s', name) + return False + + # Discard any additional settings that get_repo gives + # us that are not present in the provided arguments. + for current_setting in list(current_settings): + if current_setting not in settings: + current_settings.pop(current_setting, None) + + # Check the existing repo settings to see if they already have the desired values. + if settings == current_settings: + _LOG.debug('Settings already have the desired values for repository: %s', name) + return True + + cmd = ['repo', 'edit', '-config={}'.format(config_path)] + + repo_params = _format_repo_args(comment=comment, component=component, + distribution=distribution, + uploaders_file=uploaders_file, saltenv=saltenv) + cmd.extend(repo_params) + cmd.append(name) + + _cmd_run(cmd) + new_settings = __salt__['aptly.get_repo'](name=name) + + # Check the new repo settings to see if they have the desired values. + for setting in settings: + if settings[setting] != new_settings[setting]: + failed_settings.update({setting: settings[setting]}) + + if failed_settings: + _LOG.error('Unable to change settings for the repository: %s', name) + return False + _LOG.debug('Settings successfully changed to the desired values for repository: %s', name) + return True + + +def delete_repo(name, config_path=_DEFAULT_CONFIG_PATH, force=False): + ''' + Remove the repository. + + :param str name: The name of the repository. + :param str config_path: The path to the configuration file for the aptly instance. + :param bool force: Whether to remove the repository even if it is used as the source + of an existing snapshot. + + :return: A boolean representing whether all changes succeeded. + :rtype: bool + + CLI Example: + + .. code-block:: bash + + salt '*' aptly.delete_repo name="test-repo" + ''' + _validate_config(config_path) + + current_repo = __salt__['aptly.get_repo'](name=name) + + if not current_repo: + _LOG.debug('Repository already absent: %s', name) + return True + + cmd = ['repo', 'drop', '-config={}'.format(config_path), + '-force={}'.format(str(force).lower()), name] + + _cmd_run(cmd) + repo = __salt__['aptly.get_repo'](name=name) + + if repo: + _LOG.error('Unable to remove repo: %s', name) + return False + _LOG.debug('Removed repo: %s', name) + return True + + +def list_mirrors(config_path=_DEFAULT_CONFIG_PATH): + ''' + Get a list of all the mirrors. + + :param str config_path: The path to the configuration file for the aptly instance. + + :return: A list of the mirror names. + :rtype: list + + CLI Example: + + .. code-block:: bash + + salt '*' aptly.list_mirrors + ''' + _validate_config(config_path) + + cmd = ['mirror', 'list', '-config={}'.format(config_path), '-raw=true'] + + cmd_ret = _cmd_run(cmd) + ret = [line.strip() for line in cmd_ret.splitlines()] + + _LOG.debug('Found mirrors: %s', len(ret)) + return ret + + +def list_published(config_path=_DEFAULT_CONFIG_PATH): + ''' + Get a list of all the published repositories. + + :param str config_path: The path to the configuration file for the aptly instance. + + :return: A list of the published repository names. + :rtype: list + + CLI Example: + + .. code-block:: bash + + salt '*' aptly.list_published + ''' + _validate_config(config_path) + + cmd = ['publish', 'list', '-config={}'.format(config_path), '-raw=true'] + + cmd_ret = _cmd_run(cmd) + ret = [line.strip() for line in cmd_ret.splitlines()] + + _LOG.debug('Found published repositories: %s', len(ret)) + return ret + + +def list_snapshots(config_path=_DEFAULT_CONFIG_PATH, sort_by_time=False): + ''' + Get a list of all the snapshots. + + :param str config_path: The path to the configuration file for the aptly instance. + :param bool sort_by_time: Whether to sort by creation time instead of by name. + + :return: A list of the snapshot names. + :rtype: list + + CLI Example: + + .. code-block:: bash + + salt '*' aptly.list_snapshots + ''' + _validate_config(config_path) + + cmd = ['snapshot', 'list', '-config={}'.format(config_path), '-raw=true'] + + if sort_by_time: + cmd.append('-sort=time') + else: + cmd.append('-sort=name') + + cmd_ret = _cmd_run(cmd) + ret = [line.strip() for line in cmd_ret.splitlines()] + + _LOG.debug('Found snapshots: %s', len(ret)) + return ret + + +def cleanup_db(config_path=_DEFAULT_CONFIG_PATH, dry_run=False): + ''' + Remove data regarding unreferenced packages and delete files in the package pool that + are no longer being used by packages. + + :param bool dry_run: Report potential changes without making any changes. + + :return: A dictionary of the package keys and files that were removed. + :rtype: dict + + CLI Example: + + .. code-block:: bash + + salt '*' aptly.cleanup_db + ''' + _validate_config(config_path) + + ret = {'deleted_keys': list(), + 'deleted_files': list()} + + cmd = ['db', 'cleanup', '-config={}'.format(config_path), + '-dry-run={}'.format(str(dry_run).lower()), + '-verbose=true'] + + cmd_ret = _cmd_run(cmd) + + type_pattern = r'^List\s+[\w\s]+(?P(file|key)s)[\w\s]+:$' + list_pattern = r'^\s+-\s+(?P.*)$' + current_block = None + + for line in cmd_ret.splitlines(): + if current_block: + match = re.search(list_pattern, line) + if match: + package_type = 'deleted_{}'.format(current_block) + ret[package_type].append(match.group('package')) + else: + current_block = None + # Intentionally not using an else here, in case of a situation where + # the next list header might be bordered by the previous list. + if not current_block: + match = re.search(type_pattern, line) + if match: + current_block = match.group('package_type') + + _LOG.debug('Package keys identified for deletion: %s', len(ret['deleted_keys'])) + _LOG.debug('Package files identified for deletion: %s', len(ret['deleted_files'])) + return ret diff --git a/salt/modules/aptpkg.py b/salt/modules/aptpkg.py index c7b40b83300..d056c12a582 100644 --- a/salt/modules/aptpkg.py +++ b/salt/modules/aptpkg.py @@ -28,7 +28,7 @@ import json # Import third party libs import yaml # pylint: disable=no-name-in-module,import-error,redefined-builtin -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import range from salt.ext.six.moves.urllib.error import HTTPError from salt.ext.six.moves.urllib.request import Request as _Request, urlopen as _urlopen @@ -39,10 +39,14 @@ import salt.config import salt.syspaths from salt.modules.cmdmod import _parse_env import salt.utils +import salt.utils.args +import salt.utils.files import salt.utils.itertools +import salt.utils.path import salt.utils.pkg import salt.utils.pkg.deb import salt.utils.systemd +import salt.utils.versions from salt.exceptions import ( CommandExecutionError, MinionError, SaltInvocationError ) @@ -95,7 +99,7 @@ def __virtual__(): ''' Confirm this module is on a Debian based system ''' - if __grains__.get('os_family') in ('Kali', 'Debian', 'neon'): + if __grains__.get('os_family') in ('Kali', 'Debian', 'neon', 'Deepin'): return __virtualname__ elif __grains__.get('os_family', False) == 'Cumulus': return __virtualname__ @@ -139,19 +143,6 @@ def _reconstruct_ppa_name(owner_name, ppa_name): return 'ppa:{0}/{1}'.format(owner_name, ppa_name) -def _get_repo(**kwargs): - ''' - Check the kwargs for either 'fromrepo' or 'repo' and return the value. - 'fromrepo' takes precedence over 'repo'. - ''' - for key in ('fromrepo', 'repo'): - try: - return kwargs[key] - except KeyError: - pass - return '' - - def _check_apt(): ''' Abort if python-apt is not installed @@ -246,18 +237,11 @@ def latest_version(*names, **kwargs): ''' refresh = salt.utils.is_true(kwargs.pop('refresh', True)) show_installed = salt.utils.is_true(kwargs.pop('show_installed', False)) - if 'repo' in kwargs: - # Remember to kill _get_repo() too when removing this warning. - salt.utils.warn_until( - 'Hydrogen', - 'The \'repo\' argument to apt.latest_version is deprecated, and ' - 'will be removed in Salt {version}. Please use \'fromrepo\' ' - 'instead.' + raise SaltInvocationError( + 'The \'repo\' argument is invalid, use \'fromrepo\' instead' ) - fromrepo = _get_repo(**kwargs) - kwargs.pop('fromrepo', None) - kwargs.pop('repo', None) + fromrepo = kwargs.pop('fromrepo', None) cache_valid_time = kwargs.pop('cache_valid_time', 0) if len(names) == 0: @@ -316,7 +300,7 @@ def latest_version(*names, **kwargs): # to the install candidate, then the candidate is an upgrade, so # add it to the return dict if not any( - (salt.utils.compare_versions(ver1=x, + (salt.utils.versions.compare(ver1=x, oper='>=', ver2=candidate, cmp_func=version_cmp) @@ -761,12 +745,12 @@ def install(name=None, cver = old.get(pkgname, '') if reinstall and cver \ - and salt.utils.compare_versions(ver1=version_num, + and salt.utils.versions.compare(ver1=version_num, oper='==', ver2=cver, cmp_func=version_cmp): to_reinstall[pkgname] = pkgstr - elif not cver or salt.utils.compare_versions(ver1=version_num, + elif not cver or salt.utils.versions.compare(ver1=version_num, oper='>=', ver2=cver, cmp_func=version_cmp): @@ -1449,9 +1433,10 @@ def _get_upgradable(dist_upgrade=True, **kwargs): cmd.append('dist-upgrade') else: cmd.append('upgrade') - fromrepo = _get_repo(**kwargs) - if fromrepo: - cmd.extend(['-o', 'APT::Default-Release={0}'.format(fromrepo)]) + try: + cmd.extend(['-o', 'APT::Default-Release={0}'.format(kwargs['fromrepo'])]) + except KeyError: + pass call = __salt__['cmd.run_all'](cmd, python_shell=False, @@ -2143,44 +2128,44 @@ def mod_repo(repo, saltenv='base', **kwargs): The following options are available to modify a repo definition: - architectures - a comma separated list of supported architectures, e.g. ``amd64`` - If this option is not set, all architectures (configured in the - system) will be used. + architectures + A comma-separated list of supported architectures, e.g. ``amd64`` If + this option is not set, all architectures (configured in the system) + will be used. - comps - a comma separated list of components for the repo, e.g. ``main`` + comps + A comma separated list of components for the repo, e.g. ``main`` - file - a file name to be used + file + A file name to be used - keyserver - keyserver to get gpg key from + keyserver + Keyserver to get gpg key from - keyid - key id to load with the keyserver argument + keyid + Key ID to load with the ``keyserver`` argument - key_url - URL to a GPG key to add to the APT GPG keyring + key_url + URL to a GPG key to add to the APT GPG keyring - key_text - GPG key in string form to add to the APT GPG keyring + key_text + GPG key in string form to add to the APT GPG keyring - consolidate - if ``True``, will attempt to de-dup and consolidate sources + consolidate : False + If ``True``, will attempt to de-duplicate and consolidate sources - comments - Sometimes you want to supply additional information, but not as - enabled configuration. All comments provided here will be joined - into a single string and appended to the repo configuration with a - comment marker (#) before it. + comments + Sometimes you want to supply additional information, but not as + enabled configuration. All comments provided here will be joined + into a single string and appended to the repo configuration with a + comment marker (#) before it. - .. versionadded:: 2015.8.9 + .. versionadded:: 2015.8.9 - .. note:: Due to the way keys are stored for APT, there is a known issue - where the key won't be updated unless another change is made - at the same time. Keys should be properly added on initial - configuration. + .. note:: + Due to the way keys are stored for APT, there is a known issue where + the key won't be updated unless another change is made at the same + time. Keys should be properly added on initial configuration. CLI Examples: @@ -2189,6 +2174,17 @@ def mod_repo(repo, saltenv='base', **kwargs): salt '*' pkg.mod_repo 'myrepo definition' uri=http://new/uri salt '*' pkg.mod_repo 'myrepo definition' comps=main,universe ''' + if 'refresh_db' in kwargs: + salt.utils.versions.warn_until( + 'Neon', + 'The \'refresh_db\' argument to \'pkg.mod_repo\' has been ' + 'renamed to \'refresh\'. Support for using \'refresh_db\' will be ' + 'removed in the Neon release of Salt.' + ) + refresh = kwargs['refresh_db'] + else: + refresh = kwargs.get('refresh', True) + _check_apt() # to ensure no one sets some key values that _shouldn't_ be changed on the # object itself, this is just a white-list of "ok" to set properties @@ -2197,7 +2193,7 @@ def mod_repo(repo, saltenv='base', **kwargs): # secure PPAs cannot be supported as of the time of this code # implementation via apt-add-repository. The code path for # secure PPAs should be the same as urllib method - if salt.utils.which('apt-add-repository') \ + if salt.utils.path.which('apt-add-repository') \ and 'ppa_auth' not in kwargs: repo_info = get_repo(repo) if repo_info: @@ -2221,7 +2217,7 @@ def mod_repo(repo, saltenv='base', **kwargs): ) ) # explicit refresh when a repo is modified. - if kwargs.get('refresh_db', True): + if refresh: refresh_db() return {repo: out} else: @@ -2425,7 +2421,7 @@ def mod_repo(repo, saltenv='base', **kwargs): setattr(mod_source, key, kwargs[key]) sources.save() # on changes, explicitly refresh - if kwargs.get('refresh_db', True): + if refresh: refresh_db() return { repo: { @@ -2662,7 +2658,7 @@ def set_selections(path=None, selection=None, clear=False, saltenv='base'): if path: path = __salt__['cp.cache_file'](path, saltenv) - with salt.utils.fopen(path, 'r') as ifile: + with salt.utils.files.fopen(path, 'r') as ifile: content = ifile.readlines() selection = _parse_selections(content) @@ -2812,10 +2808,10 @@ def info_installed(*names, **kwargs): salt '*' pkg.info_installed ... salt '*' pkg.info_installed failhard=false ''' - kwargs = salt.utils.clean_kwargs(**kwargs) + kwargs = salt.utils.args.clean_kwargs(**kwargs) failhard = kwargs.pop('failhard', True) if kwargs: - salt.utils.invalid_kwargs(kwargs) + salt.utils.args.invalid_kwargs(kwargs) ret = dict() for pkg_name, pkg_nfo in __salt__['lowpkg.info'](*names, failhard=failhard).items(): diff --git a/salt/modules/archive.py b/salt/modules/archive.py index 68180fef151..70ef0bdeccf 100644 --- a/salt/modules/archive.py +++ b/salt/modules/archive.py @@ -23,7 +23,7 @@ except ImportError: from pipes import quote as _quote # Import third party libs -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves.urllib.parse import urlparse as _urlparse # pylint: disable=no-name-in-module try: import rarfile @@ -33,11 +33,16 @@ except ImportError: # Import salt libs from salt.exceptions import SaltInvocationError, CommandExecutionError -import salt.utils +import salt.utils.decorators +import salt.utils.decorators.path import salt.utils.files -import salt.utils.itertools +import salt.utils.path +import salt.utils.platform import salt.utils.templates +if salt.utils.platform.is_windows(): + import win32file + # TODO: Check that the passed arguments are correct # Don't shadow built-in's. @@ -188,7 +193,7 @@ def list_(name, info={'error': stderr} ) else: - if not salt.utils.which('tar'): + if not salt.utils.path.which('tar'): raise CommandExecutionError('\'tar\' command not available') if decompress_cmd is not None: # Guard against shell injection @@ -199,7 +204,7 @@ def list_(name, except AttributeError: raise CommandExecutionError('Invalid CLI options') else: - if salt.utils.which('xz') \ + if salt.utils.path.which('xz') \ and __salt__['cmd.retcode'](['xz', '-t', cached], python_shell=False, ignore_retcode=True) == 0: @@ -234,7 +239,7 @@ def list_(name, with contextlib.closing(zipfile.ZipFile(cached)) as zip_archive: for member in zip_archive.infolist(): path = member.filename - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): if path.endswith('/'): # zipfile.ZipInfo objects on windows use forward # slash at end of the directory name. @@ -279,7 +284,7 @@ def list_(name, else: files.append(path) else: - if not salt.utils.which('rar'): + if not salt.utils.path.which('rar'): raise CommandExecutionError( 'rar command not available, is it installed?' ) @@ -445,7 +450,7 @@ def _expand_sources(sources): for path in _glob(source)] -@salt.utils.decorators.which('tar') +@salt.utils.decorators.path.which('tar') def tar(options, tarfile, sources=None, dest=None, cwd=None, template=None, runas=None): ''' @@ -533,7 +538,7 @@ def tar(options, tarfile, sources=None, dest=None, python_shell=False).splitlines() -@salt.utils.decorators.which('gzip') +@salt.utils.decorators.path.which('gzip') def gzip(sourcefile, template=None, runas=None, options=None): ''' Uses the gzip command to create gzip files @@ -573,7 +578,7 @@ def gzip(sourcefile, template=None, runas=None, options=None): python_shell=False).splitlines() -@salt.utils.decorators.which('gunzip') +@salt.utils.decorators.path.which('gunzip') def gunzip(gzipfile, template=None, runas=None, options=None): ''' Uses the gunzip command to unpack gzip files @@ -613,7 +618,7 @@ def gunzip(gzipfile, template=None, runas=None, options=None): python_shell=False).splitlines() -@salt.utils.decorators.which('zip') +@salt.utils.decorators.path.which('zip') def cmd_zip(zip_file, sources, template=None, cwd=None, runas=None): ''' .. versionadded:: 2015.5.0 @@ -784,10 +789,9 @@ def zip_(zip_file, sources, template=None, cwd=None, runas=None): if os.path.isdir(src): for dir_name, sub_dirs, files in os.walk(src): if cwd and dir_name.startswith(cwd): - arc_dir = salt.utils.relpath(dir_name, cwd) + arc_dir = os.path.relpath(dir_name, cwd) else: - arc_dir = salt.utils.relpath(dir_name, - rel_root) + arc_dir = os.path.relpath(dir_name, rel_root) if arc_dir: archived_files.append(arc_dir + '/') zfile.write(dir_name, arc_dir) @@ -798,9 +802,9 @@ def zip_(zip_file, sources, template=None, cwd=None, runas=None): zfile.write(abs_name, arc_name) else: if cwd and src.startswith(cwd): - arc_name = salt.utils.relpath(src, cwd) + arc_name = os.path.relpath(src, cwd) else: - arc_name = salt.utils.relpath(src, rel_root) + arc_name = os.path.relpath(src, rel_root) archived_files.append(arc_name) zfile.write(src, arc_name) except Exception as exc: @@ -820,7 +824,7 @@ def zip_(zip_file, sources, template=None, cwd=None, runas=None): return archived_files -@salt.utils.decorators.which('unzip') +@salt.utils.decorators.path.which('unzip') def cmd_unzip(zip_file, dest, excludes=None, @@ -1050,7 +1054,7 @@ def unzip(zip_file, cleaned_files.extend([x for x in files if x not in excludes]) for target in cleaned_files: if target not in excludes: - if salt.utils.is_windows() is False: + if salt.utils.platform.is_windows() is False: info = zfile.getinfo(target) # Check if zipped file is a symbolic link if stat.S_ISLNK(info.external_attr >> 16): @@ -1059,15 +1063,19 @@ def unzip(zip_file, continue zfile.extract(target, dest, password) if extract_perms: - perm = zfile.getinfo(target).external_attr >> 16 - if perm == 0: - umask_ = os.umask(0) - os.umask(umask_) - if target.endswith('/'): - perm = 0o777 & ~umask_ - else: - perm = 0o666 & ~umask_ - os.chmod(os.path.join(dest, target), perm) + if not salt.utils.platform.is_windows(): + perm = zfile.getinfo(target).external_attr >> 16 + if perm == 0: + umask_ = os.umask(0) + os.umask(umask_) + if target.endswith('/'): + perm = 0o777 & ~umask_ + else: + perm = 0o666 & ~umask_ + os.chmod(os.path.join(dest, target), perm) + else: + win32_attr = zfile.getinfo(target).external_attr & 0xFF + win32file.SetFileAttributes(os.path.join(dest, target), win32_attr) except Exception as exc: if runas: os.seteuid(euid) @@ -1149,7 +1157,7 @@ def is_encrypted(name, clean=False, saltenv='base'): return ret -@salt.utils.decorators.which('rar') +@salt.utils.decorators.path.which('rar') def rar(rarfile, sources, template=None, cwd=None, runas=None): ''' Uses `rar for Linux`_ to create rar files @@ -1200,7 +1208,7 @@ def rar(rarfile, sources, template=None, cwd=None, runas=None): python_shell=False).splitlines() -@salt.utils.decorators.which_bin(('unrar', 'rar')) +@salt.utils.decorators.path.which_bin(('unrar', 'rar')) def unrar(rarfile, dest, excludes=None, template=None, runas=None, trim_output=False): ''' Uses `rar for Linux`_ to unpack rar files @@ -1235,7 +1243,7 @@ def unrar(rarfile, dest, excludes=None, template=None, runas=None, trim_output=F if isinstance(excludes, six.string_types): excludes = [entry.strip() for entry in excludes.split(',')] - cmd = [salt.utils.which_bin(('unrar', 'rar')), + cmd = [salt.utils.path.which_bin(('unrar', 'rar')), 'x', '-idp', '{0}'.format(rarfile)] if excludes is not None: for exclude in excludes: @@ -1279,14 +1287,14 @@ def _render_filenames(filenames, zip_file, saltenv, template): ''' # write out path to temp file tmp_path_fn = salt.utils.files.mkstemp() - with salt.utils.fopen(tmp_path_fn, 'w+') as fp_: + with salt.utils.files.fopen(tmp_path_fn, 'w+') as fp_: fp_.write(contents) data = salt.utils.templates.TEMPLATE_REGISTRY[template]( tmp_path_fn, to_str=True, **kwargs ) - salt.utils.safe_rm(tmp_path_fn) + salt.utils.files.safe_rm(tmp_path_fn) if not data['result']: # Failed to render the template raise CommandExecutionError( diff --git a/salt/modules/artifactory.py b/salt/modules/artifactory.py index d521e786f3a..74572cbad5e 100644 --- a/salt/modules/artifactory.py +++ b/salt/modules/artifactory.py @@ -38,7 +38,7 @@ def __virtual__(): return True -def get_latest_snapshot(artifactory_url, repository, group_id, artifact_id, packaging, target_dir='/tmp', target_file=None, classifier=None, username=None, password=None): +def get_latest_snapshot(artifactory_url, repository, group_id, artifact_id, packaging, target_dir='/tmp', target_file=None, classifier=None, username=None, password=None, use_literal_group_id=False): ''' Gets latest snapshot of the given artifact @@ -69,15 +69,15 @@ def get_latest_snapshot(artifactory_url, repository, group_id, artifact_id, pack headers = {} if username and password: headers['Authorization'] = 'Basic {0}'.format(base64.encodestring('{0}:{1}'.format(username, password)).replace('\n', '')) - artifact_metadata = _get_artifact_metadata(artifactory_url=artifactory_url, repository=repository, group_id=group_id, artifact_id=artifact_id, headers=headers) + artifact_metadata = _get_artifact_metadata(artifactory_url=artifactory_url, repository=repository, group_id=group_id, artifact_id=artifact_id, headers=headers, use_literal_group_id=use_literal_group_id) version = artifact_metadata['latest_version'] - snapshot_url, file_name = _get_snapshot_url(artifactory_url=artifactory_url, repository=repository, group_id=group_id, artifact_id=artifact_id, version=version, packaging=packaging, classifier=classifier, headers=headers) + snapshot_url, file_name = _get_snapshot_url(artifactory_url=artifactory_url, repository=repository, group_id=group_id, artifact_id=artifact_id, version=version, packaging=packaging, classifier=classifier, headers=headers, use_literal_group_id=use_literal_group_id) target_file = __resolve_target_file(file_name, target_dir, target_file) return __save_artifact(snapshot_url, target_file, headers) -def get_snapshot(artifactory_url, repository, group_id, artifact_id, packaging, version, snapshot_version=None, target_dir='/tmp', target_file=None, classifier=None, username=None, password=None): +def get_snapshot(artifactory_url, repository, group_id, artifact_id, packaging, version, snapshot_version=None, target_dir='/tmp', target_file=None, classifier=None, username=None, password=None, use_literal_group_id=False): ''' Gets snapshot of the desired version of the artifact @@ -109,13 +109,13 @@ def get_snapshot(artifactory_url, repository, group_id, artifact_id, packaging, headers = {} if username and password: headers['Authorization'] = 'Basic {0}'.format(base64.encodestring('{0}:{1}'.format(username, password)).replace('\n', '')) - snapshot_url, file_name = _get_snapshot_url(artifactory_url=artifactory_url, repository=repository, group_id=group_id, artifact_id=artifact_id, version=version, packaging=packaging, snapshot_version=snapshot_version, classifier=classifier, headers=headers) + snapshot_url, file_name = _get_snapshot_url(artifactory_url=artifactory_url, repository=repository, group_id=group_id, artifact_id=artifact_id, version=version, packaging=packaging, snapshot_version=snapshot_version, classifier=classifier, headers=headers, use_literal_group_id=use_literal_group_id) target_file = __resolve_target_file(file_name, target_dir, target_file) return __save_artifact(snapshot_url, target_file, headers) -def get_latest_release(artifactory_url, repository, group_id, artifact_id, packaging, target_dir='/tmp', target_file=None, classifier=None, username=None, password=None): +def get_latest_release(artifactory_url, repository, group_id, artifact_id, packaging, target_dir='/tmp', target_file=None, classifier=None, username=None, password=None, use_literal_group_id=False): ''' Gets the latest release of the artifact @@ -146,13 +146,13 @@ def get_latest_release(artifactory_url, repository, group_id, artifact_id, packa if username and password: headers['Authorization'] = 'Basic {0}'.format(base64.encodestring('{0}:{1}'.format(username, password)).replace('\n', '')) version = __find_latest_version(artifactory_url=artifactory_url, repository=repository, group_id=group_id, artifact_id=artifact_id, headers=headers) - release_url, file_name = _get_release_url(repository, group_id, artifact_id, packaging, version, artifactory_url, classifier) + release_url, file_name = _get_release_url(repository, group_id, artifact_id, packaging, version, artifactory_url, classifier, use_literal_group_id) target_file = __resolve_target_file(file_name, target_dir, target_file) return __save_artifact(release_url, target_file, headers) -def get_release(artifactory_url, repository, group_id, artifact_id, packaging, version, target_dir='/tmp', target_file=None, classifier=None, username=None, password=None): +def get_release(artifactory_url, repository, group_id, artifact_id, packaging, version, target_dir='/tmp', target_file=None, classifier=None, username=None, password=None, use_literal_group_id=False): ''' Gets the specified release of the artifact @@ -184,7 +184,7 @@ def get_release(artifactory_url, repository, group_id, artifact_id, packaging, v headers = {} if username and password: headers['Authorization'] = 'Basic {0}'.format(base64.encodestring('{0}:{1}'.format(username, password)).replace('\n', '')) - release_url, file_name = _get_release_url(repository, group_id, artifact_id, packaging, version, artifactory_url, classifier) + release_url, file_name = _get_release_url(repository, group_id, artifact_id, packaging, version, artifactory_url, classifier, use_literal_group_id) target_file = __resolve_target_file(file_name, target_dir, target_file) return __save_artifact(release_url, target_file, headers) @@ -196,53 +196,56 @@ def __resolve_target_file(file_name, target_dir, target_file=None): return target_file -def _get_snapshot_url(artifactory_url, repository, group_id, artifact_id, version, packaging, snapshot_version=None, classifier=None, headers=None): +def _get_snapshot_url(artifactory_url, repository, group_id, artifact_id, version, packaging, snapshot_version=None, classifier=None, headers=None, use_literal_group_id=False): if headers is None: headers = {} has_classifier = classifier is not None and classifier != "" if snapshot_version is None: - snapshot_version_metadata = _get_snapshot_version_metadata(artifactory_url=artifactory_url, repository=repository, group_id=group_id, artifact_id=artifact_id, version=version, headers=headers) + try: + snapshot_version_metadata = _get_snapshot_version_metadata(artifactory_url=artifactory_url, repository=repository, group_id=group_id, artifact_id=artifact_id, version=version, headers=headers) + if packaging not in snapshot_version_metadata['snapshot_versions']: + error_message = '''Cannot find requested packaging '{packaging}' in the snapshot version metadata. + artifactory_url: {artifactory_url} + repository: {repository} + group_id: {group_id} + artifact_id: {artifact_id} + packaging: {packaging} + classifier: {classifier} + version: {version}'''.format( + artifactory_url=artifactory_url, + repository=repository, + group_id=group_id, + artifact_id=artifact_id, + packaging=packaging, + classifier=classifier, + version=version) + raise ArtifactoryError(error_message) - if packaging not in snapshot_version_metadata['snapshot_versions']: - error_message = '''Cannot find requested packaging '{packaging}' in the snapshot version metadata. - artifactory_url: {artifactory_url} - repository: {repository} - group_id: {group_id} - artifact_id: {artifact_id} - packaging: {packaging} - classifier: {classifier} - version: {version}'''.format( - artifactory_url=artifactory_url, - repository=repository, - group_id=group_id, - artifact_id=artifact_id, - packaging=packaging, - classifier=classifier, - version=version) - raise ArtifactoryError(error_message) + if has_classifier and classifier not in snapshot_version_metadata['snapshot_versions']: + error_message = '''Cannot find requested classifier '{classifier}' in the snapshot version metadata. + artifactory_url: {artifactory_url} + repository: {repository} + group_id: {group_id} + artifact_id: {artifact_id} + packaging: {packaging} + classifier: {classifier} + version: {version}'''.format( + artifactory_url=artifactory_url, + repository=repository, + group_id=group_id, + artifact_id=artifact_id, + packaging=packaging, + classifier=classifier, + version=version) + raise ArtifactoryError(error_message) - if has_classifier and classifier not in snapshot_version_metadata['snapshot_versions']: - error_message = '''Cannot find requested classifier '{classifier}' in the snapshot version metadata. - artifactory_url: {artifactory_url} - repository: {repository} - group_id: {group_id} - artifact_id: {artifact_id} - packaging: {packaging} - classifier: {classifier} - version: {version}'''.format( - artifactory_url=artifactory_url, - repository=repository, - group_id=group_id, - artifact_id=artifact_id, - packaging=packaging, - classifier=classifier, - version=version) - raise ArtifactoryError(error_message) + snapshot_version = snapshot_version_metadata['snapshot_versions'][packaging] + except CommandExecutionError as err: + log.error('Could not fetch maven-metadata.xml. Assuming snapshot_version=%s.', version) + snapshot_version = version - snapshot_version = snapshot_version_metadata['snapshot_versions'][packaging] - - group_url = __get_group_id_subpath(group_id) + group_url = __get_group_id_subpath(group_id, use_literal_group_id) file_name = '{artifact_id}-{snapshot_version}{classifier}.{packaging}'.format( artifact_id=artifact_id, @@ -262,8 +265,8 @@ def _get_snapshot_url(artifactory_url, repository, group_id, artifact_id, versio return snapshot_url, file_name -def _get_release_url(repository, group_id, artifact_id, packaging, version, artifactory_url, classifier=None): - group_url = __get_group_id_subpath(group_id) +def _get_release_url(repository, group_id, artifact_id, packaging, version, artifactory_url, classifier=None, use_literal_group_id=False): + group_url = __get_group_id_subpath(group_id, use_literal_group_id) # for released versions the suffix for the file is same as version file_name = '{artifact_id}-{version}{classifier}.{packaging}'.format( @@ -283,8 +286,8 @@ def _get_release_url(repository, group_id, artifact_id, packaging, version, arti return release_url, file_name -def _get_artifact_metadata_url(artifactory_url, repository, group_id, artifact_id): - group_url = __get_group_id_subpath(group_id) +def _get_artifact_metadata_url(artifactory_url, repository, group_id, artifact_id, use_literal_group_id=False): + group_url = __get_group_id_subpath(group_id, use_literal_group_id) # for released versions the suffix for the file is same as version artifact_metadata_url = '{artifactory_url}/{repository}/{group_url}/{artifact_id}/maven-metadata.xml'.format( artifactory_url=artifactory_url, @@ -295,13 +298,14 @@ def _get_artifact_metadata_url(artifactory_url, repository, group_id, artifact_i return artifact_metadata_url -def _get_artifact_metadata_xml(artifactory_url, repository, group_id, artifact_id, headers): +def _get_artifact_metadata_xml(artifactory_url, repository, group_id, artifact_id, headers, use_literal_group_id=False): artifact_metadata_url = _get_artifact_metadata_url( artifactory_url=artifactory_url, repository=repository, group_id=group_id, - artifact_id=artifact_id + artifact_id=artifact_id, + use_literal_group_id=use_literal_group_id ) try: @@ -318,8 +322,8 @@ def _get_artifact_metadata_xml(artifactory_url, repository, group_id, artifact_i return artifact_metadata_xml -def _get_artifact_metadata(artifactory_url, repository, group_id, artifact_id, headers): - metadata_xml = _get_artifact_metadata_xml(artifactory_url=artifactory_url, repository=repository, group_id=group_id, artifact_id=artifact_id, headers=headers) +def _get_artifact_metadata(artifactory_url, repository, group_id, artifact_id, headers, use_literal_group_id=False): + metadata_xml = _get_artifact_metadata_xml(artifactory_url=artifactory_url, repository=repository, group_id=group_id, artifact_id=artifact_id, headers=headers, use_literal_group_id=use_literal_group_id) root = ET.fromstring(metadata_xml) assert group_id == root.find('groupId').text @@ -331,8 +335,8 @@ def _get_artifact_metadata(artifactory_url, repository, group_id, artifact_id, h # functions for handling snapshots -def _get_snapshot_version_metadata_url(artifactory_url, repository, group_id, artifact_id, version): - group_url = __get_group_id_subpath(group_id) +def _get_snapshot_version_metadata_url(artifactory_url, repository, group_id, artifact_id, version, use_literal_group_id=False): + group_url = __get_group_id_subpath(group_id, use_literal_group_id) # for released versions the suffix for the file is same as version snapshot_version_metadata_url = '{artifactory_url}/{repository}/{group_url}/{artifact_id}/{version}/maven-metadata.xml'.format( artifactory_url=artifactory_url, @@ -344,14 +348,15 @@ def _get_snapshot_version_metadata_url(artifactory_url, repository, group_id, ar return snapshot_version_metadata_url -def _get_snapshot_version_metadata_xml(artifactory_url, repository, group_id, artifact_id, version, headers): +def _get_snapshot_version_metadata_xml(artifactory_url, repository, group_id, artifact_id, version, headers, use_literal_group_id=False): snapshot_version_metadata_url = _get_snapshot_version_metadata_url( artifactory_url=artifactory_url, repository=repository, group_id=group_id, artifact_id=artifact_id, - version=version + version=version, + use_literal_group_id=use_literal_group_id ) try: @@ -388,8 +393,8 @@ def _get_snapshot_version_metadata(artifactory_url, repository, group_id, artifa } -def __get_latest_version_url(artifactory_url, repository, group_id, artifact_id): - group_url = __get_group_id_subpath(group_id) +def __get_latest_version_url(artifactory_url, repository, group_id, artifact_id, use_literal_group_id=False): + group_url = __get_group_id_subpath(group_id, use_literal_group_id) # for released versions the suffix for the file is same as version latest_version_url = '{artifactory_url}/api/search/latestVersion?g={group_url}&a={artifact_id}&repos={repository}'.format( artifactory_url=artifactory_url, @@ -400,13 +405,14 @@ def __get_latest_version_url(artifactory_url, repository, group_id, artifact_id) return latest_version_url -def __find_latest_version(artifactory_url, repository, group_id, artifact_id, headers): +def __find_latest_version(artifactory_url, repository, group_id, artifact_id, headers, use_literal_group_id=False): latest_version_url = __get_latest_version_url( artifactory_url=artifactory_url, repository=repository, group_id=group_id, - artifact_id=artifact_id + artifact_id=artifact_id, + use_literal_group_id=use_literal_group_id ) try: @@ -465,7 +471,7 @@ def __save_artifact(artifact_url, target_file, headers): try: request = urllib.request.Request(artifact_url, None, headers) f = urllib.request.urlopen(request) - with salt.utils.fopen(target_file, "wb") as local_file: + with salt.utils.files.fopen(target_file, "wb") as local_file: local_file.write(f.read()) result['status'] = True result['comment'] = __append_comment(('Artifact downloaded from URL: {0}'.format(artifact_url)), result['comment']) @@ -478,9 +484,11 @@ def __save_artifact(artifact_url, target_file, headers): return result -def __get_group_id_subpath(group_id): - group_url = group_id.replace('.', '/') - return group_url +def __get_group_id_subpath(group_id, use_literal_group_id=False): + if not use_literal_group_id: + group_url = group_id.replace('.', '/') + return group_url + return group_id def __get_classifier_url(classifier): diff --git a/salt/modules/at.py b/salt/modules/at.py index 5a27e5b5ee1..d6700f07abf 100644 --- a/salt/modules/at.py +++ b/salt/modules/at.py @@ -23,7 +23,8 @@ from salt.ext.six.moves import map from salt.exceptions import CommandNotFoundError # Import salt libs -import salt.utils +import salt.utils.path +import salt.utils.platform # OS Families that should work (Ubuntu and Debian are the default) # TODO: Refactor some of this module to remove the checks for binaries @@ -38,9 +39,9 @@ def __virtual__(): ''' Most everything has the ability to support at(1) ''' - if salt.utils.is_windows() or salt.utils.is_sunos(): + if salt.utils.platform.is_windows() or salt.utils.platform.is_sunos(): return (False, 'The at module could not be loaded: unsupported platform') - if salt.utils.which('at') is None: + if salt.utils.path.which('at') is None: return (False, 'The at module could not be loaded: at command not found') return __virtualname__ @@ -49,7 +50,7 @@ def _cmd(binary, *args): ''' Wrapper to run at(1) or return None. ''' - binary = salt.utils.which(binary) + binary = salt.utils.path.which(binary) if not binary: raise CommandNotFoundError('{0}: command not found'.format(binary)) cmd = [binary] + list(args) @@ -163,7 +164,7 @@ def atrm(*args): ''' # Need to do this here also since we use atq() - if not salt.utils.which('at'): + if not salt.utils.path.which('at'): return '\'at.atrm\' is not available.' if not args: @@ -211,7 +212,7 @@ def at(*args, **kwargs): # pylint: disable=C0103 # Shim to produce output similar to what __virtual__() should do # but __salt__ isn't available in __virtual__() - binary = salt.utils.which('at') + binary = salt.utils.path.which('at') if not binary: return '\'at.at\' is not available.' diff --git a/salt/modules/at_solaris.py b/salt/modules/at_solaris.py index 27cd878024f..2a16625df52 100644 --- a/salt/modules/at_solaris.py +++ b/salt/modules/at_solaris.py @@ -25,7 +25,9 @@ import logging from salt.ext.six.moves import map # Import salt libs -import salt.utils +import salt.utils.files +import salt.utils.path +import salt.utils.platform log = logging.getLogger(__name__) __virtualname__ = 'at' @@ -35,11 +37,11 @@ def __virtual__(): ''' We only deal with Solaris' specific version of at ''' - if not salt.utils.is_sunos(): + if not salt.utils.platform.is_sunos(): return (False, 'The at module could not be loaded: unsupported platform') - if not salt.utils.which('at') or \ - not salt.utils.which('atq') or \ - not salt.utils.which('atrm'): + if not salt.utils.path.which('at') or \ + not salt.utils.path.which('atq') or \ + not salt.utils.path.which('atrm'): return (False, 'The at module could not be loaded: at command not found') return __virtualname__ @@ -100,7 +102,7 @@ def atq(tag=None): job=job ) if __salt__['file.file_exists'](atjob_file): - with salt.utils.fopen(atjob_file, 'r') as atjob: + with salt.utils.files.fopen(atjob_file, 'r') as atjob: for line in atjob: tmp = job_kw_regex.match(line) if tmp: @@ -224,7 +226,7 @@ def atc(jobid): job=jobid ) if __salt__['file.file_exists'](atjob_file): - with salt.utils.fopen(atjob_file, 'r') as rfh: + with salt.utils.files.fopen(atjob_file, 'r') as rfh: return "".join(rfh.readlines()) else: return {'error': 'invalid job id \'{0}\''.format(jobid)} diff --git a/salt/modules/augeas_cfg.py b/salt/modules/augeas_cfg.py index 6b1f1e7b1bf..e0f78329ec8 100644 --- a/salt/modules/augeas_cfg.py +++ b/salt/modules/augeas_cfg.py @@ -30,7 +30,7 @@ import os import re import logging from salt.ext.six.moves import zip -import salt.ext.six as six +from salt.ext import six # Make sure augeas python interface is installed HAS_AUGEAS = False @@ -42,6 +42,7 @@ except ImportError: # Import salt libs import salt.utils +import salt.utils.args from salt.exceptions import SaltInvocationError log = logging.getLogger(__name__) @@ -199,7 +200,7 @@ def execute(context=None, lens=None, commands=(), load_path=None): method = METHOD_MAP[cmd] nargs = arg_map[method] - parts = salt.utils.shlex_split(arg) + parts = salt.utils.args.shlex_split(arg) if len(parts) not in nargs: err = '{0} takes {1} args: {2}'.format(method, nargs, parts) diff --git a/salt/modules/aws_sqs.py b/salt/modules/aws_sqs.py index 3c615573bdc..a906899718f 100644 --- a/salt/modules/aws_sqs.py +++ b/salt/modules/aws_sqs.py @@ -10,7 +10,8 @@ import json # Import salt libs import salt.utils -import salt.ext.six as six +import salt.utils.path +from salt.ext import six log = logging.getLogger(__name__) @@ -18,7 +19,7 @@ _OUTPUT = '--output json' def __virtual__(): - if salt.utils.which('aws'): + if salt.utils.path.which('aws'): # awscli is installed, load the module return True return (False, 'The module aws_sqs could not be loaded: aws command not found') diff --git a/salt/modules/bamboohr.py b/salt/modules/bamboohr.py index 353e736c63d..0fc4acd8fb2 100644 --- a/salt/modules/bamboohr.py +++ b/salt/modules/bamboohr.py @@ -20,7 +20,7 @@ import logging # Import salt libs import salt.utils.http -import salt.ext.six as six +from salt.ext import six from salt._compat import ElementTree as ET log = logging.getLogger(__name__) diff --git a/salt/modules/bcache.py b/salt/modules/bcache.py index 8e87256dbd8..23baba0034d 100644 --- a/salt/modules/bcache.py +++ b/salt/modules/bcache.py @@ -25,7 +25,7 @@ import re from salt.ext import six # Import salt libs -import salt.utils +import salt.utils.path log = logging.getLogger(__name__) @@ -44,14 +44,14 @@ __func_alias__ = { 'super_': 'super', } -HAS_BLKDISCARD = salt.utils.which('blkdiscard') is not None +HAS_BLKDISCARD = salt.utils.path.which('blkdiscard') is not None def __virtual__(): ''' Only work when make-bcache is installed ''' - return salt.utils.which('make-bcache') is not None + return salt.utils.path.which('make-bcache') is not None def uuid(dev=None): diff --git a/salt/modules/beacons.py b/salt/modules/beacons.py index 61e2042000f..7e095ed656c 100644 --- a/salt/modules/beacons.py +++ b/salt/modules/beacons.py @@ -14,8 +14,9 @@ import os import yaml # Import Salt libs -import salt.utils +import salt.ext.six as six import salt.utils.event +import salt.utils.files from salt.ext.six.moves import map # Get logging started @@ -69,6 +70,47 @@ def list_(return_yaml=True): return {'beacons': {}} +def list_available(return_yaml=True): + ''' + List the beacons currently available on the minion + + :param return_yaml: Whether to return YAML formatted output, default True + :return: List of currently configured Beacons. + + CLI Example: + + .. code-block:: bash + + salt '*' beacons.list_available + + ''' + beacons = None + + try: + eventer = salt.utils.event.get_event('minion', opts=__opts__) + res = __salt__['event.fire']({'func': 'list_available'}, 'manage_beacons') + if res: + event_ret = eventer.get_event(tag='/salt/minion/minion_beacons_list_available_complete', wait=30) + if event_ret and event_ret['complete']: + beacons = event_ret['beacons'] + except KeyError: + # Effectively a no-op, since we can't really return without an event system + ret = {} + ret['result'] = False + ret['comment'] = 'Event module not available. Beacon add failed.' + return ret + + if beacons: + if return_yaml: + tmp = {'beacons': beacons} + yaml_out = yaml.safe_dump(tmp, default_flow_style=False) + return yaml_out + else: + return beacons + else: + return {'beacons': {}} + + def add(name, beacon_data, **kwargs): ''' Add a beacon on the minion @@ -81,7 +123,7 @@ def add(name, beacon_data, **kwargs): .. code-block:: bash - salt '*' beacons.add ps "{'salt-master': 'stopped', 'apache2': 'stopped'}" + salt '*' beacons.add ps "[{'salt-master': 'stopped', 'apache2': 'stopped'}]" ''' ret = {'comment': 'Failed to add beacon {0}.'.format(name), @@ -95,34 +137,32 @@ def add(name, beacon_data, **kwargs): ret['result'] = True ret['comment'] = 'Beacon: {0} would be added.'.format(name) else: - # Attempt to load the beacon module so we have access to the validate function - try: - beacon_module = __import__('salt.beacons.' + name, fromlist=['validate']) - log.debug('Successfully imported beacon.') - except ImportError: - ret['comment'] = 'Beacon {0} does not exist'.format(name) - return ret - - # Attempt to validate - if hasattr(beacon_module, 'validate'): - _beacon_data = beacon_data - if 'enabled' in _beacon_data: - del _beacon_data['enabled'] - valid, vcomment = beacon_module.validate(_beacon_data) - else: - log.info('Beacon {0} does not have a validate' - ' function, skipping validation.'.format(name)) - valid = True - - if not valid: - ret['result'] = False - ret['comment'] = ('Beacon {0} configuration invalid, ' - 'not adding.\n{1}'.format(name, vcomment)) - return ret - try: + # Attempt to load the beacon module so we have access to the validate function eventer = salt.utils.event.get_event('minion', opts=__opts__) - res = __salt__['event.fire']({'name': name, 'beacon_data': beacon_data, 'func': 'add'}, 'manage_beacons') + res = __salt__['event.fire']({'name': name, + 'beacon_data': beacon_data, + 'func': 'validate_beacon'}, + 'manage_beacons') + if res: + event_ret = eventer.get_event(tag='/salt/minion/minion_beacon_validation_complete', wait=30) + valid = event_ret['valid'] + vcomment = event_ret['vcomment'] + + if not valid: + ret['result'] = False + ret['comment'] = ('Beacon {0} configuration invalid, ' + 'not adding.\n{1}'.format(name, vcomment)) + return ret + + except KeyError: + # Effectively a no-op, since we can't really return without an event system + ret['comment'] = 'Event module not available. Beacon add failed.' + + try: + res = __salt__['event.fire']({'name': name, + 'beacon_data': beacon_data, + 'func': 'add'}, 'manage_beacons') if res: event_ret = eventer.get_event(tag='/salt/minion/minion_beacon_add_complete', wait=30) if event_ret and event_ret['complete']: @@ -149,7 +189,7 @@ def modify(name, beacon_data, **kwargs): .. code-block:: bash - salt '*' beacons.modify ps "{'salt-master': 'stopped', 'apache2': 'stopped'}" + salt '*' beacons.modify ps "[{'salt-master': 'stopped', 'apache2': 'stopped'}]" ''' ret = {'comment': '', @@ -164,29 +204,32 @@ def modify(name, beacon_data, **kwargs): ret['result'] = True ret['comment'] = 'Beacon: {0} would be added.'.format(name) else: - # Attempt to load the beacon module so we have access to the validate function try: - beacon_module = __import__('salt.beacons.' + name, fromlist=['validate']) - log.debug('Successfully imported beacon.') - except ImportError: - ret['comment'] = 'Beacon {0} does not exist'.format(name) - return ret + # Attempt to load the beacon module so we have access to the validate function + eventer = salt.utils.event.get_event('minion', opts=__opts__) + res = __salt__['event.fire']({'name': name, + 'beacon_data': beacon_data, + 'func': 'validate_beacon'}, + 'manage_beacons') + if res: + event_ret = eventer.get_event(tag='/salt/minion/minion_beacon_validation_complete', wait=30) + valid = event_ret['valid'] + vcomment = event_ret['vcomment'] - # Attempt to validate - if hasattr(beacon_module, 'validate'): - _beacon_data = beacon_data - if 'enabled' in _beacon_data: - del _beacon_data['enabled'] - valid, vcomment = beacon_module.validate(_beacon_data) - else: - log.info('Beacon {0} does not have a validate' - ' function, skipping validation.'.format(name)) - valid = True + if not valid: + ret['result'] = False + ret['comment'] = ('Beacon {0} configuration invalid, ' + 'not adding.\n{1}'.format(name, vcomment)) + return ret + + except KeyError: + # Effectively a no-op, since we can't really return without an event system + ret['comment'] = 'Event module not available. Beacon modify failed.' if not valid: ret['result'] = False ret['comment'] = ('Beacon {0} configuration invalid, ' - 'not adding.\n{1}'.format(name, vcomment)) + 'not modifying.\n{1}'.format(name, vcomment)) return ret _current = current_beacons[name] @@ -196,10 +239,14 @@ def modify(name, beacon_data, **kwargs): ret['comment'] = 'Job {0} in correct state'.format(name) return ret - _current_lines = ['{0}:{1}\n'.format(key, value) - for (key, value) in sorted(_current.items())] - _new_lines = ['{0}:{1}\n'.format(key, value) - for (key, value) in sorted(_new.items())] + _current_lines = [] + for _item in _current: + _current_lines.extend(['{0}:{1}\n'.format(key, value) + for (key, value) in six.iteritems(_item)]) + _new_lines = [] + for _item in _new: + _new_lines.extend(['{0}:{1}\n'.format(key, value) + for (key, value) in six.iteritems(_item)]) _diff = difflib.unified_diff(_current_lines, _new_lines) ret['changes'] = {} @@ -252,6 +299,7 @@ def delete(name, **kwargs): if res: event_ret = eventer.get_event(tag='/salt/minion/minion_beacon_delete_complete', wait=30) if event_ret and event_ret['complete']: + log.debug('== event_ret {} =='.format(event_ret)) beacons = event_ret['beacons'] if name not in beacons: ret['result'] = True @@ -291,7 +339,7 @@ def save(): yaml_out = '' try: - with salt.utils.fopen(sfn, 'w+') as fp_: + with salt.utils.files.fopen(sfn, 'w+') as fp_: fp_.write(yaml_out) ret['comment'] = 'Beacons saved to {0}.'.format(sfn) except (IOError, OSError): diff --git a/salt/modules/bigip.py b/salt/modules/bigip.py index 8715e2ccd9b..bea9d55f311 100644 --- a/salt/modules/bigip.py +++ b/salt/modules/bigip.py @@ -19,7 +19,7 @@ except ImportError: HAS_LIBS = False # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six # Import salt libs import salt.utils @@ -970,7 +970,7 @@ def replace_pool_members(hostname, username, password, name, members): #specify members if provided if members is not None: - if isinstance(members, str): + if isinstance(members, six.string_types): members = members.split(',') pool_members = [] @@ -1164,7 +1164,7 @@ def delete_pool_member(hostname, username, password, name, member): CLI Example:: - salt '*' bigip.delete_node bigip admin admin my-pool 10.2.2.2:80 + salt '*' bigip.delete_pool_member bigip admin admin my-pool 10.2.2.2:80 ''' #build session @@ -1473,7 +1473,7 @@ def create_virtual(hostname, username, password, name, destination, payload['vlans'] = 'none' elif vlans == 'default': payload['vlans'] = 'default' - elif isinstance(vlans, str) and (vlans.startswith('enabled') or vlans.startswith('disabled')): + elif isinstance(vlans, six.string_types) and (vlans.startswith('enabled') or vlans.startswith('disabled')): try: vlans_setting = vlans.split(':')[0] payload['vlans'] = vlans.split(':')[1].split(',') diff --git a/salt/modules/boto_apigateway.py b/salt/modules/boto_apigateway.py index 701a217ee2d..6b42350b5a4 100644 --- a/salt/modules/boto_apigateway.py +++ b/salt/modules/boto_apigateway.py @@ -82,7 +82,7 @@ import json import datetime # Import Salt libs -import salt.ext.six as six +from salt.ext import six import salt.utils.boto3 import salt.utils.compat from salt.utils.versions import LooseVersion as _LooseVersion diff --git a/salt/modules/boto_asg.py b/salt/modules/boto_asg.py index b183a0b2437..294c00b8f91 100644 --- a/salt/modules/boto_asg.py +++ b/salt/modules/boto_asg.py @@ -58,7 +58,7 @@ DATE_FORMAT = "%Y-%m-%dT%H:%M:%SZ" # Import third party libs import yaml -import salt.ext.six as six +from salt.ext import six try: import boto import boto.ec2 diff --git a/salt/modules/boto_cloudfront.py b/salt/modules/boto_cloudfront.py new file mode 100644 index 00000000000..aa932884bf3 --- /dev/null +++ b/salt/modules/boto_cloudfront.py @@ -0,0 +1,462 @@ +# -*- coding: utf-8 -*- +''' +Connection module for Amazon CloudFront + +.. versionadded:: Oxygen + +:depends: boto3 + +:configuration: This module accepts explicit AWS credentials but can also + utilize IAM roles assigned to the instance through Instance Profiles or + it can read them from the ~/.aws/credentials file or from these + environment variables: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY. + Dynamic credentials are then automatically obtained from AWS API and no + further configuration is necessary. More information available at: + + .. code-block:: text + + http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ + iam-roles-for-amazon-ec2.html + + http://boto3.readthedocs.io/en/latest/guide/ + configuration.html#guide-configuration + + If IAM roles are not used you need to specify them either in a pillar or + in the minion's config file: + + .. code-block:: yaml + + cloudfront.keyid: GKTADJGHEIQSXMKKRBJ08H + cloudfront.key: askdjghsdfjkghWupUjasdflkdfklgjsdfjajkghs + + A region may also be specified in the configuration: + + .. code-block:: yaml + + cloudfront.region: us-east-1 + + If a region is not specified, the default is us-east-1. + + It's also possible to specify key, keyid and region via a profile, either + as a passed in dict, or as a string to pull from pillars or minion config: + + .. code-block:: yaml + + myprofile: + keyid: GKTADJGHEIQSXMKKRBJ08H + key: askdjghsdfjkghWupUjasdflkdfklgjsdfjajkghs + region: us-east-1 +''' +# keep lint from choking on _get_conn and _cache_id +# pylint: disable=E0602 + +# Import Python libs +from __future__ import absolute_import +import logging + +# Import Salt libs +import salt.ext.six as six +from salt.utils.odict import OrderedDict + +import yaml + +# Import third party libs +try: + # pylint: disable=unused-import + import boto3 + import botocore + # pylint: enable=unused-import + logging.getLogger('boto3').setLevel(logging.CRITICAL) + HAS_BOTO = True +except ImportError: + HAS_BOTO = False + +log = logging.getLogger(__name__) + + +def __virtual__(): + ''' + Only load if boto3 libraries exist. + ''' + if not HAS_BOTO: + msg = 'The boto_cloudfront module could not be loaded: {}.' + return (False, msg.format('boto3 libraries not found')) + __utils__['boto3.assign_funcs'](__name__, 'cloudfront') + return True + + +def _list_distributions( + conn, + name=None, + region=None, + key=None, + keyid=None, + profile=None, +): + ''' + Private function that returns an iterator over all CloudFront distributions. + The caller is responsible for all boto-related error handling. + + name + (Optional) Only yield the distribution with the given name + ''' + for dl_ in conn.get_paginator('list_distributions').paginate(): + distribution_list = dl_['DistributionList'] + if 'Items' not in distribution_list: + # If there are no items, AWS omits the `Items` key for some reason + continue + for partial_dist in distribution_list['Items']: + tags = conn.list_tags_for_resource(Resource=partial_dist['ARN']) + tags = dict( + (kv['Key'], kv['Value']) for kv in tags['Tags']['Items'] + ) + + id_ = partial_dist['Id'] + if 'Name' not in tags: + log.warning( + 'CloudFront distribution {0} has no Name tag.'.format(id_), + ) + continue + distribution_name = tags.pop('Name', None) + if name is not None and distribution_name != name: + continue + + # NOTE: list_distributions() returns a DistributionList, + # which nominally contains a list of Distribution objects. + # However, they are mangled in that they are missing values + # (`Logging`, `ActiveTrustedSigners`, and `ETag` keys) + # and moreover flatten the normally nested DistributionConfig + # attributes to the top level. + # Hence, we must call get_distribution() to get the full object, + # and we cache these objects to help lessen API calls. + distribution = _cache_id( + 'cloudfront', + sub_resource=distribution_name, + region=region, + key=key, + keyid=keyid, + profile=profile, + ) + if distribution: + yield (distribution_name, distribution) + continue + + dist_with_etag = conn.get_distribution(Id=id_) + distribution = { + 'distribution': dist_with_etag['Distribution'], + 'etag': dist_with_etag['ETag'], + 'tags': tags, + } + _cache_id( + 'cloudfront', + sub_resource=distribution_name, + resource_id=distribution, + region=region, + key=key, + keyid=keyid, + profile=profile, + ) + yield (distribution_name, distribution) + + +def get_distribution(name, region=None, key=None, keyid=None, profile=None): + ''' + Get information about a CloudFront distribution (configuration, tags) with a given name. + + name + Name of the CloudFront distribution + + region + Region to connect to + + key + Secret key to use + + keyid + Access key to use + + profile + A dict with region, key, and keyid, + or a pillar key (string) that contains such a dict. + + CLI Example: + + .. code-block:: bash + + salt myminion boto_cloudfront.get_distribution name=mydistribution profile=awsprofile + + ''' + distribution = _cache_id( + 'cloudfront', + sub_resource=name, + region=region, + key=key, + keyid=keyid, + profile=profile, + ) + if distribution: + return {'result': distribution} + + conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) + try: + for _, dist in _list_distributions( + conn, + name=name, + region=region, + key=key, + keyid=keyid, + profile=profile, + ): + # _list_distributions should only return the one distribution + # that we want (with the given name). + # In case of multiple distributions with the same name tag, + # our use of caching means list_distributions will just + # return the first one over and over again, + # so only the first result is useful. + if distribution is not None: + msg = 'More than one distribution found with name {0}' + return {'error': msg.format(name)} + distribution = dist + except botocore.exceptions.ClientError as err: + return {'error': __utils__['boto3.get_error'](err)} + if not distribution: + return {'result': None} + + _cache_id( + 'cloudfront', + sub_resource=name, + resource_id=distribution, + region=region, + key=key, + keyid=keyid, + profile=profile, + ) + return {'result': distribution} + + +def export_distributions(region=None, key=None, keyid=None, profile=None): + ''' + Get details of all CloudFront distributions. + Produces results that can be used to create an SLS file. + + CLI Example: + + .. code-block:: bash + + salt-call boto_cloudfront.export_distributions --out=txt |\ + sed "s/local: //" > cloudfront_distributions.sls + + ''' + results = OrderedDict() + conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) + try: + for name, distribution in _list_distributions( + conn, + region=region, + key=key, + keyid=keyid, + profile=profile, + ): + config = distribution['distribution']['DistributionConfig'] + tags = distribution['tags'] + + distribution_sls_data = [ + {'name': name}, + {'config': config}, + {'tags': tags}, + ] + results['Manage CloudFront distribution {0}'.format(name)] = { + 'boto_cloudfront.present': distribution_sls_data, + } + except botocore.exceptions.ClientError as err: + # Raise an exception, as this is meant to be user-invoked at the CLI + # as opposed to being called from execution or state modules + raise err + + dumper = __utils__['yamldumper.get_dumper']('IndentedSafeOrderedDumper') + return yaml.dump( + results, + default_flow_style=False, + Dumper=dumper, + ) + + +def create_distribution( + name, + config, + tags=None, + region=None, + key=None, + keyid=None, + profile=None, +): + ''' + Create a CloudFront distribution with the given name, config, and (optionally) tags. + + name + Name for the CloudFront distribution + + config + Configuration for the distribution + + tags + Tags to associate with the distribution + + region + Region to connect to + + key + Secret key to use + + keyid + Access key to use + + profile + A dict with region, key, and keyid, + or a pillar key (string) that contains such a dict. + + CLI Example: + + .. code-block:: bash + + salt myminion boto_cloudfront.create_distribution name=mydistribution profile=awsprofile \ + config='{"Comment":"partial configuration","Enabled":true}' + ''' + if tags is None: + tags = {} + if 'Name' in tags: + # Be lenient and silently accept if names match, else error + if tags['Name'] != name: + return {'error': 'Must not pass `Name` in `tags` but as `name`'} + tags['Name'] = name + tags = { + 'Items': [{'Key': k, 'Value': v} for k, v in six.iteritems(tags)] + } + + conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) + try: + conn.create_distribution_with_tags( + DistributionConfigWithTags={ + 'DistributionConfig': config, + 'Tags': tags, + }, + ) + _cache_id( + 'cloudfront', + sub_resource=name, + invalidate=True, + region=region, + key=key, + keyid=keyid, + profile=profile, + ) + except botocore.exceptions.ClientError as err: + return {'error': __utils__['boto3.get_error'](err)} + + return {'result': True} + + +def update_distribution( + name, + config, + tags=None, + region=None, + key=None, + keyid=None, + profile=None, +): + ''' + Update the config (and optionally tags) for the CloudFront distribution with the given name. + + name + Name of the CloudFront distribution + + config + Configuration for the distribution + + tags + Tags to associate with the distribution + + region + Region to connect to + + key + Secret key to use + + keyid + Access key to use + + profile + A dict with region, key, and keyid, + or a pillar key (string) that contains such a dict. + + CLI Example: + + .. code-block:: bash + + salt myminion boto_cloudfront.update_distribution name=mydistribution profile=awsprofile \ + config='{"Comment":"partial configuration","Enabled":true}' + ''' + distribution_ret = get_distribution( + name, + region=region, + key=key, + keyid=keyid, + profile=profile + ) + if 'error' in distribution_result: + return distribution_result + dist_with_tags = distribution_result['result'] + + current_distribution = dist_with_tags['distribution'] + current_config = current_distribution['DistributionConfig'] + current_tags = dist_with_tags['tags'] + etag = dist_with_tags['etag'] + + config_diff = __utils__['dictdiffer.deep_diff'](current_config, config) + if tags: + tags_diff = __utils__['dictdiffer.deep_diff'](current_tags, tags) + + conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) + try: + if 'old' in config_diff or 'new' in config_diff: + conn.update_distribution( + DistributionConfig=config, + Id=current_distribution['Id'], + IfMatch=etag, + ) + if tags: + arn = current_distribution['ARN'] + if 'new' in tags_diff: + tags_to_add = { + 'Items': [ + {'Key': k, 'Value': v} + for k, v in six.iteritems(tags_diff['new']) + ], + } + conn.tag_resource( + Resource=arn, + Tags=tags_to_add, + ) + if 'old' in tags_diff: + tags_to_remove = { + 'Items': list(tags_diff['old'].keys()), + } + conn.untag_resource( + Resource=arn, + TagKeys=tags_to_remove, + ) + except botocore.exceptions.ClientError as err: + return {'error': __utils__['boto3.get_error'](err)} + finally: + _cache_id( + 'cloudfront', + sub_resource=name, + invalidate=True, + region=region, + key=key, + keyid=keyid, + profile=profile, + ) + + return {'result': True} diff --git a/salt/modules/boto_cloudtrail.py b/salt/modules/boto_cloudtrail.py index 0140deabfcf..99a1191ec8b 100644 --- a/salt/modules/boto_cloudtrail.py +++ b/salt/modules/boto_cloudtrail.py @@ -54,7 +54,7 @@ from __future__ import absolute_import import logging # Import Salt libs -import salt.ext.six as six +from salt.ext import six import salt.utils.boto3 import salt.utils.compat import salt.utils diff --git a/salt/modules/boto_dynamodb.py b/salt/modules/boto_dynamodb.py index a7b23df5a34..ccbc90449cc 100644 --- a/salt/modules/boto_dynamodb.py +++ b/salt/modules/boto_dynamodb.py @@ -53,7 +53,7 @@ logger = logging.getLogger(__name__) logging.getLogger('boto').setLevel(logging.INFO) # Import third party libs -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import range # pylint: disable=import-error,redefined-builtin from salt.exceptions import SaltInvocationError try: diff --git a/salt/modules/boto_ec2.py b/salt/modules/boto_ec2.py index 975b7401214..cd77b8f1290 100644 --- a/salt/modules/boto_ec2.py +++ b/salt/modules/boto_ec2.py @@ -54,7 +54,7 @@ import json # Import Salt libs import salt.utils import salt.utils.compat -import salt.ext.six as six +from salt.ext import six from salt.exceptions import SaltInvocationError, CommandExecutionError from salt.utils.versions import LooseVersion as _LooseVersion @@ -1858,7 +1858,7 @@ def get_all_tags(filters=None, region=None, key=None, keyid=None, profile=None): ''' Describe all tags matching the filter criteria, or all tags in the account otherwise. - .. versionadded:: Nitrogen + .. versionadded:: Oxygen filters (dict) - Additional constraints on which volumes to return. Note that valid filters vary diff --git a/salt/modules/boto_efs.py b/salt/modules/boto_efs.py index 0bb20c31d2f..63bceb57efc 100644 --- a/salt/modules/boto_efs.py +++ b/salt/modules/boto_efs.py @@ -55,7 +55,7 @@ import logging # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six try: import boto3 HAS_BOTO3 = True @@ -97,7 +97,7 @@ def _get_conn(key=None, ''' client = None if profile: - if isinstance(profile, str): + if isinstance(profile, six.string_types): if profile in __pillar__: profile = __pillar__[profile] elif profile in __opts__: @@ -135,6 +135,7 @@ def create_file_system(name, key=None, profile=None, region=None, + creation_token=None, **kwargs): ''' Creates a new, empty file system. @@ -146,6 +147,10 @@ def create_file_system(name, (string) - The PerformanceMode of the file system. Can be either generalPurpose or maxIO + creation_token + (string) - A unique name to be used as reference when creating an EFS. + This will ensure idempotency. Set to name if not specified otherwise + returns (dict) - A dict of the data for the elastic file system @@ -155,9 +160,10 @@ def create_file_system(name, salt 'my-minion' boto_efs.create_file_system efs-name generalPurpose ''' - import os - import base64 - creation_token = base64.b64encode(os.urandom(46), ['-', '_']) + + if creation_token is None: + creation_token = name + tags = {"Key": "Name", "Value": name} client = _get_conn(key=key, keyid=keyid, profile=profile, region=region) @@ -223,10 +229,23 @@ def create_mount_target(filesystemid, client = _get_conn(key=key, keyid=keyid, profile=profile, region=region) - return client.create_mount_point(FileSystemId=filesystemid, - SubnetId=subnetid, - IpAddress=ipaddress, - SecurityGroups=securitygroups) + if ipaddress is None and securitygroups is None: + return client.create_mount_target(FileSystemId=filesystemid, + SubnetId=subnetid) + + if ipaddress is None: + return client.create_mount_target(FileSystemId=filesystemid, + SubnetId=subnetid, + SecurityGroups=securitygroups) + if securitygroups is None: + return client.create_mount_target(FileSystemId=filesystemid, + SubnetId=subnetid, + IpAddress=ipaddress) + + return client.create_mount_target(FileSystemId=filesystemid, + SubnetId=subnetid, + IpAddress=ipaddress, + SecurityGroups=securitygroups) def create_tags(filesystemid, @@ -359,6 +378,7 @@ def get_file_systems(filesystemid=None, key=None, profile=None, region=None, + creation_token=None, **kwargs): ''' Get all EFS properties or a specific instance property @@ -367,6 +387,12 @@ def get_file_systems(filesystemid=None, filesystemid (string) - ID of the file system to retrieve properties + creation_token + (string) - A unique token that identifies an EFS. + If fileysystem created via create_file_system this would + either be explictitly passed in or set to name. + You can limit your search with this. + returns (list[dict]) - list of all elastic file system properties @@ -380,9 +406,16 @@ def get_file_systems(filesystemid=None, result = None client = _get_conn(key=key, keyid=keyid, profile=profile, region=region) - if filesystemid: + if filesystemid and creation_token: + response = client.describe_file_systems(FileSystemId=filesystemid, + CreationToken=creation_token) + result = response["FileSystems"] + elif filesystemid: response = client.describe_file_systems(FileSystemId=filesystemid) result = response["FileSystems"] + elif creation_token: + response = client.describe_file_systems(CreationToken=creation_token) + result = response["FileSystems"] else: response = client.describe_file_systems() diff --git a/salt/modules/boto_elasticache.py b/salt/modules/boto_elasticache.py index 1cafff7d328..21de556b4c1 100644 --- a/salt/modules/boto_elasticache.py +++ b/salt/modules/boto_elasticache.py @@ -51,7 +51,7 @@ import logging import time # Import Salt libs -import salt.ext.six as six +from salt.ext import six from salt.exceptions import SaltInvocationError import salt.utils.odict as odict diff --git a/salt/modules/boto_elasticsearch_domain.py b/salt/modules/boto_elasticsearch_domain.py index fce241f1c3b..d4a156db5eb 100644 --- a/salt/modules/boto_elasticsearch_domain.py +++ b/salt/modules/boto_elasticsearch_domain.py @@ -80,7 +80,7 @@ import logging import json # Import Salt libs -import salt.ext.six as six +from salt.ext import six import salt.utils.boto3 import salt.utils.compat import salt.utils diff --git a/salt/modules/boto_elb.py b/salt/modules/boto_elb.py index eed2759ca03..9b300d368f0 100644 --- a/salt/modules/boto_elb.py +++ b/salt/modules/boto_elb.py @@ -49,6 +49,7 @@ from __future__ import absolute_import # Import Python libs import logging import json +import time log = logging.getLogger(__name__) @@ -57,7 +58,7 @@ import salt.utils.odict as odict from salt.utils.versions import LooseVersion as _LooseVersion # Import third party libs -import salt.ext.six as six +from salt.ext import six from salt.ext.six import string_types try: import boto @@ -160,49 +161,60 @@ def get_elb_config(name, region=None, key=None, keyid=None, profile=None): salt myminion boto_elb.exists myelb region=us-east-1 ''' conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) + wait = 60 + orig_wait = wait - try: - lb = conn.get_all_load_balancers(load_balancer_names=[name]) - lb = lb[0] - ret = {} - ret['availability_zones'] = lb.availability_zones - listeners = [] - for _listener in lb.listeners: - listener_dict = {} - listener_dict['elb_port'] = _listener.load_balancer_port - listener_dict['elb_protocol'] = _listener.protocol - listener_dict['instance_port'] = _listener.instance_port - listener_dict['instance_protocol'] = _listener.instance_protocol - listener_dict['policies'] = _listener.policy_names - if _listener.ssl_certificate_id: - listener_dict['certificate'] = _listener.ssl_certificate_id - listeners.append(listener_dict) - ret['listeners'] = listeners - backends = [] - for _backend in lb.backends: - bs_dict = {} - bs_dict['instance_port'] = _backend.instance_port - bs_dict['policies'] = [p.policy_name for p in _backend.policies] - backends.append(bs_dict) - ret['backends'] = backends - ret['subnets'] = lb.subnets - ret['security_groups'] = lb.security_groups - ret['scheme'] = lb.scheme - ret['dns_name'] = lb.dns_name - ret['tags'] = _get_all_tags(conn, name) - lb_policy_lists = [ - lb.policies.app_cookie_stickiness_policies, - lb.policies.lb_cookie_stickiness_policies, - lb.policies.other_policies - ] - policies = [] - for policy_list in lb_policy_lists: - policies += [p.policy_name for p in policy_list] - ret['policies'] = policies - return ret - except boto.exception.BotoServerError as error: - log.debug(error) - return {} + while True: + try: + lb = conn.get_all_load_balancers(load_balancer_names=[name]) + lb = lb[0] + ret = {} + ret['availability_zones'] = lb.availability_zones + listeners = [] + for _listener in lb.listeners: + listener_dict = {} + listener_dict['elb_port'] = _listener.load_balancer_port + listener_dict['elb_protocol'] = _listener.protocol + listener_dict['instance_port'] = _listener.instance_port + listener_dict['instance_protocol'] = _listener.instance_protocol + listener_dict['policies'] = _listener.policy_names + if _listener.ssl_certificate_id: + listener_dict['certificate'] = _listener.ssl_certificate_id + listeners.append(listener_dict) + ret['listeners'] = listeners + backends = [] + for _backend in lb.backends: + bs_dict = {} + bs_dict['instance_port'] = _backend.instance_port + bs_dict['policies'] = [p.policy_name for p in _backend.policies] + backends.append(bs_dict) + ret['backends'] = backends + ret['subnets'] = lb.subnets + ret['security_groups'] = lb.security_groups + ret['scheme'] = lb.scheme + ret['dns_name'] = lb.dns_name + ret['tags'] = _get_all_tags(conn, name) + lb_policy_lists = [ + lb.policies.app_cookie_stickiness_policies, + lb.policies.lb_cookie_stickiness_policies, + lb.policies.other_policies + ] + policies = [] + for policy_list in lb_policy_lists: + policies += [p.policy_name for p in policy_list] + ret['policies'] = policies + return ret + except boto.exception.BotoServerError as error: + if getattr(error, 'error_code', '') == 'Throttling': + if wait > 0: + sleep = wait if wait % 5 == wait else 5 + log.info('Throttled by AWS API, will retry in 5 seconds.') + time.sleep(sleep) + wait -= sleep + continue + log.error('API still throttling us after {0} seconds!'.format(orig_wait)) + log.error(error) + return {} def listener_dict_to_tuple(listener): @@ -706,7 +718,7 @@ def register_instances(name, instances, region=None, key=None, keyid=None, ''' # convert instances to list type, enabling consistent use of instances # variable throughout the register_instances method - if isinstance(instances, str) or isinstance(instances, six.text_type): + if isinstance(instances, six.string_types) or isinstance(instances, six.text_type): instances = [instances] conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) @@ -750,7 +762,7 @@ def deregister_instances(name, instances, region=None, key=None, keyid=None, ''' # convert instances to list type, enabling consistent use of instances # variable throughout the deregister_instances method - if isinstance(instances, str) or isinstance(instances, six.text_type): + if isinstance(instances, six.string_types) or isinstance(instances, six.text_type): instances = [instances] conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) diff --git a/salt/modules/boto_elbv2.py b/salt/modules/boto_elbv2.py index 954be55b8d5..7fd1833a8b8 100644 --- a/salt/modules/boto_elbv2.py +++ b/salt/modules/boto_elbv2.py @@ -2,7 +2,7 @@ ''' Connection module for Amazon ALB -.. versionadded:: TBD +.. versionadded:: 2017.7.0 :configuration: This module accepts explicit elb credentials but can also utilize IAM roles assigned to the instance through Instance Profiles. Dynamic @@ -47,7 +47,7 @@ log = logging.getLogger(__name__) # Import Salt libs # Import third party libs -import salt.ext.six as six +from salt.ext import six try: # pylint: disable=unused-import @@ -72,7 +72,140 @@ def __virtual__(): return True -def target_group_exists(name, region=None, key=None, keyid=None, profile=None): +def create_target_group(name, + protocol, + port, + vpc_id, + region=None, + key=None, + keyid=None, + profile=None, + health_check_protocol='HTTP', + health_check_port='traffic-port', + health_check_path='/', + health_check_interval_seconds=30, + health_check_timeout_seconds=5, + healthy_threshold_count=5, + unhealthy_threshold_count=2): + ''' + Create target group if not present. + + name + (string) - The name of the target group. + protocol + (string) - The protocol to use for routing traffic to the targets + port + (int) - The port on which the targets receive traffic. This port is used unless + you specify a port override when registering the traffic. + vpc_id + (string) - The identifier of the virtual private cloud (VPC). + health_check_protocol + (string) - The protocol the load balancer uses when performing health check on + targets. The default is the HTTP protocol. + health_check_port + (string) - The port the load balancer uses when performing health checks on + targets. The default is 'traffic-port', which indicates the port on which each + target receives traffic from the load balancer. + health_check_path + (string) - The ping path that is the destination on the targets for health + checks. The default is /. + health_check_interval_seconds + (integer) - The approximate amount of time, in seconds, between health checks + of an individual target. The default is 30 seconds. + health_check_timeout_seconds + (integer) - The amount of time, in seconds, during which no response from a + target means a failed health check. The default is 5 seconds. + healthy_threshold_count + (integer) - The number of consecutive health checks successes required before + considering an unhealthy target healthy. The default is 5. + unhealthy_threshold_count + (integer) - The number of consecutive health check failures required before + considering a target unhealthy. The default is 2. + + returns + (bool) - True on success, False on failure. + + CLI example: + .. code-block:: bash + + salt myminion boto_elbv2.create_target_group learn1give1 protocol=HTTP port=54006 vpc_id=vpc-deadbeef + ''' + + conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) + if target_group_exists(name, region, key, keyid, profile): + return True + + try: + alb = conn.create_target_group(Name=name, Protocol=protocol, Port=port, + VpcId=vpc_id, HealthCheckProtocol=health_check_protocol, + HealthCheckPort=health_check_port, + HealthCheckPath=health_check_path, + HealthCheckIntervalSeconds=health_check_interval_seconds, + HealthCheckTimeoutSeconds=health_check_timeout_seconds, + HealthyThresholdCount=healthy_threshold_count, + UnhealthyThresholdCount=unhealthy_threshold_count) + if alb: + log.info('Created ALB {0}: {1}'.format(name, + alb['TargetGroups'][0]['TargetGroupArn'])) + return True + else: + log.error('Failed to create ALB {0}'.format(name)) + return False + except ClientError as error: + log.debug(error) + log.error('Failed to create ALB {0}: {1}: {2}'.format(name, + error.response['Error']['Code'], + error.response['Error']['Message'])) + + +def delete_target_group(name, + region=None, + key=None, + keyid=None, + profile=None): + ''' + Delete target group. + + name + (string) - Target Group Name or Amazon Resource Name (ARN). + + returns + (bool) - True on success, False on failure. + + CLI example: + + .. code-block:: bash + + salt myminion boto_elbv2.delete_target_group arn:aws:elasticloadbalancing:us-west-2:644138682826:targetgroup/learn1give1-api/414788a16b5cf163 + ''' + conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) + + if not target_group_exists(name, region, key, keyid, profile): + return True + + try: + if name.startswith('arn:aws:elasticloadbalancing'): + conn.delete_target_group(TargetGroupArn=name) + log.info('Deleted target group {0}'.format(name)) + else: + tg_info = conn.describe_target_groups(Names=[name]) + if len(tg_info['TargetGroups']) != 1: + return False + arn = tg_info['TargetGroups'][0]['TargetGroupArn'] + conn.delete_target_group(TargetGroupArn=arn) + log.info('Deleted target group {0} ARN {1}'.format(name, arn)) + return True + except ClientError as error: + log.debug(error) + log.error('Failed to delete target group {0}'.format(name)) + return False + + +def target_group_exists(name, + region=None, + key=None, + keyid=None, + profile=None): ''' Check to see if an target group exists. @@ -80,23 +213,31 @@ def target_group_exists(name, region=None, key=None, keyid=None, profile=None): .. code-block:: bash - salt myminion boto_elbv2.exists arn:aws:elasticloadbalancing:us-west-2:644138682826:targetgroup/learn1give1-api/414788a16b5cf163 + salt myminion boto_elbv2.target_group_exists arn:aws:elasticloadbalancing:us-west-2:644138682826:targetgroup/learn1give1-api/414788a16b5cf163 ''' conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) try: - alb = conn.describe_target_groups(TargetGroupArns=[name]) + if name.startswith('arn:aws:elasticloadbalancing'): + alb = conn.describe_target_groups(TargetGroupArns=[name]) + else: + alb = conn.describe_target_groups(Names=[name]) if alb: return True else: log.warning('The target group does not exist in region {0}'.format(region)) return False except ClientError as error: - log.warning(error) + log.warning('target_group_exists check for {0} returned: {1}'.format(name, error)) return False -def describe_target_health(name, targets=None, region=None, key=None, keyid=None, profile=None): +def describe_target_health(name, + targets=None, + region=None, + key=None, + keyid=None, + profile=None): ''' Get the curret health check status for targets in a target group. @@ -126,8 +267,12 @@ def describe_target_health(name, targets=None, region=None, key=None, keyid=None return {} -def register_targets(name, targets, region=None, key=None, keyid=None, - profile=None): +def register_targets(name, + targets, + region=None, + key=None, + keyid=None, + profile=None): ''' Register targets to a target froup of an ALB. ``targets`` is either a instance id string or a list of instance id's. @@ -145,7 +290,7 @@ def register_targets(name, targets, region=None, key=None, keyid=None, salt myminion boto_elbv2.register_targets myelb "[instance_id,instance_id]" ''' targetsdict = [] - if isinstance(targets, str) or isinstance(targets, six.text_type): + if isinstance(targets, six.string_types) or isinstance(targets, six.text_type): targetsdict.append({"Id": targets}) else: for target in targets: @@ -156,15 +301,18 @@ def register_targets(name, targets, region=None, key=None, keyid=None, registered_targets = conn.register_targets(TargetGroupArn=name, Targets=targetsdict) if registered_targets: return True - else: - return False + return False except ClientError as error: log.warning(error) return False -def deregister_targets(name, targets, region=None, key=None, keyid=None, - profile=None): +def deregister_targets(name, + targets, + region=None, + key=None, + keyid=None, + profile=None): ''' Deregister targets to a target froup of an ALB. ``targets`` is either a instance id string or a list of instance id's. @@ -182,7 +330,7 @@ def deregister_targets(name, targets, region=None, key=None, keyid=None, salt myminion boto_elbv2.deregister_targets myelb "[instance_id,instance_id]" ''' targetsdict = [] - if isinstance(targets, str) or isinstance(targets, six.text_type): + if isinstance(targets, six.string_types) or isinstance(targets, six.text_type): targetsdict.append({"Id": targets}) else: for target in targets: @@ -193,8 +341,7 @@ def deregister_targets(name, targets, region=None, key=None, keyid=None, registered_targets = conn.deregister_targets(TargetGroupArn=name, Targets=targetsdict) if registered_targets: return True - else: - return False + return False except ClientError as error: log.warning(error) return False diff --git a/salt/modules/boto_iam.py b/salt/modules/boto_iam.py index 97eccd6616e..a4d1ab9c1eb 100644 --- a/salt/modules/boto_iam.py +++ b/salt/modules/boto_iam.py @@ -44,7 +44,7 @@ import json import yaml # Import salt libs -import salt.ext.six as six +from salt.ext import six import salt.utils.compat import salt.utils.odict as odict @@ -1955,7 +1955,7 @@ def list_policy_versions(policy_name, return ret.get('list_policy_versions_response', {}).get('list_policy_versions_result', {}).get('versions') except boto.exception.BotoServerError as e: log.debug(e) - msg = 'Failed to list {0} policy vesions.' + msg = 'Failed to list {0} policy versions.' log.error(msg.format(policy_name)) return [] diff --git a/salt/modules/boto_lambda.py b/salt/modules/boto_lambda.py index 5d1cf70a532..2b0e8bb95d3 100644 --- a/salt/modules/boto_lambda.py +++ b/salt/modules/boto_lambda.py @@ -87,9 +87,9 @@ import time import random # Import Salt libs -import salt.ext.six as six +from salt.ext import six import salt.utils.compat -import salt.utils +import salt.utils.files from salt.utils.versions import LooseVersion as _LooseVersion from salt.exceptions import SaltInvocationError from salt.ext.six.moves import range # pylint: disable=import-error @@ -201,7 +201,7 @@ def _get_role_arn(name, region=None, key=None, keyid=None, profile=None): def _filedata(infile): - with salt.utils.fopen(infile, 'rb') as f: + with salt.utils.files.fopen(infile, 'rb') as f: return f.read() diff --git a/salt/modules/boto_rds.py b/salt/modules/boto_rds.py index fbfcfecf7b0..f57b9633deb 100644 --- a/salt/modules/boto_rds.py +++ b/salt/modules/boto_rds.py @@ -63,7 +63,7 @@ from salt.utils.versions import LooseVersion as _LooseVersion log = logging.getLogger(__name__) # Import third party libs -import salt.ext.six as six +from salt.ext import six # pylint: disable=import-error try: #pylint: disable=unused-import diff --git a/salt/modules/boto_route53.py b/salt/modules/boto_route53.py index ef1e83ce3e0..23683d76eee 100644 --- a/salt/modules/boto_route53.py +++ b/salt/modules/boto_route53.py @@ -53,6 +53,7 @@ import time # Import salt libs import salt.utils.compat +import salt.utils.versions import salt.utils.odict as odict from salt.exceptions import SaltInvocationError from salt.utils.versions import LooseVersion as _LooseVersion @@ -168,11 +169,14 @@ def describe_hosted_zones(zone_id=None, domain_name=None, region=None, marker = r['ListHostedZonesResponse'].get('NextMarker', '') return ret if ret else [] except DNSServerError as e: - # if rate limit, retry: - if retries and 'Throttling' == e.code: - log.debug('Throttled by AWS API.') + if retries: + if 'Throttling' == e.code: + log.debug('Throttled by AWS API.') + elif 'PriorRequestNotComplete' == e.code: + log.debug('The request was rejected by AWS API.\ + Route 53 was still processing a prior request') time.sleep(3) - tries -= 1 + retries -= 1 continue log.error('Could not list zones: {0}'.format(e.message)) return [] @@ -235,7 +239,8 @@ def list_all_zones_by_id(region=None, key=None, keyid=None, profile=None): def zone_exists(zone, region=None, key=None, keyid=None, profile=None, - retry_on_rate_limit=True, rate_limit_retries=5): + retry_on_rate_limit=None, rate_limit_retries=None, + retry_on_errors=True, error_retries=5): ''' Check for the existence of a Route53 hosted zone. @@ -250,17 +255,33 @@ def zone_exists(zone, region=None, key=None, keyid=None, profile=None, conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) - while rate_limit_retries > 0: + if retry_on_rate_limit or rate_limit_retries is not None: + salt.utils.versions.warn_until( + 'Neon', + 'The \'retry_on_rate_limit\' and \'rate_limit_retries\' arguments ' + 'have been deprecated in favor of \'retry_on_errors\' and ' + '\'error_retries\' respectively. Their functionality will be ' + 'removed, as such, their usage is no longer required.' + ) + if retry_on_rate_limit is not None: + retry_on_errors = retry_on_rate_limit + if rate_limit_retries is not None: + error_retries = rate_limit_retries + + while error_retries > 0: try: return bool(conn.get_zone(zone)) except DNSServerError as e: - # if rate limit, retry: - if retry_on_rate_limit and 'Throttling' == e.code: - log.debug('Throttled by AWS API.') - time.sleep(2) - rate_limit_retries -= 1 - continue # the while True; try again if not out of retries + if retry_on_errors: + if 'Throttling' == e.code: + log.debug('Throttled by AWS API.') + elif 'PriorRequestNotComplete' == e.code: + log.debug('The request was rejected by AWS API.\ + Route 53 was still processing a prior request') + time.sleep(3) + error_retries -= 1 + continue raise e @@ -352,7 +373,8 @@ def _decode_name(name): def get_record(name, zone, record_type, fetch_all=False, region=None, key=None, keyid=None, profile=None, split_dns=False, private_zone=False, - identifier=None, retry_on_rate_limit=True, rate_limit_retries=5): + identifier=None, retry_on_rate_limit=None, + rate_limit_retries=None, retry_on_errors=True, error_retries=5): ''' Get a record from a zone. @@ -365,7 +387,20 @@ def get_record(name, zone, record_type, fetch_all=False, region=None, key=None, conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) - while rate_limit_retries > 0: + if retry_on_rate_limit or rate_limit_retries is not None: + salt.utils.versions.warn_until( + 'Neon', + 'The \'retry_on_rate_limit\' and \'rate_limit_retries\' arguments ' + 'have been deprecated in favor of \'retry_on_errors\' and ' + '\'error_retries\' respectively. Their functionality will be ' + 'removed, as such, their usage is no longer required.' + ) + if retry_on_rate_limit is not None: + retry_on_errors = retry_on_rate_limit + if rate_limit_retries is not None: + error_retries = rate_limit_retries + + while error_retries > 0: try: if split_dns: _zone = _get_split_zone(zone, conn, private_zone) @@ -385,12 +420,15 @@ def get_record(name, zone, record_type, fetch_all=False, region=None, key=None, break # the while True except DNSServerError as e: - # if rate limit, retry: - if retry_on_rate_limit and 'Throttling' == e.code: - log.debug('Throttled by AWS API.') - time.sleep(2) - rate_limit_retries -= 1 - continue # the while True; try again if not out of retries + if retry_on_errors: + if 'Throttling' == e.code: + log.debug('Throttled by AWS API.') + elif 'PriorRequestNotComplete' == e.code: + log.debug('The request was rejected by AWS API.\ + Route 53 was still processing a prior request') + time.sleep(3) + error_retries -= 1 + continue raise e if _record: @@ -416,7 +454,8 @@ def _munge_value(value, _type): def add_record(name, value, zone, record_type, identifier=None, ttl=None, region=None, key=None, keyid=None, profile=None, wait_for_sync=True, split_dns=False, private_zone=False, - retry_on_rate_limit=True, rate_limit_retries=5): + retry_on_rate_limit=None, rate_limit_retries=None, + retry_on_errors=True, error_retries=5): ''' Add a record to a zone. @@ -429,7 +468,20 @@ def add_record(name, value, zone, record_type, identifier=None, ttl=None, conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) - while rate_limit_retries > 0: + if retry_on_rate_limit or rate_limit_retries is not None: + salt.utils.versions.warn_until( + 'Neon', + 'The \'retry_on_rate_limit\' and \'rate_limit_retries\' arguments ' + 'have been deprecated in favor of \'retry_on_errors\' and ' + '\'error_retries\' respectively. Their functionality will be ' + 'removed, as such, their usage is no longer required.' + ) + if retry_on_rate_limit is not None: + retry_on_errors = retry_on_rate_limit + if rate_limit_retries is not None: + error_retries = rate_limit_retries + + while error_retries > 0: try: if split_dns: _zone = _get_split_zone(zone, conn, private_zone) @@ -443,16 +495,19 @@ def add_record(name, value, zone, record_type, identifier=None, ttl=None, break except DNSServerError as e: - # if rate limit, retry: - if retry_on_rate_limit and 'Throttling' == e.code: - log.debug('Throttled by AWS API.') - time.sleep(2) - rate_limit_retries -= 1 - continue # the while True; try again if not out of retries + if retry_on_errors: + if 'Throttling' == e.code: + log.debug('Throttled by AWS API.') + elif 'PriorRequestNotComplete' == e.code: + log.debug('The request was rejected by AWS API.\ + Route 53 was still processing a prior request') + time.sleep(3) + error_retries -= 1 + continue raise e _value = _munge_value(value, _type) - while rate_limit_retries > 0: + while error_retries > 0: try: # add_record requires a ttl value, annoyingly. if ttl is None: @@ -461,19 +516,23 @@ def add_record(name, value, zone, record_type, identifier=None, ttl=None, return _wait_for_sync(status.id, conn, wait_for_sync) except DNSServerError as e: - # if rate limit, retry: - if retry_on_rate_limit and 'Throttling' == e.code: - log.debug('Throttled by AWS API.') - time.sleep(2) - rate_limit_retries -= 1 - continue # the while True; try again if not out of retries + if retry_on_errors: + if 'Throttling' == e.code: + log.debug('Throttled by AWS API.') + elif 'PriorRequestNotComplete' == e.code: + log.debug('The request was rejected by AWS API.\ + Route 53 was still processing a prior request') + time.sleep(3) + error_retries -= 1 + continue raise e def update_record(name, value, zone, record_type, identifier=None, ttl=None, region=None, key=None, keyid=None, profile=None, wait_for_sync=True, split_dns=False, private_zone=False, - retry_on_rate_limit=True, rate_limit_retries=5): + retry_on_rate_limit=None, rate_limit_retries=None, + retry_on_errors=True, error_retries=5): ''' Modify a record in a zone. @@ -496,8 +555,21 @@ def update_record(name, value, zone, record_type, identifier=None, ttl=None, return False _type = record_type.upper() + if retry_on_rate_limit or rate_limit_retries is not None: + salt.utils.versions.warn_until( + 'Neon', + 'The \'retry_on_rate_limit\' and \'rate_limit_retries\' arguments ' + 'have been deprecated in favor of \'retry_on_errors\' and ' + '\'error_retries\' respectively. Their functionality will be ' + 'removed, as such, their usage is no longer required.' + ) + if retry_on_rate_limit is not None: + retry_on_errors = retry_on_rate_limit + if rate_limit_retries is not None: + error_retries = rate_limit_retries + _value = _munge_value(value, _type) - while rate_limit_retries > 0: + while error_retries > 0: try: old_record = _zone.find_records(name, _type, identifier=identifier) if not old_record: @@ -506,19 +578,23 @@ def update_record(name, value, zone, record_type, identifier=None, ttl=None, return _wait_for_sync(status.id, conn, wait_for_sync) except DNSServerError as e: - # if rate limit, retry: - if retry_on_rate_limit and 'Throttling' == e.code: - log.debug('Throttled by AWS API.') - time.sleep(2) - rate_limit_retries -= 1 - continue # the while True; try again if not out of retries + if retry_on_errors: + if 'Throttling' == e.code: + log.debug('Throttled by AWS API.') + elif 'PriorRequestNotComplete' == e.code: + log.debug('The request was rejected by AWS API.\ + Route 53 was still processing a prior request') + time.sleep(3) + error_retries -= 1 + continue raise e def delete_record(name, zone, record_type, identifier=None, all_records=False, region=None, key=None, keyid=None, profile=None, wait_for_sync=True, split_dns=False, private_zone=False, - retry_on_rate_limit=True, rate_limit_retries=5): + retry_on_rate_limit=None, rate_limit_retries=None, + retry_on_errors=True, error_retries=5): ''' Modify a record in a zone. @@ -541,7 +617,20 @@ def delete_record(name, zone, record_type, identifier=None, all_records=False, return False _type = record_type.upper() - while rate_limit_retries > 0: + if retry_on_rate_limit or rate_limit_retries is not None: + salt.utils.versions.warn_until( + 'Neon', + 'The \'retry_on_rate_limit\' and \'rate_limit_retries\' arguments ' + 'have been deprecated in favor of \'retry_on_errors\' and ' + '\'error_retries\' respectively. Their functionality will be ' + 'removed, as such, their usage is no longer required.' + ) + if retry_on_rate_limit is not None: + retry_on_errors = retry_on_rate_limit + if rate_limit_retries is not None: + error_retries = rate_limit_retries + + while error_retries > 0: try: old_record = _zone.find_records(name, _type, all=all_records, identifier=identifier) if not old_record: @@ -550,12 +639,15 @@ def delete_record(name, zone, record_type, identifier=None, all_records=False, return _wait_for_sync(status.id, conn, wait_for_sync) except DNSServerError as e: - # if rate limit, retry: - if retry_on_rate_limit and 'Throttling' == e.code: - log.debug('Throttled by AWS API.') - time.sleep(2) - rate_limit_retries -= 1 - continue # the while True; try again if not out of retries + if retry_on_errors: + if 'Throttling' == e.code: + log.debug('Throttled by AWS API.') + elif 'PriorRequestNotComplete' == e.code: + log.debug('The request was rejected by AWS API.\ + Route 53 was still processing a prior request') + time.sleep(3) + error_retries -= 1 + continue raise e @@ -693,8 +785,12 @@ def create_hosted_zone(domain_name, caller_ref=None, comment='', 'CreateHostedZoneResponse') else {} return r.get('parent', {}).get('CreateHostedZoneResponse') except DNSServerError as e: - if retries and 'Throttling' == e.code: - log.debug('Throttled by AWS API.') + if retries: + if 'Throttling' == e.code: + log.debug('Throttled by AWS API.') + elif 'PriorRequestNotComplete' == e.code: + log.debug('The request was rejected by AWS API.\ + Route 53 was still processing a prior request') time.sleep(3) retries -= 1 continue diff --git a/salt/modules/boto_secgroup.py b/salt/modules/boto_secgroup.py index 8cfba94e626..ac8c915f045 100644 --- a/salt/modules/boto_secgroup.py +++ b/salt/modules/boto_secgroup.py @@ -52,7 +52,7 @@ import logging log = logging.getLogger(__name__) # Import third party libs -import salt.ext.six as six +from salt.ext import six try: # pylint: disable=unused-import import boto @@ -270,9 +270,9 @@ def get_all_security_groups(groupnames=None, group_ids=None, filters=None, ''' conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) - if isinstance(groupnames, str): + if isinstance(groupnames, six.string_types): groupnames = [groupnames] - if isinstance(group_ids, str): + if isinstance(group_ids, six.string_types): groupnames = [group_ids] interesting = ['description', 'id', 'instances', 'name', 'owner_id', diff --git a/salt/modules/boto_sqs.py b/salt/modules/boto_sqs.py index bc02e66d68d..034ba11802b 100644 --- a/salt/modules/boto_sqs.py +++ b/salt/modules/boto_sqs.py @@ -7,7 +7,7 @@ Connection module for Amazon SQS :configuration: This module accepts explicit sqs credentials but can also utilize IAM roles assigned to the instance through Instance Profiles. Dynamic credentials are then automatically obtained from AWS API and no further - configuration is necessary. More Information available at: + configuration is necessary. More information available at: .. code-block:: text @@ -39,44 +39,68 @@ Connection module for Amazon SQS key: askdjghsdfjkghWupUjasdflkdfklgjsdfjajkghs region: us-east-1 -:depends: boto +:depends: boto3 ''' # keep lint from choking on _get_conn and _cache_id -#pylint: disable=E0602 +# pylint: disable=E0602 from __future__ import absolute_import # Import Python libs import logging import json -import salt.ext.six as six + +# Import 3rd-party libs +from salt.ext import six +from salt.ext.six.moves.urllib.parse import urlparse as _urlparse # pylint: disable=import-error,no-name-in-module log = logging.getLogger(__name__) +__func_alias__ = { + 'list_': 'list', +} + # Import third party libs try: # pylint: disable=unused-import - import boto - import boto.sqs + import boto3 + import botocore # pylint: enable=unused-import - logging.getLogger('boto').setLevel(logging.CRITICAL) - HAS_BOTO = True + logging.getLogger('boto3').setLevel(logging.CRITICAL) + HAS_BOTO3 = True except ImportError: - HAS_BOTO = False - -from salt.ext.six import string_types + HAS_BOTO3 = False def __virtual__(): ''' - Only load if boto libraries exist. + Only load if boto3 libraries exist. ''' - if not HAS_BOTO: - return (False, 'The boto_sqs module could not be loaded: boto libraries not found') - __utils__['boto.assign_funcs'](__name__, 'sqs', pack=__salt__) + if not HAS_BOTO3: + return (False, 'The boto_sqs module could not be loaded: boto3 libraries not found') + __utils__['boto3.assign_funcs'](__name__, 'sqs') return True +def _preprocess_attributes(attributes): + ''' + Pre-process incoming queue attributes before setting them + ''' + if isinstance(attributes, six.string_types): + attributes = json.loads(attributes) + + def stringified(val): + # Some attributes take full json policy documents, but they take them + # as json strings. Convert the value back into a json string. + if isinstance(val, dict): + return json.dumps(val) + return val + + return dict( + (attr, stringified(val)) for attr, val in six.iteritems(attributes) + ) + + def exists(name, region=None, key=None, keyid=None, profile=None): ''' Check to see if a queue exists. @@ -89,13 +113,24 @@ def exists(name, region=None, key=None, keyid=None, profile=None): ''' conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) - if conn.get_queue(name): - return True - else: - return False + try: + conn.get_queue_url(QueueName=name) + except botocore.exceptions.ClientError as e: + missing_code = 'AWS.SimpleQueueService.NonExistentQueue' + if e.response.get('Error', {}).get('Code') == missing_code: + return {'result': False} + return {'error': __utils__['boto3.get_error'](e)} + return {'result': True} -def create(name, region=None, key=None, keyid=None, profile=None): +def create( + name, + attributes=None, + region=None, + key=None, + keyid=None, + profile=None, +): ''' Create an SQS queue. @@ -107,15 +142,15 @@ def create(name, region=None, key=None, keyid=None, profile=None): ''' conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) - if not conn.get_queue(name): - try: - conn.create_queue(name) - except boto.exception.SQSError: - msg = 'Failed to create queue {0}'.format(name) - log.error(msg) - return False - log.info('Created queue {0}'.format(name)) - return True + if attributes is None: + attributes = {} + attributes = _preprocess_attributes(attributes) + + try: + conn.create_queue(QueueName=name, Attributes=attributes) + except botocore.exceptions.ClientError as e: + return {'error': __utils__['boto3.get_error'](e)} + return {'result': True} def delete(name, region=None, key=None, keyid=None, profile=None): @@ -130,37 +165,15 @@ def delete(name, region=None, key=None, keyid=None, profile=None): ''' conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) - queue_obj = conn.get_queue(name) - if queue_obj: - deleted_queue = conn.delete_queue(queue_obj) - if not deleted_queue: - msg = 'Failed to delete queue {0}'.format(name) - log.error(msg) - return False - return True - - -def get_all_queues(prefix=None, region=None, key=None, keyid=None, profile=None): - ''' - Return a list of Queue() objects describing all visible queues. - - .. versionadded:: 2016.11.0 - - CLI Example: - - .. code-block:: bash - - salt myminion boto_sqs.get_all_queues region=us-east-1 --output yaml - ''' - conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) try: - return conn.get_all_queues(prefix=prefix) - except boto.exception.SQSError: - log.error('Error listing queues') - return [] + url = conn.get_queue_url(QueueName=name)['QueueUrl'] + conn.delete_queue(QueueUrl=url) + except botocore.exceptions.ClientError as e: + return {'error': __utils__['boto3.get_error'](e)} + return {'result': True} -def list(prefix=None, region=None, key=None, keyid=None, profile=None): +def list_(prefix='', region=None, key=None, keyid=None, profile=None): ''' Return a list of the names of all visible queues. @@ -172,8 +185,19 @@ def list(prefix=None, region=None, key=None, keyid=None, profile=None): salt myminion boto_sqs.list region=us-east-1 ''' - return [r.name for r in get_all_queues(prefix=prefix, region=region, key=key, - keyid=keyid, profile=profile)] + conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) + + def extract_name(queue_url): + # Note: this logic taken from boto, so should be safe + return _urlparse(queue_url).path.split('/')[2] + + try: + r = conn.list_queues(QueueNamePrefix=prefix) + # The 'QueueUrls' attribute is missing if there are no queues + urls = r.get('QueueUrls', []) + return {'result': [extract_name(url) for url in urls]} + except botocore.exceptions.ClientError as e: + return {'error': __utils__['boto3.get_error'](e)} def get_attributes(name, region=None, key=None, keyid=None, profile=None): @@ -187,17 +211,23 @@ def get_attributes(name, region=None, key=None, keyid=None, profile=None): salt myminion boto_sqs.get_attributes myqueue ''' conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) - if not conn: - return {} - queue_obj = conn.get_queue(name) - if not queue_obj: - log.error('Queue {0} does not exist.'.format(name)) - return {} - return conn.get_queue_attributes(queue_obj) + + try: + url = conn.get_queue_url(QueueName=name)['QueueUrl'] + r = conn.get_queue_attributes(QueueUrl=url, AttributeNames=['All']) + return {'result': r['Attributes']} + except botocore.exceptions.ClientError as e: + return {'error': __utils__['boto3.get_error'](e)} -def set_attributes(name, attributes, region=None, key=None, keyid=None, - profile=None): +def set_attributes( + name, + attributes, + region=None, + key=None, + keyid=None, + profile=None, +): ''' Set attributes on an SQS queue. @@ -207,22 +237,13 @@ def set_attributes(name, attributes, region=None, key=None, keyid=None, salt myminion boto_sqs.set_attributes myqueue '{ReceiveMessageWaitTimeSeconds: 20}' region=us-east-1 ''' - ret = True conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) - queue_obj = conn.get_queue(name) - if not queue_obj: - log.error('Queue {0} does not exist.'.format(name)) - ret = False - if isinstance(attributes, string_types): - attributes = json.loads(attributes) - for attr, val in six.iteritems(attributes): - attr_set = queue_obj.set_attribute(attr, val) - if not attr_set: - msg = 'Failed to set attribute {0} = {1} on queue {2}' - log.error(msg.format(attr, val, name)) - ret = False - else: - msg = 'Set attribute {0} = {1} on queue {2}' - log.info(msg.format(attr, val, name)) - return ret + attributes = _preprocess_attributes(attributes) + + try: + url = conn.get_queue_url(QueueName=name)['QueueUrl'] + conn.set_queue_attributes(QueueUrl=url, Attributes=attributes) + except botocore.exceptions.ClientError as e: + return {'error': __utils__['boto3.get_error'](e)} + return {'result': True} diff --git a/salt/modules/boto_vpc.py b/salt/modules/boto_vpc.py index a7effb3c3a7..7dbe9b1eb1b 100644 --- a/salt/modules/boto_vpc.py +++ b/salt/modules/boto_vpc.py @@ -137,6 +137,7 @@ import random import salt.utils.boto import salt.utils.boto3 import salt.utils.compat +import salt.utils.versions from salt.exceptions import SaltInvocationError, CommandExecutionError from salt.utils.versions import LooseVersion as _LooseVersion @@ -150,7 +151,7 @@ ACTIVE = 'active' log = logging.getLogger(__name__) # Import third party libs -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import range # pylint: disable=import-error # pylint: disable=import-error try: @@ -2456,9 +2457,10 @@ def describe_route_table(route_table_id=None, route_table_name=None, ''' - salt.utils.warn_until('Oxygen', - 'The \'describe_route_table\' method has been deprecated and ' - 'replaced by \'describe_route_tables\'.' + salt.utils.versions.warn_until( + 'Oxygen', + 'The \'describe_route_table\' method has been deprecated and ' + 'replaced by \'describe_route_tables\'.' ) if not any((route_table_id, route_table_name, tags)): raise SaltInvocationError('At least one of the following must be specified: ' @@ -2697,11 +2699,11 @@ def request_vpc_peering_connection(requester_vpc_id=None, requester_vpc_name=Non Name tag of the requesting VPC. Exclusive with requester_vpc_id. peer_vpc_id - ID of the VPC tp crete VPC peering connection with. This can be a VPC in + ID of the VPC to create VPC peering connection with. This can be a VPC in another account. Exclusive with peer_vpc_name. peer_vpc_name - Name tag of the VPC tp crete VPC peering connection with. This can only + Name tag of the VPC to create VPC peering connection with. This can only be a VPC in the same account, else resolving it into a vpc ID will almost certainly fail. Exclusive with peer_vpc_id. diff --git a/salt/modules/bower.py b/salt/modules/bower.py index 1b88d804462..ecf7e061347 100644 --- a/salt/modules/bower.py +++ b/salt/modules/bower.py @@ -16,7 +16,7 @@ import logging import shlex # Import salt libs -import salt.utils +import salt.utils.path from salt.exceptions import CommandExecutionError from salt.utils.versions import LooseVersion as _LooseVersion @@ -33,7 +33,7 @@ def __virtual__(): ''' Only work when Bower is installed ''' - if salt.utils.which('bower') is None: + if salt.utils.path.which('bower') is None: return (False, 'The bower module could not be loaded: bower command not found') return True diff --git a/salt/modules/bridge.py b/salt/modules/bridge.py index 37799619b8d..c37e6331f25 100644 --- a/salt/modules/bridge.py +++ b/salt/modules/bridge.py @@ -6,7 +6,7 @@ from __future__ import absolute_import import sys import re -import salt.utils +import salt.utils.path __func_alias__ = { @@ -31,7 +31,7 @@ def __virtual__(): } cur_os = __grains__['kernel'] for _os in supported_os_tool: - if cur_os == _os and salt.utils.which(supported_os_tool[cur_os]): + if cur_os == _os and salt.utils.path.which(supported_os_tool[cur_os]): return True return (False, 'The bridge execution module failed to load: requires one of the following tool/os' ' combinations: ifconfig on FreeBSD/OpenBSD, brctl on Linux or brconfig on NetBSD.') @@ -41,7 +41,7 @@ def _tool_path(ostool): ''' Internal, returns tools path ''' - return salt.utils.which(ostool) + return salt.utils.path.which(ostool) def _linux_brshow(br=None): diff --git a/salt/modules/bsd_shadow.py b/salt/modules/bsd_shadow.py index f6084bd8e22..4194a7f196d 100644 --- a/salt/modules/bsd_shadow.py +++ b/salt/modules/bsd_shadow.py @@ -17,8 +17,8 @@ except ImportError: pass # Import salt libs -import salt.ext.six as six -import salt.utils +from salt.ext import six +import salt.utils.files from salt.exceptions import SaltInvocationError # Define the module's virtual name @@ -76,7 +76,7 @@ def info(name): python_shell=False).split(':')[5:7] elif __grains__['kernel'] in ('NetBSD', 'OpenBSD'): try: - with salt.utils.fopen('/etc/master.passwd', 'r') as fp_: + with salt.utils.files.fopen('/etc/master.passwd', 'r') as fp_: for line in fp_: if line.startswith('{0}:'.format(name)): key = line.split(':') diff --git a/salt/modules/btrfs.py b/salt/modules/btrfs.py index 8e37ed452ea..ed4359cba5b 100644 --- a/salt/modules/btrfs.py +++ b/salt/modules/btrfs.py @@ -27,12 +27,12 @@ import logging # Import Salt libs -import salt.utils import salt.utils.fsutils +import salt.utils.platform from salt.exceptions import CommandExecutionError # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six log = logging.getLogger(__name__) @@ -41,7 +41,7 @@ def __virtual__(): ''' Only work on POSIX-like systems ''' - return not salt.utils.is_windows() and __grains__.get('kernel') == 'Linux' + return not salt.utils.platform.is_windows() and __grains__.get('kernel') == 'Linux' def version(): diff --git a/salt/modules/cabal.py b/salt/modules/cabal.py index ed6029f337e..3260ba442b5 100644 --- a/salt/modules/cabal.py +++ b/salt/modules/cabal.py @@ -10,7 +10,7 @@ from __future__ import absolute_import import logging -import salt.utils +import salt.utils.path from salt.exceptions import CommandExecutionError logger = logging.getLogger(__name__) @@ -25,8 +25,8 @@ def __virtual__(): ''' Only work when cabal-install is installed. ''' - return (salt.utils.which('cabal') is not None) and \ - (salt.utils.which('ghc-pkg') is not None) + return (salt.utils.path.which('cabal') is not None) and \ + (salt.utils.path.which('ghc-pkg') is not None) def update(user=None, env=None): diff --git a/salt/modules/capirca_acl.py b/salt/modules/capirca_acl.py index f925f71df8f..884c0c1e7ea 100644 --- a/salt/modules/capirca_acl.py +++ b/salt/modules/capirca_acl.py @@ -32,15 +32,15 @@ import datetime log = logging.getLogger(__file__) # Import third party libs +from salt.ext import six try: import aclgen HAS_CAPIRCA = True except ImportError: HAS_CAPIRCA = False -# Import Salt modules -import salt.utils -from salt.ext import six +# Import Salt libs +import salt.utils.files # ------------------------------------------------------------------------------ # module properties @@ -213,7 +213,7 @@ def _get_services_mapping(): return _SERVICES services_txt = '' try: - with salt.utils.fopen('/etc/services', 'r') as srv_f: + with salt.utils.files.fopen('/etc/services', 'r') as srv_f: services_txt = srv_f.read() except IOError as ioe: log.error('Unable to read from /etc/services:') diff --git a/salt/modules/cassandra.py b/salt/modules/cassandra.py index 4cd790fcfa0..ba5433120d6 100644 --- a/salt/modules/cassandra.py +++ b/salt/modules/cassandra.py @@ -18,7 +18,7 @@ import logging log = logging.getLogger(__name__) # Import salt libs -import salt.utils +import salt.utils.path HAS_PYCASSA = False try: @@ -35,7 +35,7 @@ def __virtual__(): if not HAS_PYCASSA: return (False, 'The cassandra execution module cannot be loaded: pycassa not installed.') - if HAS_PYCASSA and salt.utils.which('nodetool'): + if HAS_PYCASSA and salt.utils.path.which('nodetool'): return 'cassandra' return (False, 'The cassandra execution module cannot be loaded: nodetool not found.') diff --git a/salt/modules/cassandra_cql.py b/salt/modules/cassandra_cql.py index 67887308e45..b377a495560 100644 --- a/salt/modules/cassandra_cql.py +++ b/salt/modules/cassandra_cql.py @@ -85,7 +85,7 @@ import ssl # Import Salt Libs from salt.exceptions import CommandExecutionError -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import range SSL_VERSION = 'ssl_version' diff --git a/salt/modules/chassis.py b/salt/modules/chassis.py index 9d53de72905..6176070f870 100644 --- a/salt/modules/chassis.py +++ b/salt/modules/chassis.py @@ -17,7 +17,7 @@ from __future__ import absolute_import # Import python libs import logging -import salt.utils +import salt.utils.platform log = logging.getLogger(__name__) @@ -30,7 +30,7 @@ def __virtual__(): ''' Only work on proxy ''' - if salt.utils.is_proxy(): + if salt.utils.platform.is_proxy(): return __virtualname__ return (False, 'The chassis execution module cannot be loaded: ' 'this only works in proxy minions.') diff --git a/salt/modules/chef.py b/salt/modules/chef.py index e2e1345ff94..45ec239e1db 100644 --- a/salt/modules/chef.py +++ b/salt/modules/chef.py @@ -10,11 +10,12 @@ import os import tempfile # Import Salt libs -import salt.utils -import salt.utils.decorators as decorators +import salt.utils.path +import salt.utils.platform +import salt.utils.decorators.path # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six log = logging.getLogger(__name__) @@ -23,7 +24,7 @@ def __virtual__(): ''' Only load if chef is installed ''' - if not salt.utils.which('chef-client'): + if not salt.utils.path.which('chef-client'): return (False, 'Cannot load chef module: chef-client not found') return True @@ -32,7 +33,7 @@ def _default_logfile(exe_name): ''' Retrieve the logfile name ''' - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): tmp_dir = os.path.join(__opts__['cachedir'], 'tmp') if not os.path.isdir(tmp_dir): os.mkdir(tmp_dir) @@ -43,7 +44,7 @@ def _default_logfile(exe_name): logfile = logfile_tmp.name logfile_tmp.close() else: - logfile = salt.utils.path_join( + logfile = salt.utils.path.join( '/var/log', '{0}.log'.format(exe_name) ) @@ -51,7 +52,7 @@ def _default_logfile(exe_name): return logfile -@decorators.which('chef-client') +@salt.utils.decorators.path.which('chef-client') def client(whyrun=False, localmode=False, logfile=None, @@ -140,7 +141,7 @@ def client(whyrun=False, return _exec_cmd(*args, **kwargs) -@decorators.which('chef-solo') +@salt.utils.decorators.path.which('chef-solo') def solo(whyrun=False, logfile=None, **kwargs): diff --git a/salt/modules/chocolatey.py b/salt/modules/chocolatey.py index 87895af3236..1226505f47f 100644 --- a/salt/modules/chocolatey.py +++ b/salt/modules/chocolatey.py @@ -15,6 +15,7 @@ import tempfile # Import salt libs import salt.utils +import salt.utils.platform from salt.utils.versions import LooseVersion as _LooseVersion from salt.exceptions import CommandExecutionError, CommandNotFoundError, \ SaltInvocationError @@ -36,7 +37,7 @@ def __virtual__(): for simulating UAC forces a GUI prompt, and is not compatible with salt-minion running as SYSTEM. ''' - if not salt.utils.is_windows(): + if not salt.utils.platform.is_windows(): return (False, 'Cannot load module chocolatey: Chocolatey requires ' 'Windows') diff --git a/salt/modules/chronos.py b/salt/modules/chronos.py index 6b4e53c337c..066e0b80ca5 100644 --- a/salt/modules/chronos.py +++ b/salt/modules/chronos.py @@ -10,8 +10,8 @@ from __future__ import absolute_import import json import logging -import salt.utils import salt.utils.http +import salt.utils.platform from salt.exceptions import get_error_message @@ -21,7 +21,7 @@ log = logging.getLogger(__file__) def __virtual__(): # only valid in proxy minions for now - return salt.utils.is_proxy() and 'proxy' in __opts__ + return salt.utils.platform.is_proxy() and 'proxy' in __opts__ def _base_url(): diff --git a/salt/modules/cimc.py b/salt/modules/cimc.py new file mode 100644 index 00000000000..ddadaec6a46 --- /dev/null +++ b/salt/modules/cimc.py @@ -0,0 +1,710 @@ +# -*- coding: utf-8 -*- +''' +Module to provide Cisco UCS compatibility to Salt. + +:codeauthor: :email:`Spencer Ervin ` +:maturity: new +:depends: none +:platform: unix + + +Configuration +============= +This module accepts connection configuration details either as +parameters, or as configuration settings in pillar as a Salt proxy. +Options passed into opts will be ignored if options are passed into pillar. + +.. seealso:: + :prox:`Cisco UCS Proxy Module ` + +About +===== +This execution module was designed to handle connections to a Cisco UCS server. This module adds support to send +connections directly to the device through the rest API. + +''' + +# Import Python Libs +from __future__ import absolute_import +import logging + +# Import Salt Libs +import salt.utils.platform +import salt.proxy.cimc + +log = logging.getLogger(__name__) + +__virtualname__ = 'cimc' + + +def __virtual__(): + ''' + Will load for the cimc proxy minions. + ''' + try: + if salt.utils.platform.is_proxy() and \ + __opts__['proxy']['proxytype'] == 'cimc': + return __virtualname__ + except KeyError: + pass + + return False, 'The cimc execution module can only be loaded for cimc proxy minions.' + + +def activate_backup_image(reset=False): + ''' + Activates the firmware backup image. + + CLI Example: + + Args: + reset(bool): Reset the CIMC device on activate. + + .. code-block:: bash + + salt '*' cimc.activate_backup_image + salt '*' cimc.activate_backup_image reset=True + + ''' + + dn = "sys/rack-unit-1/mgmt/fw-boot-def/bootunit-combined" + + r = "no" + + if reset is True: + r = "yes" + + inconfig = """""".format(r) + + ret = __proxy__['cimc.set_config_modify'](dn, inconfig, False) + + return ret + + +def create_user(uid=None, username=None, password=None, priv=None): + ''' + Create a CIMC user with username and password. + + Args: + uid(int): The user ID slot to create the user account in. + + username(str): The name of the user. + + password(str): The clear text password of the user. + + priv(str): The privilege level of the user. + + CLI Example: + + .. code-block:: bash + + salt '*' cimc.create_user 11 username=admin password=foobar priv=admin + + ''' + + if not uid: + raise salt.exceptions.CommandExecutionError("The user ID must be specified.") + + if not username: + raise salt.exceptions.CommandExecutionError("The username must be specified.") + + if not password: + raise salt.exceptions.CommandExecutionError("The password must be specified.") + + if not priv: + raise salt.exceptions.CommandExecutionError("The privilege level must be specified.") + + dn = "sys/user-ext/user-{0}".format(uid) + + inconfig = """""".format(uid, + username, + priv, + password) + + ret = __proxy__['cimc.set_config_modify'](dn, inconfig, False) + + return ret + + +def get_bios_defaults(): + ''' + Get the default values of BIOS tokens. + + CLI Example: + + .. code-block:: bash + + salt '*' cimc.get_bios_defaults + + ''' + ret = __proxy__['cimc.get_config_resolver_class']('biosPlatformDefaults', True) + + return ret + + +def get_bios_settings(): + ''' + Get the C240 server BIOS token values. + + CLI Example: + + .. code-block:: bash + + salt '*' cimc.get_bios_settings + + ''' + ret = __proxy__['cimc.get_config_resolver_class']('biosSettings', True) + + return ret + + +def get_boot_order(): + ''' + Retrieves the configured boot order table. + + CLI Example: + + .. code-block:: bash + + salt '*' cimc.get_boot_order + + ''' + ret = __proxy__['cimc.get_config_resolver_class']('lsbootDef', True) + + return ret + + +def get_cpu_details(): + ''' + Get the CPU product ID details. + + CLI Example: + + .. code-block:: bash + + salt '*' cimc.get_cpu_details + + ''' + ret = __proxy__['cimc.get_config_resolver_class']('pidCatalogCpu', True) + + return ret + + +def get_disks(): + ''' + Get the HDD product ID details. + + CLI Example: + + .. code-block:: bash + + salt '*' cimc.get_disks + + ''' + ret = __proxy__['cimc.get_config_resolver_class']('pidCatalogHdd', True) + + return ret + + +def get_ethernet_interfaces(): + ''' + Get the adapter Ethernet interface details. + + CLI Example: + + .. code-block:: bash + + salt '*' cimc.get_ethernet_interfaces + + ''' + ret = __proxy__['cimc.get_config_resolver_class']('adaptorHostEthIf', True) + + return ret + + +def get_fibre_channel_interfaces(): + ''' + Get the adapter fibre channel interface details. + + CLI Example: + + .. code-block:: bash + + salt '*' cimc.get_fibre_channel_interfaces + + ''' + ret = __proxy__['cimc.get_config_resolver_class']('adaptorHostFcIf', True) + + return ret + + +def get_firmware(): + ''' + Retrieves the current running firmware versions of server components. + + CLI Example: + + .. code-block:: bash + + salt '*' cimc.get_firmware + + ''' + ret = __proxy__['cimc.get_config_resolver_class']('firmwareRunning', False) + + return ret + + +def get_ldap(): + ''' + Retrieves LDAP server details. + + CLI Example: + + .. code-block:: bash + + salt '*' cimc.get_ldap + + ''' + ret = __proxy__['cimc.get_config_resolver_class']('aaaLdap', True) + + return ret + + +def get_management_interface(): + ''' + Retrieve the management interface details. + + CLI Example: + + .. code-block:: bash + + salt '*' cimc.get_management_interface + + ''' + ret = __proxy__['cimc.get_config_resolver_class']('mgmtIf', False) + + return ret + + +def get_memory_token(): + ''' + Get the memory RAS BIOS token. + + CLI Example: + + .. code-block:: bash + + salt '*' cimc.get_memory_token + + ''' + ret = __proxy__['cimc.get_config_resolver_class']('biosVfSelectMemoryRASConfiguration', False) + + return ret + + +def get_memory_unit(): + ''' + Get the IMM/Memory unit product ID details. + + CLI Example: + + .. code-block:: bash + + salt '*' cimc.get_memory_unit + + ''' + ret = __proxy__['cimc.get_config_resolver_class']('pidCatalogDimm', True) + + return ret + + +def get_network_adapters(): + ''' + Get the list of network adapaters and configuration details. + + CLI Example: + + .. code-block:: bash + + salt '*' cimc.get_network_adapters + + ''' + ret = __proxy__['cimc.get_config_resolver_class']('networkAdapterEthIf', True) + + return ret + + +def get_ntp(): + ''' + Retrieves the current running NTP configuration. + + CLI Example: + + .. code-block:: bash + + salt '*' cimc.get_ntp + + ''' + ret = __proxy__['cimc.get_config_resolver_class']('commNtpProvider', False) + + return ret + + +def get_pci_adapters(): + ''' + Get the PCI adapter product ID details. + + CLI Example: + + .. code-block:: bash + + salt '*' cimc.get_disks + + ''' + ret = __proxy__['cimc.get_config_resolver_class']('pidCatalogPCIAdapter', True) + + return ret + + +def get_power_supplies(): + ''' + Retrieves the power supply unit details. + + CLI Example: + + .. code-block:: bash + + salt '*' cimc.get_power_supplies + + ''' + ret = __proxy__['cimc.get_config_resolver_class']('equipmentPsu', False) + + return ret + + +def get_snmp_config(): + ''' + Get the snmp configuration details. + + CLI Example: + + .. code-block:: bash + + salt '*' cimc.get_snmp_config + + ''' + ret = __proxy__['cimc.get_config_resolver_class']('commSnmp', False) + + return ret + + +def get_syslog(): + ''' + Get the Syslog client-server details. + + CLI Example: + + .. code-block:: bash + + salt '*' cimc.get_syslog + + ''' + ret = __proxy__['cimc.get_config_resolver_class']('commSyslogClient', False) + + return ret + + +def get_system_info(): + ''' + Get the system information. + + CLI Example: + + .. code-block:: bash + + salt '*' cimc.get_system_info + + ''' + ret = __proxy__['cimc.get_config_resolver_class']('computeRackUnit', False) + + return ret + + +def get_users(): + ''' + Get the CIMC users. + + CLI Example: + + .. code-block:: bash + + salt '*' cimc.get_users + + ''' + ret = __proxy__['cimc.get_config_resolver_class']('aaaUser', False) + + return ret + + +def get_vic_adapters(): + ''' + Get the VIC adapter general profile details. + + CLI Example: + + .. code-block:: bash + + salt '*' cimc.get_vic_adapters + + ''' + ret = __proxy__['cimc.get_config_resolver_class']('adaptorGenProfile', True) + + return ret + + +def get_vic_uplinks(): + ''' + Get the VIC adapter uplink port details. + + CLI Example: + + .. code-block:: bash + + salt '*' cimc.get_vic_uplinks + + ''' + ret = __proxy__['cimc.get_config_resolver_class']('adaptorExtEthIf', True) + + return ret + + +def mount_share(name=None, + remote_share=None, + remote_file=None, + mount_type="nfs", + username=None, + password=None): + ''' + Mounts a remote file through a remote share. Currently, this feature is supported in version 1.5 or greater. + The remote share can be either NFS, CIFS, or WWW. + + Some of the advantages of CIMC Mounted vMedia include: + Communication between mounted media and target stays local (inside datacenter) + Media mounts can be scripted/automated + No vKVM requirements for media connection + Multiple share types supported + Connections supported through all CIMC interfaces + + Note: CIMC Mounted vMedia is enabled through BIOS configuration. + + Args: + name(str): The name of the volume on the CIMC device. + + remote_share(str): The file share link that will be used to mount the share. This can be NFS, CIFS, or WWW. This + must be the directory path and not the full path to the remote file. + + remote_file(str): The name of the remote file to mount. It must reside within remote_share. + + mount_type(str): The type of share to mount. Valid options are nfs, cifs, and www. + + username(str): An optional requirement to pass credentials to the remote share. If not provided, an + unauthenticated connection attempt will be made. + + password(str): An optional requirement to pass a password to the remote share. If not provided, an + unauthenticated connection attempt will be made. + + CLI Example: + + .. code-block:: bash + + salt '*' cimc.mount_share name=WIN7 remote_share=10.xxx.27.xxx:/nfs remote_file=sl1huu.iso + + salt '*' cimc.mount_share name=WIN7 remote_share=10.xxx.27.xxx:/nfs remote_file=sl1huu.iso username=bob password=badpassword + + ''' + + if not name: + raise salt.exceptions.CommandExecutionError("The share name must be specified.") + + if not remote_share: + raise salt.exceptions.CommandExecutionError("The remote share path must be specified.") + + if not remote_file: + raise salt.exceptions.CommandExecutionError("The remote file name must be specified.") + + if username and password: + mount_options = " mountOptions='username={0},password={1}'".format(username, password) + else: + mount_options = "" + + dn = 'sys/svc-ext/vmedia-svc/vmmap-{0}'.format(name) + inconfig = """""".format(name, mount_type, mount_options, remote_file, remote_share) + + ret = __proxy__['cimc.set_config_modify'](dn, inconfig, False) + + return ret + + +def reboot(): + ''' + Power cycling the server. + + CLI Example: + + .. code-block:: bash + + salt '*' cimc.reboot + + ''' + + dn = "sys/rack-unit-1" + + inconfig = """""" + + ret = __proxy__['cimc.set_config_modify'](dn, inconfig, False) + + return ret + + +def set_ntp_server(server1='', server2='', server3='', server4=''): + ''' + Sets the NTP servers configuration. This will also enable the client NTP service. + + Args: + server1(str): The first IP address or FQDN of the NTP servers. + + server2(str): The second IP address or FQDN of the NTP servers. + + server3(str): The third IP address or FQDN of the NTP servers. + + server4(str): The fourth IP address or FQDN of the NTP servers. + + CLI Example: + + .. code-block:: bash + + salt '*' cimc.set_ntp_server 10.10.10.1 + + salt '*' cimc.set_ntp_server 10.10.10.1 foo.bar.com + + ''' + + dn = "sys/svc-ext/ntp-svc" + inconfig = """""".format(server1, server2, server3, server4) + + ret = __proxy__['cimc.set_config_modify'](dn, inconfig, False) + + return ret + + +def set_syslog_server(server=None, type="primary"): + ''' + Set the SYSLOG server on the host. + + Args: + server(str): The hostname or IP address of the SYSLOG server. + + type(str): Specifies the type of SYSLOG server. This can either be primary (default) or secondary. + + CLI Example: + + .. code-block:: bash + + salt '*' cimc.set_syslog_server foo.bar.com + + salt '*' cimc.set_syslog_server foo.bar.com primary + + salt '*' cimc.set_syslog_server foo.bar.com secondary + + ''' + + if not server: + raise salt.exceptions.CommandExecutionError("The SYSLOG server must be specified.") + + if type == "primary": + dn = "sys/svc-ext/syslog/client-primary" + inconfig = """ """.format(server) + elif type == "secondary": + dn = "sys/svc-ext/syslog/client-secondary" + inconfig = """ """.format(server) + else: + raise salt.exceptions.CommandExecutionError("The SYSLOG type must be either primary or secondary.") + + ret = __proxy__['cimc.set_config_modify'](dn, inconfig, False) + + return ret + + +def tftp_update_bios(server=None, path=None): + ''' + Update the BIOS firmware through TFTP. + + Args: + server(str): The IP address or hostname of the TFTP server. + + path(str): The TFTP path and filename for the BIOS image. + + CLI Example: + + .. code-block:: bash + + salt '*' cimc.tftp_update_bios foo.bar.com HP-SL2.cap + + ''' + + if not server: + raise salt.exceptions.CommandExecutionError("The server name must be specified.") + + if not path: + raise salt.exceptions.CommandExecutionError("The TFTP path must be specified.") + + dn = "sys/rack-unit-1/bios/fw-updatable" + + inconfig = """""".format(server, path) + + ret = __proxy__['cimc.set_config_modify'](dn, inconfig, False) + + return ret + + +def tftp_update_cimc(server=None, path=None): + ''' + Update the CIMC firmware through TFTP. + + Args: + server(str): The IP address or hostname of the TFTP server. + + path(str): The TFTP path and filename for the CIMC image. + + CLI Example: + + .. code-block:: bash + + salt '*' cimc.tftp_update_cimc foo.bar.com HP-SL2.bin + + ''' + + if not server: + raise salt.exceptions.CommandExecutionError("The server name must be specified.") + + if not path: + raise salt.exceptions.CommandExecutionError("The TFTP path must be specified.") + + dn = "sys/rack-unit-1/mgmt/fw-updatable" + + inconfig = """""".format(server, path) + + ret = __proxy__['cimc.set_config_modify'](dn, inconfig, False) + + return ret diff --git a/salt/modules/cisconso.py b/salt/modules/cisconso.py index a6d1f2b0d9f..6b830867140 100644 --- a/salt/modules/cisconso.py +++ b/salt/modules/cisconso.py @@ -9,15 +9,15 @@ for :mod:`salt.proxy.cisconso`. ''' from __future__ import absolute_import -import salt.utils -import salt.ext.six as six +import salt.utils.platform +from salt.ext import six __proxyenabled__ = ['cisconso'] __virtualname__ = 'cisconso' def __virtual__(): - if salt.utils.is_proxy(): + if salt.utils.platform.is_proxy(): return __virtualname__ return (False, 'The cisconso execution module failed to load: ' 'only available on proxy minions.') diff --git a/salt/modules/cloud.py b/salt/modules/cloud.py index 062386b72be..796da4c622e 100644 --- a/salt/modules/cloud.py +++ b/salt/modules/cloud.py @@ -21,7 +21,7 @@ import salt.utils from salt.exceptions import SaltCloudConfigError # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six log = logging.getLogger(__name__) diff --git a/salt/modules/cmdmod.py b/salt/modules/cmdmod.py index ca4a33996e8..653e9a170b6 100644 --- a/salt/modules/cmdmod.py +++ b/salt/modules/cmdmod.py @@ -25,13 +25,18 @@ import tempfile # Import salt libs import salt.utils +import salt.utils.args import salt.utils.files +import salt.utils.path +import salt.utils.platform import salt.utils.powershell -import salt.utils.timed_subprocess -import salt.grains.extra -import salt.ext.six as six -from salt.utils import vt +import salt.utils.stringutils import salt.utils.templates +import salt.utils.timed_subprocess +import salt.utils.versions +import salt.utils.vt +import salt.grains.extra +from salt.ext import six from salt.exceptions import CommandExecutionError, TimedProcTimeoutError, \ SaltInvocationError from salt.log import LOG_LEVELS @@ -45,7 +50,7 @@ try: except ImportError: pass -if salt.utils.is_windows(): +if salt.utils.platform.is_windows(): from salt.utils.win_runas import runas as win_runas HAS_WIN_RUNAS = True else: @@ -142,14 +147,14 @@ def _render_cmd(cmd, cwd, template, saltenv='base', pillarenv=None, pillar_overr def _render(contents): # write out path to temp file tmp_path_fn = salt.utils.files.mkstemp() - with salt.utils.fopen(tmp_path_fn, 'w+') as fp_: + with salt.utils.files.fopen(tmp_path_fn, 'w+') as fp_: fp_.write(contents) data = salt.utils.templates.TEMPLATE_REGISTRY[template]( tmp_path_fn, to_str=True, **kwargs ) - salt.utils.safe_rm(tmp_path_fn) + salt.utils.files.safe_rm(tmp_path_fn) if not data['result']: # Failed to render the template raise CommandExecutionError( @@ -214,7 +219,7 @@ def _gather_pillar(pillarenv, pillar_override): __grains__, __opts__['id'], __opts__['environment'], - pillar=pillar_override, + pillar_override=pillar_override, pillarenv=pillarenv ) ret = pillar.compile_pillar() @@ -294,6 +299,9 @@ def _run(cmd, if runas is None and '__context__' in globals(): runas = __context__.get('runas') + if password is None and '__context__' in globals(): + password = __context__.get('runas_password') + # Set the default working directory to the home directory of the user # salt-minion is running as. Defaults to home directory of user under which # the minion is running. @@ -306,18 +314,18 @@ def _run(cmd, # the euid might not have access to it. See issue #1844 if not os.access(cwd, os.R_OK): cwd = '/' - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): cwd = os.path.abspath(os.sep) else: # Handle edge cases where numeric/other input is entered, and would be # yaml-ified into non-string types - cwd = str(cwd) + cwd = six.text_type(cwd) - if not salt.utils.is_windows(): + if not salt.utils.platform.is_windows(): if not os.path.isfile(shell) or not os.access(shell, os.X_OK): msg = 'The shell {0} is not available'.format(shell) raise CommandExecutionError(msg) - if salt.utils.is_windows() and use_vt: # Memozation so not much overhead + if salt.utils.platform.is_windows() and use_vt: # Memozation so not much overhead raise CommandExecutionError('VT not available on windows') if shell.lower().strip() == 'powershell': @@ -362,8 +370,8 @@ def _run(cmd, def _get_stripped(cmd): # Return stripped command string copies to improve logging. if isinstance(cmd, list): - return [x.strip() if isinstance(x, str) else x for x in cmd] - elif isinstance(cmd, str): + return [x.strip() if isinstance(x, six.string_types) else x for x in cmd] + elif isinstance(cmd, six.string_types): return cmd.strip() else: return cmd @@ -373,7 +381,7 @@ def _run(cmd, # requested. The command output is what will be controlled by the # 'loglevel' parameter. msg = ( - 'Executing command {0}{1}{0} {2}in directory \'{3}\'{4}'.format( + u'Executing command {0}{1}{0} {2}in directory \'{3}\'{4}'.format( '\'' if not isinstance(cmd, list) else '', _get_stripped(cmd), 'as user \'{0}\' '.format(runas) if runas else '', @@ -384,7 +392,7 @@ def _run(cmd, ) log.info(log_callback(msg)) - if runas and salt.utils.is_windows(): + if runas and salt.utils.platform.is_windows(): if not password: msg = 'password is a required argument for runas on Windows' raise CommandExecutionError(msg) @@ -394,7 +402,7 @@ def _run(cmd, raise CommandExecutionError(msg) if not isinstance(cmd, list): - cmd = salt.utils.shlex_split(cmd, posix=False) + cmd = salt.utils.args.shlex_split(cmd, posix=False) cmd = ' '.join(cmd) @@ -457,7 +465,7 @@ def _run(cmd, ) if reset_system_locale is True: - if not salt.utils.is_windows(): + if not salt.utils.platform.is_windows(): # Default to C! # Salt only knows how to parse English words # Don't override if the user has passed LC_ALL @@ -520,7 +528,7 @@ def _run(cmd, runas, _umask) - if not salt.utils.is_windows(): + if not salt.utils.platform.is_windows(): # close_fds is not supported on Windows platforms if you redirect # stdin/stdout/stderr if kwargs['shell'] is True: @@ -535,18 +543,33 @@ def _run(cmd, if python_shell is not True and not isinstance(cmd, list): posix = True - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): posix = False - cmd = salt.utils.shlex_split(cmd, posix=posix) + cmd = salt.utils.args.shlex_split(cmd, posix=posix) if not use_vt: # This is where the magic happens try: proc = salt.utils.timed_subprocess.TimedProc(cmd, **kwargs) except (OSError, IOError) as exc: - raise CommandExecutionError( + msg = ( 'Unable to run command \'{0}\' with the context \'{1}\', ' - 'reason: {2}'.format(cmd, kwargs, exc) + 'reason: '.format( + cmd if _check_loglevel(output_loglevel) is not None + else 'REDACTED', + kwargs + ) ) + try: + if exc.filename is None: + msg += 'command not found' + else: + msg += '{0}: {1}'.format(exc, exc.filename) + except AttributeError: + # Both IOError and OSError have the filename attribute, so this + # is a precaution in case the exception classes in the previous + # try/except are changed. + msg += 'unknown' + raise CommandExecutionError(msg) try: proc.run() @@ -559,17 +582,21 @@ def _run(cmd, ret['retcode'] = 1 return ret - out, err = proc.stdout, proc.stderr - if err is None: - # Will happen if redirect_stderr is True, since stderr was sent to - # stdout. - err = '' + try: + out = proc.stdout.decode(__salt_system_encoding__) + except AttributeError: + out = u'' + + try: + err = proc.stderr.decode(__salt_system_encoding__) + except AttributeError: + err = u'' if rstrip: if out is not None: - out = salt.utils.to_str(out).rstrip() + out = out.rstrip() if err is not None: - err = salt.utils.to_str(err).rstrip() + err = err.rstrip() ret['pid'] = proc.process.pid ret['retcode'] = proc.process.returncode ret['stdout'] = out @@ -588,7 +615,7 @@ def _run(cmd, else: will_timeout = -1 try: - proc = vt.Terminal(cmd, + proc = salt.utils.vt.Terminal(cmd, shell=True, log_stdout=True, log_stderr=True, @@ -627,7 +654,7 @@ def _run(cmd, ret['stderr'] = 'SALT: User break\n{0}'.format(stderr) ret['retcode'] = 1 break - except vt.TerminalException as exc: + except salt.utils.vt.TerminalException as exc: log.error( 'VT: {0}'.format(exc), exc_info_on_loglevel=logging.DEBUG) @@ -755,6 +782,7 @@ def run(cmd, bg=False, password=None, encoded_cmd=False, + raise_err=False, **kwargs): r''' Execute the passed command and return the output as a string @@ -858,6 +886,10 @@ def run(cmd, :param bool encoded_cmd: Specify if the supplied command is encoded. Only applies to shell 'powershell'. + :param bool raise_err: Specifies whether to raise a CommandExecutionError. + If False, the error will be logged, but no exception will be raised. + Default is False. + .. warning:: This function does not process commands through a shell unless the python_shell flag is set to True. This means that any @@ -947,7 +979,9 @@ def run(cmd, ) ) log.error(log_callback(msg)) - log.log(lvl, 'output: {0}'.format(log_callback(ret['stdout']))) + if raise_err: + raise CommandExecutionError(log_callback(ret['stdout'])) + log.log(lvl, u'output: %s', log_callback(ret['stdout'])) return ret['stdout'] @@ -1623,7 +1657,7 @@ def run_all(cmd, :param bool encoded_cmd: Specify if the supplied command is encoded. Only applies to shell 'powershell'. - .. versionadded:: Nitrogen + .. versionadded:: Oxygen :param bool redirect_stderr: If set to ``True``, then stderr will be redirected to stdout. This is helpful for cases where obtaining both the @@ -1705,9 +1739,9 @@ def run_all(cmd, ) log.error(log_callback(msg)) if ret['stdout']: - log.log(lvl, 'stdout: {0}'.format(log_callback(ret['stdout']))) + log.log(lvl, u'stdout: {0}'.format(log_callback(ret['stdout']))) if ret['stderr']: - log.log(lvl, 'stderr: {0}'.format(log_callback(ret['stderr']))) + log.log(lvl, u'stderr: {0}'.format(log_callback(ret['stderr']))) if ret['retcode']: log.log(lvl, 'retcode: {0}'.format(ret['retcode'])) return ret @@ -2086,15 +2120,10 @@ def script(source, ) if '__env__' in kwargs: - salt.utils.warn_until( - 'Oxygen', - 'Parameter \'__env__\' has been detected in the argument list. This ' - 'parameter is no longer used and has been replaced by \'saltenv\' ' - 'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.' - ) + # "env" is not supported; Use "saltenv". kwargs.pop('__env__') - if salt.utils.is_windows() and runas and cwd is None: + if salt.utils.platform.is_windows() and runas and cwd is None: cwd = tempfile.mkdtemp(dir=__opts__['cachedir']) __salt__['win_dacl.add_ace']( cwd, 'File', runas, 'READ&EXECUTE', 'ALLOW', @@ -2112,7 +2141,7 @@ def script(source, saltenv, **kwargs) if not fn_: - if salt.utils.is_windows() and runas: + if salt.utils.platform.is_windows() and runas: _cleanup_tempfile(cwd) else: _cleanup_tempfile(path) @@ -2124,7 +2153,7 @@ def script(source, else: fn_ = __salt__['cp.cache_file'](source, saltenv) if not fn_: - if salt.utils.is_windows() and runas: + if salt.utils.platform.is_windows() and runas: _cleanup_tempfile(cwd) else: _cleanup_tempfile(path) @@ -2134,7 +2163,7 @@ def script(source, 'stderr': '', 'cache_error': True} shutil.copyfile(fn_, path) - if not salt.utils.is_windows(): + if not salt.utils.platform.is_windows(): os.chmod(path, 320) os.chown(path, __salt__['file.user_to_uid'](runas), -1) ret = _run(path + ' ' + str(args) if args else path, @@ -2154,7 +2183,7 @@ def script(source, bg=bg, password=password, **kwargs) - if salt.utils.is_windows() and runas: + if salt.utils.platform.is_windows() and runas: _cleanup_tempfile(cwd) else: _cleanup_tempfile(path) @@ -2302,12 +2331,7 @@ def script_retcode(source, salt '*' cmd.script_retcode salt://scripts/runme.sh stdin='one\\ntwo\\nthree\\nfour\\nfive\\n' ''' if '__env__' in kwargs: - salt.utils.warn_until( - 'Oxygen', - 'Parameter \'__env__\' has been detected in the argument list. This ' - 'parameter is no longer used and has been replaced by \'saltenv\' ' - 'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.' - ) + # "env" is not supported; Use "saltenv". kwargs.pop('__env__') return script(source=source, @@ -2340,7 +2364,7 @@ def which(cmd): salt '*' cmd.which cat ''' - return salt.utils.which(cmd) + return salt.utils.path.which(cmd) def which_bin(cmds): @@ -2353,7 +2377,7 @@ def which_bin(cmds): salt '*' cmd.which_bin '[pip2, pip, pip-python]' ''' - return salt.utils.which_bin(cmds) + return salt.utils.path.which_bin(cmds) def has_exec(cmd): @@ -2369,33 +2393,39 @@ def has_exec(cmd): return which(cmd) is not None -def exec_code(lang, code, cwd=None): +def exec_code(lang, code, cwd=None, args=None, **kwargs): ''' Pass in two strings, the first naming the executable language, aka - python2, python3, ruby, perl, lua, etc. the second string containing the code you wish to execute. The stdout will be returned. + All parameters from :mod:`cmd.run_all ` except python_shell can be used. + CLI Example: .. code-block:: bash salt '*' cmd.exec_code ruby 'puts "cheese"' + salt '*' cmd.exec_code ruby 'puts "cheese"' args='["arg1", "arg2"]' env='{"FOO": "bar"}' ''' - return exec_code_all(lang, code, cwd)['stdout'] + return exec_code_all(lang, code, cwd, args, **kwargs)['stdout'] -def exec_code_all(lang, code, cwd=None): +def exec_code_all(lang, code, cwd=None, args=None, **kwargs): ''' Pass in two strings, the first naming the executable language, aka - python2, python3, ruby, perl, lua, etc. the second string containing the code you wish to execute. All cmd artifacts (stdout, stderr, retcode, pid) will be returned. + All parameters from :mod:`cmd.run_all ` except python_shell can be used. + CLI Example: .. code-block:: bash salt '*' cmd.exec_code_all ruby 'puts "cheese"' + salt '*' cmd.exec_code_all ruby 'puts "cheese"' args='["arg1", "arg2"]' env='{"FOO": "bar"}' ''' powershell = lang.lower().startswith("powershell") @@ -2404,7 +2434,7 @@ def exec_code_all(lang, code, cwd=None): else: codefile = salt.utils.files.mkstemp() - with salt.utils.fopen(codefile, 'w+t', binary=False) as fp_: + with salt.utils.files.fopen(codefile, 'w+t', binary=False) as fp_: fp_.write(code) if powershell: @@ -2412,7 +2442,12 @@ def exec_code_all(lang, code, cwd=None): else: cmd = [lang, codefile] - ret = run_all(cmd, cwd=cwd, python_shell=False) + if isinstance(args, six.string_types): + cmd.append(args) + elif isinstance(args, list): + cmd += args + + ret = run_all(cmd, cwd=cwd, python_shell=False, **kwargs) os.remove(codefile) return ret @@ -2435,8 +2470,8 @@ def tty(device, echo=''): else: return {'Error': 'The specified device is not a valid TTY'} try: - with salt.utils.fopen(teletype, 'wb') as tty_device: - tty_device.write(salt.utils.to_bytes(echo)) + with salt.utils.files.fopen(teletype, 'wb') as tty_device: + tty_device.write(salt.utils.stringutils.to_bytes(echo)) return { 'Success': 'Message was successfully echoed to {0}'.format(teletype) } @@ -2641,13 +2676,13 @@ def _is_valid_shell(shell): Attempts to search for valid shells on a system and see if a given shell is in the list ''' - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): return True # Don't even try this for Windows shells = '/etc/shells' available_shells = [] if os.path.exists(shells): try: - with salt.utils.fopen(shells, 'r') as shell_fp: + with salt.utils.files.fopen(shells, 'r') as shell_fp: lines = shell_fp.read().splitlines() for line in lines: if line.startswith('#'): @@ -2679,7 +2714,7 @@ def shells(): ret = [] if os.path.exists(shells_fn): try: - with salt.utils.fopen(shells_fn, 'r') as shell_fp: + with salt.utils.files.fopen(shells_fn, 'r') as shell_fp: lines = shell_fp.read().splitlines() for line in lines: line = line.strip() @@ -2752,7 +2787,7 @@ def shell_info(shell, list_modules=False): } # Ensure ret['installed'] always as a value of True, False or None (not sure) ret = {'installed': False} - if salt.utils.is_windows() and shell == 'powershell': + if salt.utils.platform.is_windows() and shell == 'powershell': pw_keys = __salt__['reg.list_keys']( 'HKEY_LOCAL_MACHINE', 'Software\\Microsoft\\PowerShell') @@ -2816,7 +2851,7 @@ def shell_info(shell, list_modules=False): # We need to assume ports of unix shells to windows will look after # themselves in setting HOME as they do it in many different ways newenv = os.environ - if ('HOME' not in newenv) and (not salt.utils.is_windows()): + if ('HOME' not in newenv) and (not salt.utils.platform.is_windows()): newenv['HOME'] = os.path.expanduser('~') log.debug('HOME environment set to {0}'.format(newenv['HOME'])) try: @@ -3184,7 +3219,7 @@ def powershell_all(cmd, salt '*' cmd.run_all '$PSVersionTable.CLRVersion' shell=powershell salt '*' cmd.run_all 'Get-NetTCPConnection' shell=powershell - .. versionadded:: Nitrogen + .. versionadded:: Oxygen .. warning:: diff --git a/salt/modules/composer.py b/salt/modules/composer.py index f880dbb18c8..a7a38f28bc0 100644 --- a/salt/modules/composer.py +++ b/salt/modules/composer.py @@ -9,7 +9,8 @@ import logging import os.path # Import salt libs -import salt.utils +import salt.utils.args +import salt.utils.path from salt.exceptions import ( CommandExecutionError, CommandNotFoundError, @@ -35,7 +36,7 @@ def _valid_composer(composer): ''' Validate the composer file is indeed there. ''' - if salt.utils.which(composer): + if salt.utils.path.which(composer): return True return False @@ -152,7 +153,7 @@ def _run_composer(action, cmd = [composer, action, '--no-interaction', '--no-ansi'] if extra_flags is not None: - cmd.extend(salt.utils.shlex_split(extra_flags)) + cmd.extend(salt.utils.args.shlex_split(extra_flags)) # If php is set, prepend it if php is not None: diff --git a/salt/modules/config.py b/salt/modules/config.py index 9916950472a..e7b1445a8d7 100644 --- a/salt/modules/config.py +++ b/salt/modules/config.py @@ -13,6 +13,7 @@ import logging # Import salt libs import salt.config import salt.utils +import salt.utils.platform try: # Gated for salt-ssh (salt.utils.cloud imports msgpack) import salt.utils.cloud @@ -25,9 +26,9 @@ import salt.syspaths as syspaths import salt.utils.sdb as sdb # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six -if salt.utils.is_windows(): +if salt.utils.platform.is_windows(): _HOSTS_FILE = os.path.join( os.environ['SystemRoot'], 'System32', 'drivers', 'etc', 'hosts') else: @@ -176,14 +177,14 @@ def merge(value, if not omit_opts: if value in __opts__: ret = __opts__[value] - if isinstance(ret, str): + if isinstance(ret, six.string_types): return ret if not omit_master: if value in __pillar__.get('master', {}): tmp = __pillar__['master'][value] if ret is None: ret = tmp - if isinstance(ret, str): + if isinstance(ret, six.string_types): return ret elif isinstance(ret, dict) and isinstance(tmp, dict): tmp.update(ret) @@ -196,7 +197,7 @@ def merge(value, tmp = __pillar__[value] if ret is None: ret = tmp - if isinstance(ret, str): + if isinstance(ret, six.string_types): return ret elif isinstance(ret, dict) and isinstance(tmp, dict): tmp.update(ret) diff --git a/salt/modules/consul.py b/salt/modules/consul.py index 049831b0829..68a4bd32884 100644 --- a/salt/modules/consul.py +++ b/salt/modules/consul.py @@ -205,7 +205,11 @@ def get(consul_url=None, key=None, recurse=False, decode=False, raw=False): if ret['res']: if decode: for item in ret['data']: - item['Value'] = base64.b64decode(item['Value']) + if item['Value'] != None: + item['Value'] = base64.b64decode(item['Value']) + else: + item['Value'] = "" + return ret @@ -1978,6 +1982,8 @@ def acl_create(consul_url=None, **kwargs): Create a new ACL token. :param consul_url: The Consul server URL. + :param id: Unique identifier for the ACL to create + leave it blank to let consul server generate one :param name: Meaningful indicator of the ACL's purpose. :param type: Type is either client or management. A management token is comparable to a root user and has the @@ -2008,6 +2014,9 @@ def acl_create(consul_url=None, **kwargs): else: raise SaltInvocationError('Required argument "name" is missing.') + if 'id' in kwargs: + data['ID'] = kwargs['id'] + if 'type' in kwargs: data['Type'] = kwargs['type'] @@ -2126,7 +2135,7 @@ def acl_delete(consul_url=None, **kwargs): ret['res'] = False return ret - function = 'acl/delete/{0}'.format(kwargs['id']) + function = 'acl/destroy/{0}'.format(kwargs['id']) res = _query(consul_url=consul_url, data=data, method='PUT', diff --git a/salt/modules/container_resource.py b/salt/modules/container_resource.py index 4148a906599..b3ddc46dfad 100644 --- a/salt/modules/container_resource.py +++ b/salt/modules/container_resource.py @@ -21,9 +21,10 @@ import time import traceback # Import salt libs -import salt.utils +import salt.utils.args +import salt.utils.path +import salt.utils.vt from salt.exceptions import CommandExecutionError, SaltInvocationError -from salt.utils import vt log = logging.getLogger(__name__) @@ -54,12 +55,12 @@ def _validate(wrapped): 'Invalid command execution driver. Valid drivers are: {0}' .format(', '.join(valid_driver[container_type])) ) - if exec_driver == 'lxc-attach' and not salt.utils.which('lxc-attach'): + if exec_driver == 'lxc-attach' and not salt.utils.path.which('lxc-attach'): raise SaltInvocationError( 'The \'lxc-attach\' execution driver has been chosen, but ' 'lxc-attach is not available. LXC may not be installed.' ) - return wrapped(*args, **salt.utils.clean_kwargs(**kwargs)) + return wrapped(*args, **salt.utils.args.clean_kwargs(**kwargs)) return wrapper @@ -220,7 +221,7 @@ def run(name, ignore_retcode=ignore_retcode) else: stdout, stderr = '', '' - proc = vt.Terminal( + proc = salt.utils.vt.Terminal( full_cmd, shell=python_shell, log_stdin_level='quiet' if output_loglevel == 'quiet' else 'info', @@ -251,7 +252,7 @@ def run(name, 'pid': 2, 'stdout': stdout, 'stderr': stderr} - except vt.TerminalException: + except salt.utils.vt.TerminalException: trace = traceback.format_exc() log.error(trace) ret = stdout if output is None \ diff --git a/salt/modules/cp.py b/salt/modules/cp.py index 146b6312f3c..86634d559c3 100644 --- a/salt/modules/cp.py +++ b/salt/modules/cp.py @@ -17,6 +17,8 @@ import salt.fileclient import salt.utils import salt.utils.files import salt.utils.gzip_util +import salt.utils.locales +import salt.utils.templates import salt.utils.url import salt.crypt import salt.transport @@ -24,7 +26,7 @@ from salt.exceptions import CommandExecutionError from salt.ext.six.moves.urllib.parse import urlparse as _urlparse # pylint: disable=import-error,no-name-in-module # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six log = logging.getLogger(__name__) @@ -49,7 +51,7 @@ def _gather_pillar(pillarenv, pillar_override): __grains__, __opts__['id'], __opts__['environment'], - pillar=pillar_override, + pillar_override=pillar_override, pillarenv=pillarenv ) ret = pillar.compile_pillar() @@ -58,7 +60,36 @@ def _gather_pillar(pillarenv, pillar_override): return ret -def recv(dest, chunk, append=False, compressed=True, mode=None): +def recv(files, dest): + ''' + Used with salt-cp, pass the files dict, and the destination. + + This function receives small fast copy files from the master via salt-cp. + It does not work via the CLI. + ''' + ret = {} + for path, data in six.iteritems(files): + if os.path.basename(path) == os.path.basename(dest) \ + and not os.path.isdir(dest): + final = dest + elif os.path.isdir(dest): + final = os.path.join(dest, os.path.basename(path)) + elif os.path.isdir(os.path.dirname(dest)): + final = dest + else: + return 'Destination unavailable' + + try: + with salt.utils.files.fopen(final, 'w+') as fp_: + fp_.write(data) + ret[final] = True + except IOError: + ret[final] = False + + return ret + + +def recv_chunked(dest, chunk, append=False, compressed=True, mode=None): ''' This function receives files copied to the minion using ``salt-cp`` and is not intended to be used directly on the CLI. @@ -86,7 +117,7 @@ def recv(dest, chunk, append=False, compressed=True, mode=None): open_mode = 'ab' if append else 'wb' try: - fh_ = salt.utils.fopen(dest, open_mode) # pylint: disable=W8470 + fh_ = salt.utils.files.fopen(dest, open_mode) # pylint: disable=W8470 except (IOError, OSError) as exc: if exc.errno != errno.ENOENT: # Parent dir does not exist, we need to create it @@ -96,7 +127,7 @@ def recv(dest, chunk, append=False, compressed=True, mode=None): except (IOError, OSError) as makedirs_exc: # Failed to make directory return _error(makedirs_exc.__str__()) - fh_ = salt.utils.fopen(dest, open_mode) # pylint: disable=W8470 + fh_ = salt.utils.files.fopen(dest, open_mode) # pylint: disable=W8470 try: # Write the chunk to disk @@ -178,14 +209,14 @@ def _render_filenames(path, dest, saltenv, template, **kw): ''' # write out path to temp file tmp_path_fn = salt.utils.files.mkstemp() - with salt.utils.fopen(tmp_path_fn, 'w+') as fp_: + with salt.utils.files.fopen(tmp_path_fn, 'w+') as fp_: fp_.write(contents) data = salt.utils.templates.TEMPLATE_REGISTRY[template]( tmp_path_fn, to_str=True, **kwargs ) - salt.utils.safe_rm(tmp_path_fn) + salt.utils.files.safe_rm(tmp_path_fn) if not data['result']: # Failed to render the template raise CommandExecutionError( @@ -209,6 +240,9 @@ def get_file(path, gzip=None, **kwargs): ''' + .. versionchanged:: Oxygen + ``dest`` can now be a directory + Used to get a single file from the salt master CLI Example: @@ -320,6 +354,9 @@ def get_dir(path, dest, saltenv='base', template=None, gzip=None, **kwargs): def get_url(path, dest='', saltenv='base', makedirs=False): ''' + .. versionchanged:: Oxygen + ``dest`` can now be a directory + Used to get a single file from a URL. path @@ -385,7 +422,7 @@ def get_file_str(path, saltenv='base'): fn_ = cache_file(path, saltenv) if isinstance(fn_, six.string_types): try: - with salt.utils.fopen(fn_, 'r') as fp_: + with salt.utils.files.fopen(fn_, 'r') as fp_: return fp_.read() except IOError: return False @@ -421,7 +458,11 @@ def cache_file(path, saltenv='base'): It may be necessary to quote the URL when using the querystring method, depending on the shell being used to run the command. ''' - contextkey = '{0}_|-{1}_|-{2}'.format('cp.cache_file', path, saltenv) + path = salt.utils.locales.sdecode(path) + saltenv = salt.utils.locales.sdecode(saltenv) + + contextkey = u'{0}_|-{1}_|-{2}'.format('cp.cache_file', path, saltenv) + path_is_remote = _urlparse(path).scheme in ('http', 'https', 'ftp') try: if path_is_remote and contextkey in __context__: @@ -447,9 +488,8 @@ def cache_file(path, saltenv='base'): result = _client().cache_file(path, saltenv) if not result: log.error( - 'Unable to cache file \'{0}\' from saltenv \'{1}\'.'.format( - path, saltenv - ) + u'Unable to cache file \'%s\' from saltenv \'%s\'.', + path, saltenv ) if path_is_remote: # Cache was successful, store the result in __context__ to prevent @@ -762,7 +802,7 @@ def push(path, keep_symlinks=False, upload_path=None, remove_source=False): 'path': load_path_list, 'tok': auth.gen_token('salt')} channel = salt.transport.Channel.factory(__opts__) - with salt.utils.fopen(path, 'rb') as fp_: + with salt.utils.files.fopen(path, 'rb') as fp_: init_send = False while True: load['loc'] = fp_.tell() @@ -770,7 +810,7 @@ def push(path, keep_symlinks=False, upload_path=None, remove_source=False): if not load['data'] and init_send: if remove_source: try: - salt.utils.rm_rf(path) + salt.utils.files.rm_rf(path) log.debug('Removing source file \'{0}\''.format(path)) except IOError: log.error('cp.push failed to remove file \ diff --git a/salt/modules/cpan.py b/salt/modules/cpan.py index c070c6af6c2..509ce789138 100644 --- a/salt/modules/cpan.py +++ b/salt/modules/cpan.py @@ -12,7 +12,8 @@ import os.path import logging # Import salt libs -import salt.utils +import salt.utils.files +import salt.utils.path log = logging.getLogger(__name__) @@ -26,7 +27,7 @@ def __virtual__(): ''' Only work on supported POSIX-like systems ''' - if salt.utils.which('cpan'): + if salt.utils.path.which('cpan'): return True return (False, 'Unable to locate cpan. Make sure it is installed and in the PATH.') @@ -107,7 +108,7 @@ def remove(module, details=False): if 'MANIFEST' not in contents: continue mfile = os.path.join(build_dir, 'MANIFEST') - with salt.utils.fopen(mfile, 'r') as fh_: + with salt.utils.files.fopen(mfile, 'r') as fh_: for line in fh_.readlines(): if line.startswith('lib/'): files.append(line.replace('lib/', ins_path).strip()) diff --git a/salt/modules/cron.py b/salt/modules/cron.py index 26ea7524aaf..c387da3d637 100644 --- a/salt/modules/cron.py +++ b/salt/modules/cron.py @@ -16,6 +16,8 @@ import random # Import salt libs import salt.utils import salt.utils.files +import salt.utils.path +import salt.utils.stringutils from salt.ext.six.moves import range from salt.utils.locales import sdecode @@ -25,7 +27,7 @@ SALT_CRON_NO_IDENTIFIER = 'NO ID SET' def __virtual__(): - if salt.utils.which('crontab'): + if salt.utils.path.which('crontab'): return True else: return (False, 'Cannot load cron module: crontab command not found') @@ -33,7 +35,7 @@ def __virtual__(): def _encode(string): try: - string = salt.utils.to_str(string) + string = salt.utils.stringutils.to_str(string) except TypeError: if not string: string = '' @@ -227,13 +229,13 @@ def _write_cron_lines(user, lines): path = salt.utils.files.mkstemp() if _check_instance_uid_match(user) or __grains__.get('os_family') in ('Solaris', 'AIX'): # In some cases crontab command should be executed as user rather than root - with salt.utils.fpopen(path, 'w+', uid=__salt__['file.user_to_uid'](user), mode=0o600) as fp_: + with salt.utils.files.fpopen(path, 'w+', uid=__salt__['file.user_to_uid'](user), mode=0o600) as fp_: fp_.writelines(lines) ret = __salt__['cmd.run_all'](_get_cron_cmdstr(path), runas=user, python_shell=False) else: - with salt.utils.fpopen(path, 'w+', mode=0o600) as fp_: + with salt.utils.files.fpopen(path, 'w+', mode=0o600) as fp_: fp_.writelines(lines) ret = __salt__['cmd.run_all'](_get_cron_cmdstr(path, user), python_shell=False) diff --git a/salt/modules/cryptdev.py b/salt/modules/cryptdev.py index fefac223688..3063bb1936a 100644 --- a/salt/modules/cryptdev.py +++ b/salt/modules/cryptdev.py @@ -12,11 +12,12 @@ import os import re # Import salt libraries -import salt.utils +import salt.utils.files +import salt.utils.platform from salt.exceptions import CommandExecutionError # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import filter, zip # pylint: disable=import-error,redefined-builtin # Set up logger @@ -30,7 +31,7 @@ def __virtual__(): ''' Only load on POSIX-like systems ''' - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): return (False, 'The cryptdev module cannot be loaded: not a POSIX-like system') return True @@ -140,7 +141,7 @@ def crypttab(config='/etc/crypttab'): ret = {} if not os.path.isfile(config): return ret - with salt.utils.fopen(config) as ifile: + with salt.utils.files.fopen(config) as ifile: for line in ifile: try: entry = _crypttab_entry.dict_from_line(line) @@ -177,7 +178,7 @@ def rm_crypttab(name, config='/etc/crypttab'): # the list. At the end, re-create the config from just those lines. lines = [] try: - with salt.utils.fopen(config, 'r') as ifile: + with salt.utils.files.fopen(config, 'r') as ifile: for line in ifile: try: if criteria.match(line): @@ -194,7 +195,7 @@ def rm_crypttab(name, config='/etc/crypttab'): if modified: try: - with salt.utils.fopen(config, 'w+') as ofile: + with salt.utils.files.fopen(config, 'w+') as ofile: ofile.writelines(lines) except (IOError, OSError) as exc: msg = "Couldn't write to {0}: {1}" @@ -271,7 +272,7 @@ def set_crypttab( raise CommandExecutionError('Bad config file "{0}"'.format(config)) try: - with salt.utils.fopen(config, 'r') as ifile: + with salt.utils.files.fopen(config, 'r') as ifile: for line in ifile: try: if criteria.match(line): @@ -301,7 +302,7 @@ def set_crypttab( if ret != 'present': # ret in ['new', 'change']: if not test: try: - with salt.utils.fopen(config, 'w+') as ofile: + with salt.utils.files.fopen(config, 'w+') as ofile: # The line was changed, commit it! ofile.writelines(lines) except (IOError, OSError): diff --git a/salt/modules/csf.py b/salt/modules/csf.py index 353462c6a1f..264a159cdec 100644 --- a/salt/modules/csf.py +++ b/salt/modules/csf.py @@ -12,8 +12,8 @@ from __future__ import absolute_import import re # Import Salt Libs +import salt.utils.path from salt.exceptions import CommandExecutionError, SaltInvocationError -import salt.utils from salt.ext.six.moves import map @@ -21,7 +21,7 @@ def __virtual__(): ''' Only load if csf exists on the system ''' - if salt.utils.which('csf') is None: + if salt.utils.path.which('csf') is None: return (False, 'The csf execution module cannot be loaded: csf unavailable.') else: @@ -77,7 +77,7 @@ def __csf_cmd(cmd): ''' Execute csf command ''' - csf_cmd = '{0} {1}'.format(salt.utils.which('csf'), cmd) + csf_cmd = '{0} {1}'.format(salt.utils.path.which('csf'), cmd) out = __salt__['cmd.run_all'](csf_cmd) if out['retcode'] != 0: diff --git a/salt/modules/cyg.py b/salt/modules/cyg.py index d92456c307b..2174be017ff 100644 --- a/salt/modules/cyg.py +++ b/salt/modules/cyg.py @@ -16,9 +16,13 @@ import bz2 from salt.ext.six.moves.urllib.request import urlopen as _urlopen # pylint: disable=no-name-in-module,import-error # Import Salt libs -import salt.utils +import salt.utils.files +import salt.utils.platform from salt.exceptions import SaltInvocationError +# Import 3rd-party libs +from salt.ext import six + LOG = logging.getLogger(__name__) @@ -30,8 +34,10 @@ __virtualname__ = 'cyg' def __virtual__(): - """Only works on Windows systems.""" - if salt.utils.is_windows(): + ''' + Only works on Windows systems + ''' + if salt.utils.platform.is_windows(): return __virtualname__ return (False, 'Module cyg: module only works on Windows systems.') @@ -146,7 +152,7 @@ def _run_silent_cygwin(cyg_arch='x86_64', os.remove(cyg_setup_path) file_data = _urlopen(cyg_setup_source) - with salt.utils.fopen(cyg_setup_path, "wb") as fhw: + with salt.utils.files.fopen(cyg_setup_path, "wb") as fhw: fhw.write(file_data.read()) setup_command = cyg_setup_path @@ -305,7 +311,7 @@ def list_(package='', cyg_arch='x86_64'): args = ' '.join(['-c', '-d', package]) stdout = _cygcheck(args, cyg_arch=cyg_arch) lines = [] - if isinstance(stdout, str): + if isinstance(stdout, six.string_types): lines = str(stdout).splitlines() for line in lines: match = re.match(r'^([^ ]+) *([^ ]+)', line) diff --git a/salt/modules/daemontools.py b/salt/modules/daemontools.py index 367bb6cee3b..326e97086e0 100644 --- a/salt/modules/daemontools.py +++ b/salt/modules/daemontools.py @@ -21,7 +21,7 @@ import os.path import re # Import salt libs -import salt.utils +import salt.utils.path from salt.exceptions import CommandExecutionError # Function alias to not shadow built-ins. @@ -48,7 +48,7 @@ for service_dir in VALID_SERVICE_DIRS: def __virtual__(): # Ensure that daemontools is installed properly. BINS = frozenset(('svc', 'supervise', 'svok')) - if all(salt.utils.which(b) for b in BINS) and SERVICE_DIR: + if all(salt.utils.path.which(b) for b in BINS) and SERVICE_DIR: return __virtualname__ return False diff --git a/salt/modules/data.py b/salt/modules/data.py index 5fc60ed88eb..548016203ae 100644 --- a/salt/modules/data.py +++ b/salt/modules/data.py @@ -11,11 +11,11 @@ import ast import logging # Import salt libs -import salt.utils +import salt.utils.files import salt.payload # Import 3rd-party lib -import salt.ext.six as six +from salt.ext import six log = logging.getLogger(__name__) @@ -52,7 +52,7 @@ def load(): try: datastore_path = os.path.join(__opts__['cachedir'], 'datastore') - with salt.utils.fopen(datastore_path, 'rb') as rfh: + with salt.utils.files.fopen(datastore_path, 'rb') as rfh: return serial.loads(rfh.read()) except (IOError, OSError, NameError): return {} @@ -76,7 +76,7 @@ def dump(new_data): try: datastore_path = os.path.join(__opts__['cachedir'], 'datastore') - with salt.utils.fopen(datastore_path, 'w+b') as fn_: + with salt.utils.files.fopen(datastore_path, 'w+b') as fn_: serial = salt.payload.Serial(__opts__) serial.dump(new_data, fn_) diff --git a/salt/modules/datadog_api.py b/salt/modules/datadog_api.py new file mode 100644 index 00000000000..e59ba305ff2 --- /dev/null +++ b/salt/modules/datadog_api.py @@ -0,0 +1,264 @@ +# -*- coding: utf-8 -*- +''' +An execution module that interacts with the Datadog API + +The following parameters are required for all functions. + +api_key + The datadog API key + +app_key + The datadog application key + +Full argument reference is available on the Datadog API reference page +https://docs.datadoghq.com/api/ +''' + +# Import salt libs +from __future__ import absolute_import +from salt.exceptions import SaltInvocationError + +# Import third party libs +import requests +HAS_DATADOG = True +try: + import datadog +except ImportError: + HAS_DATADOG = False + +# Define the module's virtual name +__virtualname__ = 'datadog' + + +def __virtual__(): + if HAS_DATADOG: + return 'datadog' + else: + message = 'Unable to import the python datadog module. Is it installed?' + return False, message + + +def _initialize_connection(api_key, app_key): + ''' + Initialize Datadog connection + ''' + if api_key is None: + raise SaltInvocationError('api_key must be specified') + if app_key is None: + raise SaltInvocationError('app_key must be specified') + options = { + 'api_key': api_key, + 'app_key': app_key + } + datadog.initialize(**options) + + +def schedule_downtime(scope, + api_key=None, + app_key=None, + monitor_id=None, + start=None, + end=None, + message=None, + recurrence=None, + timezone=None, + test=False): + ''' + Schedule downtime for a scope of monitors. + + CLI Example: + + .. code-block:: bash + + salt-call datadog.schedule_downtime 'host:app2' \\ + stop=$(date --date='30 minutes' +%s) \\ + app_key='0123456789' \\ + api_key='9876543210' + + Optional arguments + + :param monitor_id: The ID of the monitor + :param start: Start time in seconds since the epoch + :param end: End time in seconds since the epoch + :param message: A message to send in a notification for this downtime + :param recurrence: Repeat this downtime periodically + :param timezone: Specify the timezone + ''' + ret = {'result': False, + 'response': None, + 'comment': ''} + + if api_key is None: + raise SaltInvocationError('api_key must be specified') + if app_key is None: + raise SaltInvocationError('app_key must be specified') + if test is True: + ret['result'] = True + ret['comment'] = 'A schedule downtime API call would have been made.' + return ret + _initialize_connection(api_key, app_key) + + # Schedule downtime + try: + response = datadog.api.Downtime.create(scope=scope, + monitor_id=monitor_id, + start=start, + end=end, + message=message, + recurrence=recurrence, + timezone=timezone) + except ValueError: + comment = ('Unexpected exception in Datadog Schedule Downtime API ' + 'call. Are your keys correct?') + ret['comment'] = comment + return ret + + ret['response'] = response + if 'active' in response.keys(): + ret['result'] = True + ret['comment'] = 'Successfully scheduled downtime' + return ret + + +def cancel_downtime(api_key=None, + app_key=None, + scope=None, + id=None): + ''' + Cancel a downtime by id or by scope. + + CLI Example: + + .. code-block:: bash + + salt-call datadog.cancel_downtime scope='host:app01' \\ + api_key='0123456789' \\ + app_key='9876543210'` + + Arguments - Either scope or id is required. + + :param id: The downtime ID + :param scope: The downtime scope + ''' + if api_key is None: + raise SaltInvocationError('api_key must be specified') + if app_key is None: + raise SaltInvocationError('app_key must be specified') + _initialize_connection(api_key, app_key) + + ret = {'result': False, + 'response': None, + 'comment': ''} + if id: + response = datadog.api.Downtime.delete(id) + ret['response'] = response + if not response: # Then call has succeeded + ret['result'] = True + ret['comment'] = 'Successfully cancelled downtime' + return ret + elif scope: + params = { + 'api_key': api_key, + 'application_key': app_key, + 'scope': scope + } + response = requests.post( + 'https://app.datadoghq.com/api/v1/downtime/cancel/by_scope', + params=params + ) + if response.status_code == 200: + ret['result'] = True + ret['response'] = response.json() + ret['comment'] = 'Successfully cancelled downtime' + else: + ret['response'] = response.text + ret['comment'] = 'Status Code: {}'.format(response.status_code) + return ret + else: + raise SaltInvocationError('One of id or scope must be specified') + + return ret + + +def post_event(api_key=None, + app_key=None, + title=None, + text=None, + date_happened=None, + priority=None, + host=None, + tags=None, + alert_type=None, + aggregation_key=None, + source_type_name=None): + ''' + Post an event to the Datadog stream. + + CLI Example + + .. code-block:: bash + + salt-call datadog.post_event api_key='0123456789' \\ + app_key='9876543210' \\ + title='Salt Highstate' \\ + text="Salt highstate was run on $(salt-call grains.get id)" \\ + tags='["service:salt", "event:highstate"]' + + Required arguments + + :param title: The event title. Limited to 100 characters. + :param text: The body of the event. Limited to 4000 characters. The text + supports markdown. + + Optional arguments + + :param date_happened: POSIX timestamp of the event. + :param priority: The priority of the event ('normal' or 'low'). + :param host: Host name to associate with the event. + :param tags: A list of tags to apply to the event. + :param alert_type: "error", "warning", "info" or "success". + :param aggregation_key: An arbitrary string to use for aggregation, + max length of 100 characters. + :param source_type_name: The type of event being posted. + ''' + _initialize_connection(api_key, app_key) + if title is None: + raise SaltInvocationError('title must be specified') + if text is None: + raise SaltInvocationError('text must be specified') + if alert_type not in [None, 'error', 'warning', 'info', 'success']: + # Datadog only supports these alert types but the API doesn't return an + # error for an incorrect alert_type, so we can do it here for now. + # https://github.com/DataDog/datadogpy/issues/215 + message = ('alert_type must be one of "error", "warning", "info", or ' + '"success"') + raise SaltInvocationError(message) + + ret = {'result': False, + 'response': None, + 'comment': ''} + + try: + response = datadog.api.Event.create(title=title, + text=text, + date_happened=date_happened, + priority=priority, + host=host, + tags=tags, + alert_type=alert_type, + aggregation_key=aggregation_key, + source_type_name=source_type_name + ) + except ValueError: + comment = ('Unexpected exception in Datadog Post Event API ' + 'call. Are your keys correct?') + ret['comment'] = comment + return ret + + ret['response'] = response + if 'status' in response.keys(): + ret['result'] = True + ret['comment'] = 'Successfully sent event' + else: + ret['comment'] = 'Error in posting event.' + return ret diff --git a/salt/modules/ddns.py b/salt/modules/ddns.py index 0832851a9bd..3cc55e25e77 100644 --- a/salt/modules/ddns.py +++ b/salt/modules/ddns.py @@ -39,7 +39,7 @@ try: except ImportError as e: dns_support = False -import salt.utils +import salt.utils.files def __virtual__(): @@ -70,7 +70,7 @@ def _config(name, key=None, **kwargs): def _get_keyring(keyfile): keyring = None if keyfile: - with salt.utils.fopen(keyfile) as _f: + with salt.utils.files.fopen(keyfile) as _f: keyring = dns.tsigkeyring.from_text(json.load(_f)) return keyring diff --git a/salt/modules/deb_apache.py b/salt/modules/deb_apache.py index 191d8e40807..2321b9cdcc0 100644 --- a/salt/modules/deb_apache.py +++ b/salt/modules/deb_apache.py @@ -13,7 +13,8 @@ import os import logging # Import salt libs -import salt.utils +import salt.utils.decorators.path +import salt.utils.path log = logging.getLogger(__name__) @@ -27,7 +28,7 @@ def __virtual__(): Only load the module if apache is installed ''' cmd = _detect_os() - if salt.utils.which(cmd) and __grains__['os_family'] == 'Debian': + if salt.utils.path.which(cmd) and __grains__['os_family'] == 'Debian': return __virtualname__ return (False, 'apache execution module not loaded: apache not installed.') @@ -253,7 +254,7 @@ def check_conf_enabled(conf): return os.path.islink('/etc/apache2/conf-enabled/{0}'.format(conf_file)) -@salt.utils.decorators.which('a2enconf') +@salt.utils.decorators.path.which('a2enconf') def a2enconf(conf): ''' .. versionadded:: 2016.3.0 @@ -290,7 +291,7 @@ def a2enconf(conf): return ret -@salt.utils.decorators.which('a2disconf') +@salt.utils.decorators.path.which('a2disconf') def a2disconf(conf): ''' .. versionadded:: 2016.3.0 diff --git a/salt/modules/deb_postgres.py b/salt/modules/deb_postgres.py index 0f6976b6b3c..d77ea9d2361 100644 --- a/salt/modules/deb_postgres.py +++ b/salt/modules/deb_postgres.py @@ -10,7 +10,7 @@ import logging import pipes # Import salt libs -import salt.utils +import salt.utils.path # Import 3rd-party libs @@ -23,7 +23,7 @@ def __virtual__(): ''' Only load this module if the pg_createcluster bin exists ''' - if salt.utils.which('pg_createcluster'): + if salt.utils.path.which('pg_createcluster'): return __virtualname__ return (False, 'postgres execution module not loaded: pg_createcluste command not found.') @@ -52,7 +52,7 @@ def cluster_create(version, salt '*' postgres.cluster_create '9.3' locale='fr_FR' ''' - cmd = [salt.utils.which('pg_createcluster')] + cmd = [salt.utils.path.which('pg_createcluster')] if port: cmd += ['--port', str(port)] if locale: @@ -83,7 +83,7 @@ def cluster_list(verbose=False): salt '*' postgres.cluster_list verbose=True ''' - cmd = [salt.utils.which('pg_lsclusters'), '--no-header'] + cmd = [salt.utils.path.which('pg_lsclusters'), '--no-header'] ret = __salt__['cmd.run_all'](' '.join([pipes.quote(c) for c in cmd])) if ret.get('retcode', 0) != 0: log.error('Error listing clusters') @@ -127,7 +127,7 @@ def cluster_remove(version, salt '*' postgres.cluster_remove '9.3' 'main' stop=True ''' - cmd = [salt.utils.which('pg_dropcluster')] + cmd = [salt.utils.path.which('pg_dropcluster')] if stop: cmd += ['--stop'] cmd += [version, name] diff --git a/salt/modules/debbuild.py b/salt/modules/debbuild.py index 2cb9deed2a7..37c7da1123d 100644 --- a/salt/modules/debbuild.py +++ b/salt/modules/debbuild.py @@ -22,9 +22,14 @@ import time import traceback # Import salt libs -from salt.ext.six.moves.urllib.parse import urlparse as _urlparse # pylint: disable=no-name-in-module,import-error +import salt.utils.files +import salt.utils.path +import salt.utils.vt from salt.exceptions import SaltInvocationError, CommandExecutionError -import salt.utils + +# Import 3rd-party libs +from salt.ext import six +from salt.ext.six.moves.urllib.parse import urlparse as _urlparse # pylint: disable=no-name-in-module,import-error HAS_LIBS = False @@ -49,7 +54,7 @@ def __virtual__(): missing_util = False utils_reqd = ['gpg', 'debuild', 'pbuilder', 'reprepro'] for named_util in utils_reqd: - if not salt.utils.which(named_util): + if not salt.utils.path.which(named_util): missing_util = True break if HAS_LIBS and not missing_util: @@ -60,17 +65,22 @@ def __virtual__(): return (False, 'The debbuild module could not be loaded: unsupported OS family') -def _check_repo_sign_utils_support(): - util_name = 'debsign' - if salt.utils.which(util_name): +def _check_repo_sign_utils_support(name): + ''' + Check for specified command name in search path + ''' + if salt.utils.path.which(name): return True else: raise CommandExecutionError( - 'utility \'{0}\' needs to be installed'.format(util_name) + 'utility \'{0}\' needs to be installed or made available in search path'.format(name) ) def _check_repo_gpg_phrase_utils_support(): + ''' + Check for /usr/lib/gnupg2/gpg-preset-passphrase is installed + ''' util_name = '/usr/lib/gnupg2/gpg-preset-passphrase' if __salt__['file.file_exists'](util_name): return True @@ -170,8 +180,8 @@ def _get_repo_dists_env(env): 'ORIGIN': ('O', 'Origin', 'SaltStack'), 'LABEL': ('O', 'Label', 'salt_debian'), 'SUITE': ('O', 'Suite', 'stable'), - 'VERSION': ('O', 'Version', '8.1'), - 'CODENAME': ('M', 'Codename', 'jessie'), + 'VERSION': ('O', 'Version', '9.0'), + 'CODENAME': ('M', 'Codename', 'stretch'), 'ARCHS': ('M', 'Architectures', 'i386 amd64 source'), 'COMPONENTS': ('M', 'Components', 'main'), 'DESCRIPTION': ('O', 'Description', 'SaltStack debian package repo'), @@ -205,7 +215,7 @@ def _get_repo_dists_env(env): else: env_dists += '{0}: {1}\n'.format(key, value) - ## ensure mandatories are included + # ensure mandatories are included env_keys = list(env.keys()) for key in env_keys: if key in dflts_keys and dflts_dict[key][0] == 'M' and key not in env_man_seen: @@ -247,7 +257,7 @@ def _create_pbuilders(env): env_overrides = _get_build_env(env) if env_overrides and not env_overrides.isspace(): - with salt.utils.fopen(pbuilderrc, 'a') as fow: + with salt.utils.files.fopen(pbuilderrc, 'a') as fow: fow.write('{0}'.format(env_overrides)) @@ -307,12 +317,12 @@ def make_src_pkg(dest_dir, spec, sources, env=None, template=None, saltenv='base spec_pathfile = _get_spec(tree_base, spec, template, saltenv) # build salt equivalents from scratch - if isinstance(sources, str): + if isinstance(sources, six.string_types): sources = sources.split(',') for src in sources: _get_src(tree_base, src, saltenv) - #.dsc then assumes sources already build + # .dsc then assumes sources already build if spec_pathfile.endswith('.dsc'): for efile in os.listdir(tree_base): full = os.path.join(tree_base, efile) @@ -570,15 +580,16 @@ def make_repo(repodir, codename, repocfg_dists = _get_repo_dists_env(env) repoconfdist = os.path.join(repoconf, 'distributions') - with salt.utils.fopen(repoconfdist, 'w') as fow: + with salt.utils.files.fopen(repoconfdist, 'w') as fow: fow.write('{0}'.format(repocfg_dists)) repocfg_opts = _get_repo_options_env(env) repoconfopts = os.path.join(repoconf, 'options') - with salt.utils.fopen(repoconfopts, 'w') as fow: + with salt.utils.files.fopen(repoconfopts, 'w') as fow: fow.write('{0}'.format(repocfg_opts)) - local_fingerprint = None + local_keygrip_to_use = None + local_key_fingerprint = None local_keyid = None phrase = '' @@ -587,17 +598,14 @@ def make_repo(repodir, gpg_tty_info_file = '{0}/gpg-tty-info-salt'.format(gnupghome) gpg_tty_info_dict = {} - # test if using older than gnupg 2.1, env file exists + # if using older than gnupg 2.1, then env file exists older_gnupg = __salt__['file.file_exists'](gpg_info_file) - # interval of 0.125 is really too fast on some systems - interval = 0.5 - if keyid is not None: - with salt.utils.fopen(repoconfdist, 'a') as fow: + with salt.utils.files.fopen(repoconfdist, 'a') as fow: fow.write('SignWith: {0}\n'.format(keyid)) - ## import_keys + # import_keys pkg_pub_key_file = '{0}/{1}'.format(gnupghome, __salt__['pillar.get']('gpg_pkg_pub_keyname', None)) pkg_priv_key_file = '{0}/{1}'.format(gnupghome, __salt__['pillar.get']('gpg_pkg_priv_keyname', None)) @@ -621,24 +629,40 @@ def make_repo(repodir, local_keys = __salt__['gpg.list_keys'](user=runas, gnupghome=gnupghome) for gpg_key in local_keys: if keyid == gpg_key['keyid'][8:]: - local_fingerprint = gpg_key['fingerprint'] + local_keygrip_to_use = gpg_key['fingerprint'] + local_key_fingerprint = gpg_key['fingerprint'] local_keyid = gpg_key['keyid'] break + if not older_gnupg: + _check_repo_sign_utils_support('gpg2') + cmd = '{0} --with-keygrip --list-secret-keys'.format(salt.utils.path.which('gpg2')) + local_keys2_keygrip = __salt__['cmd.run'](cmd, runas=runas) + local_keys2 = iter(local_keys2_keygrip.splitlines()) + try: + for line in local_keys2: + if line.startswith('sec'): + line_fingerprint = next(local_keys2).lstrip().rstrip() + if local_key_fingerprint == line_fingerprint: + lkeygrip = next(local_keys2).split('=') + local_keygrip_to_use = lkeygrip[1].lstrip().rstrip() + break + except StopIteration: + raise SaltInvocationError( + 'unable to find keygrip associated with fingerprint \'{0}\' for keyid \'{1}\'' + .format(local_key_fingerprint, local_keyid) + ) + if local_keyid is None: raise SaltInvocationError( 'The key ID \'{0}\' was not found in GnuPG keyring at \'{1}\'' .format(keyid, gnupghome) ) - _check_repo_sign_utils_support() - - if use_passphrase: - _check_repo_gpg_phrase_utils_support() - phrase = __salt__['pillar.get']('gpg_passphrase') + _check_repo_sign_utils_support('debsign') if older_gnupg: - with salt.utils.fopen(gpg_info_file, 'r') as fow: + with salt.utils.files.fopen(gpg_info_file, 'r') as fow: gpg_raw_info = fow.readlines() for gpg_info_line in gpg_raw_info: @@ -647,7 +671,7 @@ def make_repo(repodir, __salt__['environ.setenv'](gpg_info_dict) break else: - with salt.utils.fopen(gpg_tty_info_file, 'r') as fow: + with salt.utils.files.fopen(gpg_tty_info_file, 'r') as fow: gpg_raw_info = fow.readlines() for gpg_tty_info_line in gpg_raw_info: @@ -656,10 +680,30 @@ def make_repo(repodir, __salt__['environ.setenv'](gpg_tty_info_dict) break - ## sign_it_here - for file in os.listdir(repodir): - if file.endswith('.dsc'): - abs_file = os.path.join(repodir, file) + if use_passphrase: + _check_repo_gpg_phrase_utils_support() + phrase = __salt__['pillar.get']('gpg_passphrase') + cmd = '/usr/lib/gnupg2/gpg-preset-passphrase --verbose --preset --passphrase "{0}" {1}'.format(phrase, local_keygrip_to_use) + __salt__['cmd.run'](cmd, runas=runas) + + for debfile in os.listdir(repodir): + abs_file = os.path.join(repodir, debfile) + if debfile.endswith('.changes'): + os.remove(abs_file) + + if debfile.endswith('.dsc'): + # sign_it_here + if older_gnupg: + if local_keyid is not None: + cmd = 'debsign --re-sign -k {0} {1}'.format(keyid, abs_file) + __salt__['cmd.run'](cmd, cwd=repodir, use_vt=True) + + cmd = 'reprepro --ignore=wrongdistribution --component=main -Vb . includedsc {0} {1}'.format(codename, abs_file) + __salt__['cmd.run'](cmd, cwd=repodir, use_vt=True) + else: + # interval of 0.125 is really too fast on some systems + interval = 0.5 + if local_keyid is not None: number_retries = timeout / interval times_looped = 0 error_msg = 'Failed to debsign file {0}'.format(abs_file) @@ -702,27 +746,6 @@ def make_repo(repodir, finally: proc.close(terminate=True, kill=True) - if use_passphrase: - cmd = '/usr/lib/gnupg2/gpg-preset-passphrase --verbose --forget {0}'.format(local_fingerprint) - __salt__['cmd.run'](cmd, runas=runas) - - cmd = '/usr/lib/gnupg2/gpg-preset-passphrase --verbose --preset --passphrase "{0}" {1}'.format(phrase, local_fingerprint) - __salt__['cmd.run'](cmd, runas=runas) - - for debfile in os.listdir(repodir): - abs_file = os.path.join(repodir, debfile) - if debfile.endswith('.changes'): - os.remove(abs_file) - - if debfile.endswith('.dsc'): - if older_gnupg: - if local_keyid is not None: - cmd = 'debsign --re-sign -k {0} {1}'.format(keyid, abs_file) - __salt__['cmd.run'](cmd, cwd=repodir, use_vt=True) - - cmd = 'reprepro --ignore=wrongdistribution --component=main -Vb . includedsc {0} {1}'.format(codename, abs_file) - __salt__['cmd.run'](cmd, cwd=repodir, use_vt=True) - else: number_retries = timeout / interval times_looped = 0 error_msg = 'Failed to reprepro includedsc file {0}'.format(abs_file) @@ -747,8 +770,7 @@ def make_repo(repodir, if times_looped > number_retries: raise SaltInvocationError( - 'Attemping to reprepro includedsc for file {0} failed, timed out after {1} loops' - .format(abs_file, int(times_looped * interval)) + 'Attemping to reprepro includedsc for file {0} failed, timed out after {1} loops'.format(abs_file, times_looped) ) time.sleep(interval) @@ -770,8 +792,4 @@ def make_repo(repodir, cmd = 'reprepro --ignore=wrongdistribution --component=main -Vb . includedeb {0} {1}'.format(codename, abs_file) res = __salt__['cmd.run_all'](cmd, cwd=repodir, use_vt=True) - if use_passphrase and local_keyid is not None: - cmd = '/usr/lib/gnupg2/gpg-preset-passphrase --forget {0}'.format(local_fingerprint) - res = __salt__['cmd.run_all'](cmd, runas=runas) - return res diff --git a/salt/modules/debconfmod.py b/salt/modules/debconfmod.py index 98fd1aaaa49..18e19d1cce1 100644 --- a/salt/modules/debconfmod.py +++ b/salt/modules/debconfmod.py @@ -11,7 +11,9 @@ import re # Import salt libs import salt.utils +import salt.utils.path import salt.utils.files +import salt.utils.versions log = logging.getLogger(__name__) @@ -32,7 +34,7 @@ def __virtual__(): return (False, 'The debconfmod module could not be loaded: ' 'unsupported OS family') - if salt.utils.which('debconf-get-selections') is None: + if salt.utils.path.which('debconf-get-selections') is None: return (False, 'The debconfmod module could not be loaded: ' 'debconf-utils is not installed.') @@ -184,12 +186,7 @@ def set_file(path, saltenv='base', **kwargs): salt '*' debconf.set_file salt://pathto/pkg.selections ''' if '__env__' in kwargs: - salt.utils.warn_until( - 'Oxygen', - 'Parameter \'__env__\' has been detected in the argument list. This ' - 'parameter is no longer used and has been replaced by \'saltenv\' ' - 'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.' - ) + # "env" is not supported; Use "saltenv". kwargs.pop('__env__') path = __salt__['cp.cache_file'](path, saltenv) diff --git a/salt/modules/debian_ip.py b/salt/modules/debian_ip.py index 782f97cf64d..6b5c760c924 100644 --- a/salt/modules/debian_ip.py +++ b/salt/modules/debian_ip.py @@ -19,14 +19,15 @@ import time # Import third party libs import jinja2 import jinja2.exceptions -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import StringIO # pylint: disable=import-error,no-name-in-module # Import salt libs -import salt.utils +import salt.utils.files +import salt.utils.odict +import salt.utils.stringutils import salt.utils.templates import salt.utils.validate.net -import salt.utils.odict # Set up logging @@ -219,8 +220,8 @@ def _read_file(path): Reads and returns the contents of a text file ''' try: - with salt.utils.flopen(path, 'rb') as contents: - return [salt.utils.to_str(line) for line in contents.readlines()] + with salt.utils.files.flopen(path, 'rb') as contents: + return [salt.utils.stringutils.to_str(line) for line in contents.readlines()] except (OSError, IOError): return '' @@ -280,7 +281,7 @@ def _parse_current_network_settings(): opts['networking'] = '' if os.path.isfile(_DEB_NETWORKING_FILE): - with salt.utils.fopen(_DEB_NETWORKING_FILE) as contents: + with salt.utils.files.fopen(_DEB_NETWORKING_FILE) as contents: for line in contents: if line.startswith('#'): continue @@ -574,7 +575,7 @@ def _parse_interfaces(interface_files=None): method = -1 for interface_file in interface_files: - with salt.utils.fopen(interface_file) as interfaces: + with salt.utils.files.fopen(interface_file) as interfaces: # This ensures iface_dict exists, but does not ensure we're not reading a new interface. iface_dict = {} for line in interfaces: @@ -1489,7 +1490,7 @@ def _write_file(iface, data, folder, pattern): msg = msg.format(filename, folder) log.error(msg) raise AttributeError(msg) - with salt.utils.flopen(filename, 'w') as fout: + with salt.utils.files.flopen(filename, 'w') as fout: fout.write(data) return filename @@ -1514,7 +1515,7 @@ def _write_file_routes(iface, data, folder, pattern): msg = msg.format(filename, folder) log.error(msg) raise AttributeError(msg) - with salt.utils.flopen(filename, 'w') as fout: + with salt.utils.files.flopen(filename, 'w') as fout: fout.write(data) __salt__['file.set_mode'](filename, '0755') @@ -1533,7 +1534,7 @@ def _write_file_network(data, filename, create=False): msg = msg.format(filename) log.error(msg) raise AttributeError(msg) - with salt.utils.flopen(filename, 'w') as fout: + with salt.utils.files.flopen(filename, 'w') as fout: fout.write(data) @@ -1609,7 +1610,7 @@ def _write_file_ifaces(iface, data, **settings): msg = msg.format(os.path.dirname(filename)) log.error(msg) raise AttributeError(msg) - with salt.utils.flopen(filename, 'w') as fout: + with salt.utils.files.flopen(filename, 'w') as fout: if _SEPARATE_FILE: fout.write(saved_ifcfg) else: @@ -1642,7 +1643,7 @@ def _write_file_ppp_ifaces(iface, data): msg = msg.format(os.path.dirname(filename)) log.error(msg) raise AttributeError(msg) - with salt.utils.fopen(filename, 'w') as fout: + with salt.utils.files.fopen(filename, 'w') as fout: fout.write(ifcfg) # Return as a array so the difflib works @@ -2037,19 +2038,12 @@ def build_network_settings(**settings): # Write settings _write_file_network(network, _DEB_NETWORKING_FILE, True) - # Write hostname to /etc/hostname + # Get hostname and domain from opts sline = opts['hostname'].split('.', 1) opts['hostname'] = sline[0] - hostname = '{0}\n' . format(opts['hostname']) current_domainname = current_network_settings['domainname'] current_searchdomain = current_network_settings['searchdomain'] - # Only write the hostname if it has changed - if not opts['hostname'] == current_network_settings['hostname']: - if not ('test' in settings and settings['test']): - # TODO replace wiht a call to network.mod_hostname instead - _write_file_network(hostname, _DEB_HOSTNAME_FILE) - new_domain = False if len(sline) > 1: new_domainname = sline[1] diff --git a/salt/modules/defaults.py b/salt/modules/defaults.py index 27ba5f86208..8056aef431d 100644 --- a/salt/modules/defaults.py +++ b/salt/modules/defaults.py @@ -7,10 +7,10 @@ import yaml import salt.fileclient import salt.utils +import salt.utils.dictupdate as dictupdate +import salt.utils.files import salt.utils.url -from salt.utils import dictupdate - __virtualname__ = 'defaults' @@ -60,7 +60,7 @@ def _load(formula): if os.path.exists(file_): log.debug("Reading defaults from %r", file_) - with salt.utils.fopen(file_) as fhr: + with salt.utils.files.fopen(file_) as fhr: defaults = loader.load(fhr) log.debug("Read defaults %r", defaults) diff --git a/salt/modules/dig.py b/salt/modules/dig.py index 397bf6538ba..e30e9d57c48 100644 --- a/salt/modules/dig.py +++ b/salt/modules/dig.py @@ -6,8 +6,8 @@ The 'dig' command line tool must be installed in order to use this module. from __future__ import absolute_import # Import salt libs -import salt.utils import salt.utils.network +import salt.utils.path # Import python libs import logging @@ -22,7 +22,7 @@ def __virtual__(): ''' Only load module if dig binary is present ''' - if salt.utils.which('dig'): + if salt.utils.path.which('dig'): return __virtualname__ return (False, 'The dig execution module cannot be loaded: ' 'the dig binary is not in the path.') diff --git a/salt/modules/disk.py b/salt/modules/disk.py index 968a9e9b7aa..28ddfd377b6 100644 --- a/salt/modules/disk.py +++ b/salt/modules/disk.py @@ -17,22 +17,27 @@ from salt.ext import six from salt.ext.six.moves import zip # Import salt libs -import salt.utils -import salt.utils.decorators as decorators -from salt.utils.decorators import depends +import salt.utils.decorators +import salt.utils.decorators.path +import salt.utils.path +import salt.utils.platform from salt.exceptions import CommandExecutionError +__func_alias__ = { + 'format_': 'format' +} + log = logging.getLogger(__name__) -HAS_HDPARM = salt.utils.which('hdparm') is not None -HAS_IOSTAT = salt.utils.which('iostat') is not None +HAS_HDPARM = salt.utils.path.which('hdparm') is not None +HAS_IOSTAT = salt.utils.path.which('iostat') is not None def __virtual__(): ''' Only work on POSIX-like systems ''' - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): return False, 'This module doesn\'t work on Windows.' return True @@ -253,7 +258,7 @@ def percent(args=None): return ret -@decorators.which('blkid') +@salt.utils.decorators.path.which('blkid') def blkid(device=None): ''' Return block device attributes: UUID, LABEL, etc. This function only works @@ -402,8 +407,8 @@ def resize2fs(device): return True -@decorators.which('sync') -@decorators.which('mkfs') +@salt.utils.decorators.path.which('sync') +@salt.utils.decorators.path.which('mkfs') def format_(device, fs_type='ext4', inode_size=None, @@ -470,7 +475,7 @@ def format_(device, return all([mkfs_success, sync_success]) -@decorators.which_bin(['lsblk', 'df']) +@salt.utils.decorators.path.which_bin(['lsblk', 'df']) def fstype(device): ''' Return the filesystem name of the specified device @@ -486,14 +491,14 @@ def fstype(device): salt '*' disk.fstype /dev/sdX1 ''' - if salt.utils.which('lsblk'): + if salt.utils.path.which('lsblk'): lsblk_out = __salt__['cmd.run']('lsblk -o fstype {0}'.format(device)).splitlines() if len(lsblk_out) > 1: fs_type = lsblk_out[1].strip() if fs_type: return fs_type - if salt.utils.which('df'): + if salt.utils.path.which('df'): # the fstype was not set on the block device, so inspect the filesystem # itself for its type if __grains__['kernel'] == 'AIX' and os.path.isfile('/usr/sysv/bin/df'): @@ -512,7 +517,7 @@ def fstype(device): return '' -@depends(HAS_HDPARM) +@salt.utils.decorators.depends(HAS_HDPARM) def _hdparm(args, failhard=True): ''' Execute hdparm @@ -531,7 +536,7 @@ def _hdparm(args, failhard=True): return result['stdout'] -@depends(HAS_HDPARM) +@salt.utils.decorators.depends(HAS_HDPARM) def hdparms(disks, args=None): ''' Retrieve all info's for all disks @@ -608,7 +613,7 @@ def hdparms(disks, args=None): return out -@depends(HAS_HDPARM) +@salt.utils.decorators.depends(HAS_HDPARM) def hpa(disks, size=None): ''' Get/set Host Protected Area settings @@ -737,7 +742,7 @@ def smart_attributes(dev, attributes=None, values=None): return smart_attr -@depends(HAS_IOSTAT) +@salt.utils.decorators.depends(HAS_IOSTAT) def iostat(interval=1, count=5, disks=None): ''' Gather and return (averaged) IO stats. @@ -753,11 +758,11 @@ def iostat(interval=1, count=5, disks=None): salt '*' disk.iostat 1 5 disks=sda ''' - if salt.utils.is_linux(): + if salt.utils.platform.is_linux(): return _iostat_linux(interval, count, disks) - elif salt.utils.is_freebsd(): + elif salt.utils.platform.is_freebsd(): return _iostat_fbsd(interval, count, disks) - elif salt.utils.is_aix(): + elif salt.utils.platform.is_aix(): return _iostat_aix(interval, count, disks) diff --git a/salt/modules/djangomod.py b/salt/modules/djangomod.py index 76cb536d91e..e74d263dc42 100644 --- a/salt/modules/djangomod.py +++ b/salt/modules/djangomod.py @@ -9,11 +9,11 @@ from __future__ import absolute_import import os # Import Salt libs -import salt.utils +import salt.utils.path import salt.exceptions # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six # Define the module's virtual name __virtualname__ = 'django' @@ -28,9 +28,9 @@ def _get_django_admin(bin_env): Return the django admin ''' if not bin_env: - if salt.utils.which('django-admin.py'): + if salt.utils.path.which('django-admin.py'): return 'django-admin.py' - elif salt.utils.which('django-admin'): + elif salt.utils.path.which('django-admin'): return 'django-admin' else: raise salt.exceptions.CommandExecutionError( diff --git a/salt/modules/dnsmasq.py b/salt/modules/dnsmasq.py index db126bde139..b62fcce90bb 100644 --- a/salt/modules/dnsmasq.py +++ b/salt/modules/dnsmasq.py @@ -9,9 +9,13 @@ import logging import os # Import salt libs -import salt.utils +import salt.utils.files +import salt.utils.platform from salt.exceptions import CommandExecutionError +# Import 3rd-party libs +from salt.ext import six + log = logging.getLogger(__name__) @@ -19,8 +23,12 @@ def __virtual__(): ''' Only work on POSIX-like systems. ''' - if salt.utils.is_windows(): - return (False, 'dnsmasq execution module cannot be loaded: only works on non-Windows systems.') + if salt.utils.platform.is_windows(): + return ( + False, + 'dnsmasq execution module cannot be loaded: only works on ' + 'non-Windows systems.' + ) return True @@ -109,7 +117,7 @@ def set_config(config_file='/etc/dnsmasq.conf', follow=True, **kwargs): ret_kwargs[key] = kwargs[key] if key in dnsopts: - if isinstance(dnsopts[key], str): + if isinstance(dnsopts[key], six.string_types): for config in includes: __salt__['file.sed'](path=config, before='^{0}=.*'.format(key), @@ -163,7 +171,7 @@ def _parse_dnamasq(filename): 'Error: No such file \'{0}\''.format(filename) ) - with salt.utils.fopen(filename, 'r') as fp_: + with salt.utils.files.fopen(filename, 'r') as fp_: for line in fp_: if not line.strip(): continue @@ -172,7 +180,7 @@ def _parse_dnamasq(filename): if '=' in line: comps = line.split('=') if comps[0] in fileopts: - if isinstance(fileopts[comps[0]], str): + if isinstance(fileopts[comps[0]], six.string_types): temp = fileopts[comps[0]] fileopts[comps[0]] = [temp] fileopts[comps[0]].append(comps[1].strip()) diff --git a/salt/modules/dnsutil.py b/salt/modules/dnsutil.py index 36b1469a4bb..f05e15f9039 100644 --- a/salt/modules/dnsutil.py +++ b/salt/modules/dnsutil.py @@ -1,16 +1,20 @@ # -*- coding: utf-8 -*- ''' -Compendium of generic DNS utilities +Compendium of generic DNS utilities. + +.. note:: + + Some functions in the ``dnsutil`` execution module depend on ``dig``. ''' +# Import python libs from __future__ import absolute_import +import logging +import socket +import time # Import salt libs -import salt.utils -import socket - -# Import python libs -import logging -import time +import salt.utils.files +import salt.utils.path log = logging.getLogger(__name__) @@ -35,7 +39,7 @@ def parse_hosts(hostsfile='/etc/hosts', hosts=None): ''' if not hosts: try: - with salt.utils.fopen(hostsfile, 'r') as fp_: + with salt.utils.files.fopen(hostsfile, 'r') as fp_: hosts = fp_.read() except Exception: return 'Error: hosts data was not found' @@ -75,7 +79,7 @@ def hosts_append(hostsfile='/etc/hosts', ip_addr=None, entries=None): return 'No additional hosts were added to {0}'.format(hostsfile) append_line = '\n{0} {1}'.format(ip_addr, ' '.join(host_list)) - with salt.utils.fopen(hostsfile, 'a') as fp_: + with salt.utils.files.fopen(hostsfile, 'a') as fp_: fp_.write(append_line) return 'The following line was added to {0}:{1}'.format(hostsfile, @@ -95,11 +99,11 @@ def hosts_remove(hostsfile='/etc/hosts', entries=None): salt '*' dnsutil.hosts_remove /etc/hosts ad1.yuk.co salt '*' dnsutil.hosts_remove /etc/hosts ad2.yuk.co,ad1.yuk.co ''' - with salt.utils.fopen(hostsfile, 'r') as fp_: + with salt.utils.files.fopen(hostsfile, 'r') as fp_: hosts = fp_.read() host_list = entries.split(',') - with salt.utils.fopen(hostsfile, 'w') as out_file: + with salt.utils.files.fopen(hostsfile, 'w') as out_file: for line in hosts.splitlines(): if not line or line.strip().startswith('#'): out_file.write('{0}\n'.format(line)) @@ -125,7 +129,7 @@ def parse_zone(zonefile=None, zone=None): ''' if zonefile: try: - with salt.utils.fopen(zonefile, 'r') as fp_: + with salt.utils.files.fopen(zonefile, 'r') as fp_: zone = fp_.read() except Exception: pass @@ -221,7 +225,7 @@ def _has_dig(): because they are also DNS utilities, a compatibility layer exists. This function helps add that layer. ''' - return salt.utils.which('dig') is not None + return salt.utils.path.which('dig') is not None def check_ip(ip_addr): @@ -232,7 +236,7 @@ def check_ip(ip_addr): .. code-block:: bash - salt ns1 dig.check_ip 127.0.0.1 + salt ns1 dnsutil.check_ip 127.0.0.1 ''' if _has_dig(): return __salt__['dig.check_ip'](ip_addr) @@ -242,7 +246,7 @@ def check_ip(ip_addr): def A(host, nameserver=None): ''' - Return the A record(s) for `host`. + Return the A record(s) for ``host``. Always returns a list. @@ -267,7 +271,7 @@ def A(host, nameserver=None): def AAAA(host, nameserver=None): ''' - Return the AAAA record(s) for `host`. + Return the AAAA record(s) for ``host``. Always returns a list. @@ -302,7 +306,7 @@ def NS(domain, resolve=True, nameserver=None): .. code-block:: bash - salt ns1 dig.NS google.com + salt ns1 dnsutil.NS google.com ''' if _has_dig(): @@ -323,7 +327,7 @@ def SPF(domain, record='SPF', nameserver=None): .. code-block:: bash - salt ns1 dig.SPF google.com + salt ns1 dnsutil.SPF google.com ''' if _has_dig(): return __salt__['dig.SPF'](domain, record, nameserver) @@ -346,7 +350,7 @@ def MX(domain, resolve=False, nameserver=None): .. code-block:: bash - salt ns1 dig.MX google.com + salt ns1 dnsutil.MX google.com ''' if _has_dig(): return __salt__['dig.MX'](domain, resolve, nameserver) diff --git a/salt/modules/dockercompose.py b/salt/modules/dockercompose.py index f364afa5b37..a73321e9cfd 100644 --- a/salt/modules/dockercompose.py +++ b/salt/modules/dockercompose.py @@ -106,7 +106,7 @@ import inspect import logging import os import re -import salt.utils +import salt.utils.files from operator import attrgetter try: @@ -179,7 +179,7 @@ def __read_docker_compose(path): return __standardize_result(False, 'Path does not exist or docker-compose.yml is not present', None, None) - f = salt.utils.fopen(os.path.join(path, dc_filename), 'r') # pylint: disable=resource-leakage + f = salt.utils.files.fopen(os.path.join(path, dc_filename), 'r') # pylint: disable=resource-leakage result = {'docker-compose.yml': ''} if f: for line in f: @@ -207,7 +207,7 @@ def __write_docker_compose(path, docker_compose): if os.path.isdir(path) is False: os.mkdir(path) - f = salt.utils.fopen(os.path.join(path, dc_filename), 'w') # pylint: disable=resource-leakage + f = salt.utils.files.fopen(os.path.join(path, dc_filename), 'w') # pylint: disable=resource-leakage if f: f.write(docker_compose) f.close() diff --git a/salt/modules/dockermod.py b/salt/modules/dockermod.py index 14c988f413f..5e9ef7d3b42 100644 --- a/salt/modules/dockermod.py +++ b/salt/modules/dockermod.py @@ -83,14 +83,14 @@ in the ``docker-registries`` Pillar key, as well as any key ending in username: foo password: s3cr3t -To login to the configured registries, use the :py:func:`dockerng.login -` function. This only needs to be done once for a +To login to the configured registries, use the :py:func:`docker.login +` function. This only needs to be done once for a given registry, and it will store/update the credentials in ``~/.docker/config.json``. .. note:: - For Salt releases before 2016.3.7 and 2016.11.4, :py:func:`dockerng.login - ` is not available. Instead, Salt will try to + For Salt releases before 2016.3.7 and 2016.11.4, :py:func:`docker.login + ` is not available. Instead, Salt will try to authenticate using each of your configured registries for each push/pull, behavior which is not correct and has been resolved in newer releases. @@ -199,12 +199,15 @@ import subprocess # Import Salt libs from salt.exceptions import CommandExecutionError, SaltInvocationError -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import map # pylint: disable=import-error,redefined-builtin import salt.utils +import salt.utils.args import salt.utils.decorators import salt.utils.docker import salt.utils.files +import salt.utils.path +import salt.utils.stringutils import salt.utils.thin import salt.pillar import salt.exceptions @@ -230,7 +233,8 @@ except ImportError: HAS_LZMA = False # pylint: enable=import-error -HAS_NSENTER = bool(salt.utils.which('nsenter')) +HAS_NSENTER = bool(salt.utils.path.which('nsenter')) +HUB_PREFIX = 'docker.io/' # Set up logging log = logging.getLogger(__name__) @@ -343,7 +347,7 @@ def _get_client(timeout=NOTSET, **kwargs): python_shell=False) try: docker_machine_json = \ - json.loads(salt.utils.to_str(docker_machine_json)) + json.loads(salt.utils.stringutils.to_str(docker_machine_json)) docker_machine_tls = \ docker_machine_json['HostOptions']['AuthOptions'] docker_machine_ip = docker_machine_json['Driver']['IPAddress'] @@ -389,7 +393,7 @@ def _docker_client(wrapped): ''' Ensure that the client is present ''' - kwargs = salt.utils.clean_kwargs(**kwargs) + kwargs = salt.utils.args.clean_kwargs(**kwargs) timeout = kwargs.pop('client_timeout', NOTSET) if 'docker.client' not in __context__ \ or not hasattr(__context__['docker.client'], 'timeout'): @@ -421,7 +425,7 @@ def _ensure_exists(wrapped): raise CommandExecutionError( 'Container \'{0}\' does not exist'.format(name) ) - return wrapped(name, *args, **salt.utils.clean_kwargs(**kwargs)) + return wrapped(name, *args, **salt.utils.args.clean_kwargs(**kwargs)) return wrapper @@ -434,7 +438,7 @@ def _refresh_mine_cache(wrapped): ''' refresh salt mine on exit. ''' - returned = wrapped(*args, **salt.utils.clean_kwargs(**kwargs)) + returned = wrapped(*args, **salt.utils.args.clean_kwargs(**kwargs)) __salt__['mine.send']('docker.ps', verbose=True, all=True, host=True) return returned return wrapper @@ -559,6 +563,21 @@ def _prep_pull(): __context__['docker._pull_status'] = [x[:12] for x in images(all=True)] +def _scrub_links(links, name): + ''' + Remove container name from HostConfig:Links values to enable comparing + container configurations correctly. + ''' + if isinstance(links, list): + ret = [] + for l in links: + ret.append(l.replace('/{0}/'.format(name), '/', 1)) + else: + ret = links + + return ret + + def _size_fmt(num): ''' Format bytes as human-readable file sizes @@ -619,7 +638,7 @@ def _client_wrapper(attr, *args, **kwargs): api_events = [] try: for event in ret: - api_events.append(json.loads(salt.utils.to_str(event))) + api_events.append(json.loads(salt.utils.stringutils.to_str(event))) except Exception as exc: raise CommandExecutionError( 'Unable to interpret API event: \'{0}\''.format(event), @@ -783,7 +802,7 @@ def get_client_args(): salt myminion docker.get_client_args ''' - return salt.utils.docker.get_client_args() + return __utils__['docker.get_client_args']() def _get_create_kwargs(image, @@ -884,8 +903,20 @@ def compare_container(first, second, ignore=None): continue val1 = result1[conf_dict][item] val2 = result2[conf_dict].get(item) - if val1 != val2: - ret.setdefault(conf_dict, {})[item] = {'old': val1, 'new': val2} + if item in ('OomKillDisable',) or (val1 is None or val2 is None): + if bool(val1) != bool(val2): + ret.setdefault(conf_dict, {})[item] = {'old': val1, 'new': val2} + elif item == 'Image': + image1 = inspect_image(val1)['Id'] + image2 = inspect_image(val2)['Id'] + if image1 != image2: + ret.setdefault(conf_dict, {})[item] = {'old': image1, 'new': image2} + else: + if item == 'Links': + val1 = _scrub_links(val1, first) + val2 = _scrub_links(val2, second) + if val1 != val2: + ret.setdefault(conf_dict, {})[item] = {'old': val1, 'new': val2} # Check for optionally-present items that were in the second container # and not the first. for item in result2[conf_dict]: @@ -895,8 +926,20 @@ def compare_container(first, second, ignore=None): continue val1 = result1[conf_dict].get(item) val2 = result2[conf_dict][item] - if val1 != val2: - ret.setdefault(conf_dict, {})[item] = {'old': val1, 'new': val2} + if item in ('OomKillDisable',) or (val1 is None or val2 is None): + if bool(val1) != bool(val2): + ret.setdefault(conf_dict, {})[item] = {'old': val1, 'new': val2} + elif item == 'Image': + image1 = inspect_image(val1)['Id'] + image2 = inspect_image(val2)['Id'] + if image1 != image2: + ret.setdefault(conf_dict, {})[item] = {'old': image1, 'new': image2} + else: + if item == 'Links': + val1 = _scrub_links(val1, first) + val2 = _scrub_links(val2, second) + if val1 != val2: + ret.setdefault(conf_dict, {})[item] = {'old': val1, 'new': val2} return ret @@ -927,9 +970,9 @@ def login(*registries): .. code-block:: bash - salt myminion dockerng.login - salt myminion dockerng.login hub - salt myminion dockerng.login hub https://mydomain.tld/registry/ + salt myminion docker.login + salt myminion docker.login hub + salt myminion docker.login hub https://mydomain.tld/registry/ ''' # NOTE: This function uses the "docker login" CLI command so that login # information is added to the config.json, since docker-py isn't designed @@ -978,6 +1021,10 @@ def login(*registries): cmd = ['docker', 'login', '-u', username, '-p', password] if registry.lower() != 'hub': cmd.append(registry) + log.debug( + 'Attempting to login to docker registry \'%s\' as user \'%s\'', + registry, username + ) login_cmd = __salt__['cmd.run_all']( cmd, python_shell=False, @@ -1443,6 +1490,43 @@ def list_tags(): return sorted(ret) +def resolve_tag(name, tags=None): + ''' + .. versionadded:: 2017.7.2,Oxygen + + Given an image tag, check the locally-pulled tags (using + :py:func:`docker.list_tags `) and return + the matching tag. This helps disambiguate differences on some platforms + where images from the Docker Hub are prefixed with ``docker.io/``. If an + image name with no tag is passed, a tag of ``latest`` is assumed. + + If the specified image is not pulled locally, this function will return + ``False``. + + tags + An optional Python list of tags to check against. If passed, then + :py:func:`docker.list_tags ` will not + be run to get a list of tags. This is useful when resolving a number of + tags at the same time. + + CLI Examples: + + .. code-block:: bash + + salt myminion docker.resolve_tag busybox + salt myminion docker.resolve_tag busybox:latest + ''' + tag_name = ':'.join(salt.utils.docker.get_repo_tag(name)) + if tags is None: + tags = list_tags() + if tag_name in tags: + return tag_name + full_name = HUB_PREFIX + tag_name + if not name.startswith(HUB_PREFIX) and full_name in tags: + return full_name + return False + + def logs(name): ''' Returns the logs for the container. Equivalent to running the ``docker @@ -1803,7 +1887,7 @@ def create(image, generate one for you (it will be included in the return data). skip_translate - This function translates Salt CLI input into the format which + This function translates Salt CLI or SLS input into the format which docker-py_ expects. However, in the event that Salt's translation logic fails (due to potential changes in the Docker Remote API, or to bugs in the translation code), this argument can be used to exert granular @@ -1867,19 +1951,49 @@ def create(image, binds Files/directories to bind mount. Each bind mount should be passed in - the format ``::``, where - ```` is one of ``rw`` (for read-write access) or ``ro`` (for - read-only access). Optionally, the read-only information can be left - off the end and the bind mount will be assumed to be read-write. + one of the following formats: + + - ``:`` - ``host_path`` is mounted within + the container as ``container_path`` with read-write access. + - ``::`` - ``host_path`` is + mounted within the container as ``container_path`` with read-write + access. Additionally, the specified selinux context will be set + within the container. + - ``::`` - ``host_path`` is + mounted within the container as ``container_path``, with the + read-only or read-write setting explicitly defined. + - ``::,`` - + ``host_path`` is mounted within the container as ``container_path``, + with the read-only or read-write setting explicitly defined. + Additionally, the specified selinux context will be set within the + container. + + ```` can be either ``ro`` for read-write access, or ``ro`` + for read-only access. When omitted, it is assumed to be read-write. + + ```` can be ``z`` if the volume is shared between + multiple containers, or ``Z`` if the volume should be private. + + .. note:: + When both ```` and ```` are specified, + there must be a comma before ````. + + Binds can be expressed as a comma-separated list or a Python list, + however in cases where both ro/rw and an selinux context are specified, + the binds *must* be specified as a Python list. Examples: - ``binds=/srv/www:/var/www:ro`` - ``binds=/srv/www:/var/www:rw`` - ``binds=/srv/www:/var/www`` + - ``binds="['/srv/www:/var/www:ro,Z']"`` + - ``binds="['/srv/www:/var/www:rw,Z']"`` + - ``binds=/srv/www:/var/www:Z`` .. note:: - The second and third examples above are equivalent. + The second and third examples above are equivalent to each other, + as are the last two examples. blkio_weight Block IO weight (relative weight), accepts a weight value between 10 @@ -1898,15 +2012,20 @@ def create(image, comma-separated list or a Python list. Requires Docker 1.2.0 or newer. - Example: ``cap_add=SYS_ADMIN,MKNOD``, ``cap_add="[SYS_ADMIN, MKNOD]"`` + Examples: + + - ``cap_add=SYS_ADMIN,MKNOD`` + - ``cap_add="[SYS_ADMIN, MKNOD]"`` cap_drop List of capabilities to drop within the container. Can be passed as a comma-separated string or a Python list. Requires Docker 1.2.0 or newer. - Example: ``cap_drop=SYS_ADMIN,MKNOD``, - ``cap_drop="[SYS_ADMIN, MKNOD]"`` + Examples: + + - ``cap_drop=SYS_ADMIN,MKNOD``, + - ``cap_drop="[SYS_ADMIN, MKNOD]"`` command (or *cmd*) Command to run in the container @@ -2030,13 +2149,15 @@ def create(image, List of DNS search domains. Can be passed as a comma-separated list or a Python list. - Example: ``dns_search=foo1.domain.tld,foo2.domain.tld`` or - ``dns_search="[foo1.domain.tld, foo2.domain.tld]"`` + Examples: + + - ``dns_search=foo1.domain.tld,foo2.domain.tld`` + - ``dns_search="[foo1.domain.tld, foo2.domain.tld]"`` domainname - Set custom DNS search domains + The domain name to use for the container - Example: ``domainname=domain.tld,domain2.tld`` + Example: ``domainname=domain.tld`` entrypoint Entrypoint for the container. Either a string (e.g. ``"mycmd --arg1 @@ -2825,7 +2946,7 @@ def export(name, # open the filehandle. If not using gzip, we need to open the # filehandle here. We make sure to close it in the "finally" block # below. - out = salt.utils.fopen(path, 'wb') # pylint: disable=resource-leakage + out = salt.utils.files.fopen(path, 'wb') # pylint: disable=resource-leakage response = _client_wrapper('export', name) buf = None while buf != '': @@ -2905,10 +3026,10 @@ def rm_(name, force=False, volumes=False, **kwargs): salt myminion docker.rm mycontainer salt myminion docker.rm mycontainer force=True ''' - kwargs = salt.utils.clean_kwargs(**kwargs) + kwargs = salt.utils.args.clean_kwargs(**kwargs) stop_ = kwargs.pop('stop', False) if kwargs: - salt.utils.invalid_kwargs(kwargs) + salt.utils.args.invalid_kwargs(kwargs) if state(name) == 'running' and not (force or stop_): raise CommandExecutionError( @@ -3057,7 +3178,7 @@ def build(path=None, stream_data = [] for line in response: stream_data.extend( - json.loads(salt.utils.to_str(line), cls=DockerJSONDecoder) + json.loads(salt.utils.stringutils.to_str(line), cls=DockerJSONDecoder) ) errors = [] # Iterate through API response and collect information @@ -3775,7 +3896,6 @@ def save(name, if os.path.exists(path) and not overwrite: raise CommandExecutionError('{0} already exists'.format(path)) - compression = kwargs.get('compression') if compression is None: if path.endswith('.tar.gz') or path.endswith('.tgz'): compression = 'gzip' @@ -3813,8 +3933,9 @@ def save(name, saved_path = salt.utils.files.mkstemp() else: saved_path = path - - cmd = ['docker', 'save', '-o', saved_path, inspect_image(name)['Id']] + # use the image name if its valid if not use the image id + image_to_save = name if name in inspect_image(name)['RepoTags'] else inspect_image(name)['Id'] + cmd = ['docker', 'save', '-o', saved_path, image_to_save] time_started = time.time() result = __salt__['cmd.run_all'](cmd, python_shell=False) if result['retcode'] != 0: @@ -3837,12 +3958,12 @@ def save(name, compressor = lzma.LZMACompressor() try: - with salt.utils.fopen(saved_path, 'rb') as uncompressed: + with salt.utils.files.fopen(saved_path, 'rb') as uncompressed: if compression != 'gzip': # gzip doesn't use a Compressor object, it uses a .open() # method to open the filehandle. If not using gzip, we need # to open the filehandle here. - out = salt.utils.fopen(path, 'wb') + out = salt.utils.files.fopen(path, 'wb') buf = None while buf != '': buf = uncompressed.read(4096) @@ -3881,7 +4002,7 @@ def save(name, ret['Size_Human'] = _size_fmt(ret['Size']) # Process push - if kwargs.get(push, False): + if kwargs.get('push', False): ret['Push'] = __salt__['cp.push'](path) return ret @@ -3961,7 +4082,10 @@ def networks(names=None, ids=None): def create_network(name, driver=None, - driver_opts=None): + driver_opts=None, + gateway=None, + ip_range=None, + subnet=None): ''' Create a new network @@ -3974,16 +4098,46 @@ def create_network(name, driver_opts Options for the network driver. + gateway + IPv4 or IPv6 gateway for the master subnet + + ip_range + Allocate container IP from a sub-range within the subnet + + subnet: + Subnet in CIDR format that represents a network segment + CLI Example: .. code-block:: bash salt myminion docker.create_network web_network driver=bridge + salt myminion docker.create_network macvlan_network \ + driver=macvlan \ + driver_opts="{'parent':'eth0'}" \ + gateway=172.20.0.1 \ + subnet=172.20.0.0/24 ''' + # If any settings which need to be set via the IPAM config are specified, create the IPAM config data structure + # with these values set. + if gateway or ip_range or subnet: + ipam = { + 'Config': [{ + 'Gateway': gateway, + 'IPRange': ip_range, + 'Subnet': subnet + }], + 'Driver': 'default', + 'Options': {} + } + else: + ipam = None + response = _client_wrapper('create_network', name, driver=driver, options=driver_opts, + ipam=ipam, check_duplicate=True) _clear_context() @@ -5135,7 +5289,7 @@ def _gather_pillar(pillarenv, pillar_override, **grains): # Not sure if these two are correct __opts__['id'], __opts__['environment'], - pillar=pillar_override, + pillar_override=pillar_override, pillarenv=pillarenv ) ret = pillar.compile_pillar() @@ -5304,7 +5458,7 @@ def sls(name, mods=None, saltenv='base', **kwargs): ) if not isinstance(ret, dict): __context__['retcode'] = 1 - elif not salt.utils.check_state_result(ret): + elif not __utils__['state.check_result'](ret): __context__['retcode'] = 2 else: __context__['retcode'] = 0 @@ -5359,7 +5513,7 @@ def sls_build(name, base='opensuse/python', mods=None, saltenv='base', salt myminion docker.sls_build imgname base=mybase mods=rails,web ''' - create_kwargs = salt.utils.clean_kwargs(**copy.deepcopy(kwargs)) + create_kwargs = salt.utils.args.clean_kwargs(**copy.deepcopy(kwargs)) for key in ('image', 'name', 'cmd', 'interactive', 'tty'): try: del create_kwargs[key] @@ -5378,7 +5532,7 @@ def sls_build(name, base='opensuse/python', mods=None, saltenv='base', # Now execute the state into the container ret = sls(id_, mods, saltenv, **kwargs) # fail if the state was not successful - if not dryrun and not salt.utils.check_state_result(ret): + if not dryrun and not __utils__['state.check_result'](ret): raise CommandExecutionError(ret) if dryrun is False: ret = commit(id_, name) diff --git a/salt/modules/dpkg.py b/salt/modules/dpkg.py index 425f5e2e757..0ca210a6e8d 100644 --- a/salt/modules/dpkg.py +++ b/salt/modules/dpkg.py @@ -11,7 +11,10 @@ import re import datetime # Import salt libs -import salt.utils +import salt.utils # Can be removed once compare_dicts is moved +import salt.utils.args +import salt.utils.files +import salt.utils.path from salt.exceptions import CommandExecutionError, SaltInvocationError log = logging.getLogger(__name__) @@ -257,10 +260,10 @@ def _get_pkg_info(*packages, **kwargs): :param failhard: Throw an exception if no packages found. :return: ''' - kwargs = salt.utils.clean_kwargs(**kwargs) + kwargs = salt.utils.args.clean_kwargs(**kwargs) failhard = kwargs.pop('failhard', True) if kwargs: - salt.utils.invalid_kwargs(kwargs) + salt.utils.args.invalid_kwargs(kwargs) if __grains__['os'] == 'Ubuntu' and __grains__['osrelease_info'] < (12, 4): bin_var = '${binary}' @@ -323,7 +326,7 @@ def _get_pkg_license(pkg): licenses = set() cpr = "/usr/share/doc/{0}/copyright".format(pkg) if os.path.exists(cpr): - with salt.utils.fopen(cpr) as fp_: + with salt.utils.files.fopen(cpr) as fp_: for line in fp_.read().split(os.linesep): if line.startswith("License:"): licenses.add(line.split(":", 1)[1].strip()) @@ -354,14 +357,14 @@ def _get_pkg_ds_avail(): :return: ''' avail = "/var/lib/dpkg/available" - if not salt.utils.which('dselect') or not os.path.exists(avail): + if not salt.utils.path.which('dselect') or not os.path.exists(avail): return dict() # Do not update with dselect, just read what is. ret = dict() pkg_mrk = "Package:" pkg_name = "package" - with salt.utils.fopen(avail) as fp_: + with salt.utils.files.fopen(avail) as fp_: for pkg_info in fp_.read().split(pkg_mrk): nfo = dict() for line in (pkg_mrk + pkg_info).split(os.linesep): @@ -405,10 +408,10 @@ def info(*packages, **kwargs): # However, this file is operated by dselect which has to be installed. dselect_pkg_avail = _get_pkg_ds_avail() - kwargs = salt.utils.clean_kwargs(**kwargs) + kwargs = salt.utils.args.clean_kwargs(**kwargs) failhard = kwargs.pop('failhard', True) if kwargs: - salt.utils.invalid_kwargs(kwargs) + salt.utils.args.invalid_kwargs(kwargs) ret = dict() for pkg in _get_pkg_info(*packages, failhard=failhard): diff --git a/salt/modules/drac.py b/salt/modules/drac.py index fa481f91cf5..51d44cab751 100644 --- a/salt/modules/drac.py +++ b/salt/modules/drac.py @@ -8,17 +8,17 @@ from __future__ import absolute_import import logging # Import Salt libs -import salt.utils +import salt.utils.path # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import range # pylint: disable=import-error,no-name-in-module,redefined-builtin log = logging.getLogger(__name__) def __virtual__(): - if salt.utils.which('racadm'): + if salt.utils.path.which('racadm'): return True return (False, 'The drac execution module cannot be loaded: racadm binary not in path.') diff --git a/salt/modules/dracr.py b/salt/modules/dracr.py index 0324382513e..d42ae4b83d3 100644 --- a/salt/modules/dracr.py +++ b/salt/modules/dracr.py @@ -13,10 +13,10 @@ import re # Import Salt libs from salt.exceptions import CommandExecutionError -import salt.utils +import salt.utils.path # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import range # pylint: disable=import-error,no-name-in-module,redefined-builtin from salt.ext.six.moves import map @@ -34,7 +34,7 @@ except (NameError, KeyError): def __virtual__(): - if salt.utils.which('racadm'): + if salt.utils.path.which('racadm'): return True return (False, 'The drac execution module cannot be loaded: racadm binary not in path.') diff --git a/salt/modules/dummyproxy_package.py b/salt/modules/dummyproxy_package.py index 64f5dacda1a..66f0c43637d 100644 --- a/salt/modules/dummyproxy_package.py +++ b/salt/modules/dummyproxy_package.py @@ -7,6 +7,7 @@ from __future__ import absolute_import # Import python libs import logging import salt.utils +import salt.utils.platform log = logging.getLogger(__name__) @@ -20,12 +21,21 @@ def __virtual__(): Only work on systems that are a proxy minion ''' try: - if salt.utils.is_proxy() and __opts__['proxy']['proxytype'] == 'dummy': + if salt.utils.platform.is_proxy() \ + and __opts__['proxy']['proxytype'] == 'dummy': return __virtualname__ except KeyError: - return (False, 'The dummyproxy_package execution module failed to load. Check the proxy key in pillar or /etc/salt/proxy.') + return ( + False, + 'The dummyproxy_package execution module failed to load. Check ' + 'the proxy key in pillar or /etc/salt/proxy.' + ) - return (False, 'The dummyproxy_package execution module failed to load: only works on a dummy proxy minion.') + return ( + False, + 'The dummyproxy_package execution module failed to load: only works ' + 'on a dummy proxy minion.' + ) def list_pkgs(versions_as_list=False, **kwargs): diff --git a/salt/modules/dummyproxy_service.py b/salt/modules/dummyproxy_service.py index c085768b2c0..1be19ff5826 100644 --- a/salt/modules/dummyproxy_service.py +++ b/salt/modules/dummyproxy_service.py @@ -5,10 +5,11 @@ Provide the service module for the dummy proxy used in integration tests ''' # Import python libs from __future__ import absolute_import -import salt.utils - import logging +# Import Salt libs +import salt.utils.platform + log = logging.getLogger(__name__) __func_alias__ = { @@ -25,12 +26,21 @@ def __virtual__(): Only work on systems that are a proxy minion ''' try: - if salt.utils.is_proxy() and __opts__['proxy']['proxytype'] == 'dummy': + if salt.utils.platform.is_proxy() \ + and __opts__['proxy']['proxytype'] == 'dummy': return __virtualname__ except KeyError: - return (False, 'The dummyproxy_service execution module failed to load. Check the proxy key in pillar or /etc/salt/proxy.') + return ( + False, + 'The dummyproxy_service execution module failed to load. Check ' + 'the proxy key in pillar or /etc/salt/proxy.' + ) - return (False, 'The dummyproxy_service execution module failed to load: only works on the integration testsuite dummy proxy minion.') + return ( + False, + 'The dummyproxy_service execution module failed to load: only works ' + 'on the integration testsuite dummy proxy minion.' + ) def get_all(): diff --git a/salt/modules/ebuild.py b/salt/modules/ebuild.py index 2f1e4e28f87..5295c2652c9 100644 --- a/salt/modules/ebuild.py +++ b/salt/modules/ebuild.py @@ -22,10 +22,13 @@ import re # Import salt libs import salt.utils +import salt.utils.args +import salt.utils.path import salt.utils.pkg import salt.utils.systemd +import salt.utils.versions from salt.exceptions import CommandExecutionError, MinionError -import salt.ext.six as six +from salt.ext import six # Import third party libs HAS_PORTAGE = False @@ -72,9 +75,20 @@ def _porttree(): def _p_to_cp(p): - ret = _porttree().dbapi.xmatch("match-all", p) - if ret: - return portage.cpv_getkey(ret[0]) + try: + ret = portage.dep_getkey(p) + if ret: + return ret + except portage.exception.InvalidAtom: + pass + + try: + ret = _porttree().dbapi.xmatch('bestmatch-visible', p) + if ret: + return portage.dep_getkey(ret) + except portage.exception.InvalidAtom: + pass + return None @@ -88,11 +102,14 @@ def _allnodes(): def _cpv_to_cp(cpv): - ret = portage.cpv_getkey(cpv) - if ret: - return ret - else: - return cpv + try: + ret = portage.dep_getkey(cpv) + if ret: + return ret + except portage.exception.InvalidAtom: + pass + + return cpv def _cpv_to_version(cpv): @@ -234,7 +251,7 @@ def latest_version(*names, **kwargs): ret[name] = '' installed = _cpv_to_version(_vartree().dep_bestmatch(name)) avail = _cpv_to_version(_porttree().dep_bestmatch(name)) - if avail and (not installed or salt.utils.compare_versions(ver1=installed, oper='<', ver2=avail, cmp_func=version_cmp)): + if avail and (not installed or salt.utils.versions.compare(ver1=installed, oper='<', ver2=avail, cmp_func=version_cmp)): ret[name] = avail # Return a string if only one package name passed @@ -422,7 +439,7 @@ def refresh_db(): # GPG sign verify is supported only for "webrsync" cmd = 'emerge-webrsync -q' # We prefer 'delta-webrsync' to 'webrsync' - if salt.utils.which('emerge-delta-webrsync'): + if salt.utils.path.which('emerge-delta-webrsync'): cmd = 'emerge-delta-webrsync -q' return __salt__['cmd.retcode'](cmd, python_shell=False) == 0 else: @@ -432,7 +449,7 @@ def refresh_db(): # We fall back to "webrsync" if "rsync" fails for some reason cmd = 'emerge-webrsync -q' # We prefer 'delta-webrsync' to 'webrsync' - if salt.utils.which('emerge-delta-webrsync'): + if salt.utils.path.which('emerge-delta-webrsync'): cmd = 'emerge-delta-webrsync -q' return __salt__['cmd.retcode'](cmd, python_shell=False) == 0 @@ -1109,10 +1126,10 @@ def version_cmp(pkg1, pkg2, **kwargs): # definition (and thus have it show up in the docs), we just pop it out of # the kwargs dict and then raise an exception if any kwargs other than # ignore_epoch were passed. - kwargs = salt.utils.clean_kwargs(**kwargs) + kwargs = salt.utils.args.clean_kwargs(**kwargs) kwargs.pop('ignore_epoch', None) if kwargs: - salt.utils.invalid_kwargs(kwargs) + salt.utils.args.invalid_kwargs(kwargs) regex = r'^~?([^:\[]+):?[^\[]*\[?.*$' ver1 = re.match(regex, pkg1) diff --git a/salt/modules/eix.py b/salt/modules/eix.py index 728d237ba38..d9748792dbe 100644 --- a/salt/modules/eix.py +++ b/salt/modules/eix.py @@ -5,14 +5,14 @@ Support for Eix from __future__ import absolute_import # Import salt libs -import salt.utils +import salt.utils.path def __virtual__(): ''' Only work on Gentoo systems with eix installed ''' - if __grains__['os'] == 'Gentoo' and salt.utils.which('eix'): + if __grains__['os'] == 'Gentoo' and salt.utils.path.which('eix'): return 'eix' return (False, 'The eix execution module cannot be loaded: either the system is not Gentoo or the eix binary is not in the path.') @@ -30,7 +30,7 @@ def sync(): cmd = 'eix-sync -q -C "--ask" -C "n"' if 'makeconf.features_contains'in __salt__ and __salt__['makeconf.features_contains']('webrsync-gpg'): # GPG sign verify is supported only for "webrsync" - if salt.utils.which('emerge-delta-webrsync'): # We prefer 'delta-webrsync' to 'webrsync' + if salt.utils.path.which('emerge-delta-webrsync'): # We prefer 'delta-webrsync' to 'webrsync' cmd += ' -W' else: cmd += ' -w' @@ -39,7 +39,7 @@ def sync(): if __salt__['cmd.retcode'](cmd) == 0: return True # We fall back to "webrsync" if "rsync" fails for some reason - if salt.utils.which('emerge-delta-webrsync'): # We prefer 'delta-webrsync' to 'webrsync' + if salt.utils.path.which('emerge-delta-webrsync'): # We prefer 'delta-webrsync' to 'webrsync' cmd += ' -W' else: cmd += ' -w' diff --git a/salt/modules/elasticsearch.py b/salt/modules/elasticsearch.py index df9eeb9a195..9d81164383a 100644 --- a/salt/modules/elasticsearch.py +++ b/salt/modules/elasticsearch.py @@ -50,7 +50,7 @@ Module to provide Elasticsearch compatibility to Salt ''' from __future__ import absolute_import -from salt.exceptions import CommandExecutionError +from salt.exceptions import CommandExecutionError, SaltInvocationError # Import Python libs import logging @@ -83,10 +83,12 @@ def _get_instance(hosts=None, profile=None): Return the elasticsearch instance ''' es = None - proxies = {} + proxies = None use_ssl = False - ca_certs = False - verify_certs = False + ca_certs = None + verify_certs = True + http_auth = None + timeout = 10 if profile is None: profile = 'elasticsearch' @@ -96,37 +98,26 @@ def _get_instance(hosts=None, profile=None): elif isinstance(profile, dict): _profile = profile if _profile: - hosts = _profile.get('host', None) + hosts = _profile.get('host', hosts) if not hosts: - hosts = _profile.get('hosts', None) - proxies = _profile.get('proxies', {}) + hosts = _profile.get('hosts', hosts) + proxies = _profile.get('proxies', None) use_ssl = _profile.get('use_ssl', False) - ca_certs = _profile.get('ca_certs', False) - verify_certs = _profile.get('verify_certs', False) + ca_certs = _profile.get('ca_certs', None) + verify_certs = _profile.get('verify_certs', True) username = _profile.get('username', None) password = _profile.get('password', None) + timeout = _profile.get('timeout', 10) + + if username and password: + http_auth = (username, password) if not hosts: hosts = ['127.0.0.1:9200'] if isinstance(hosts, string_types): hosts = [hosts] try: - if proxies == {}: - es = elasticsearch.Elasticsearch( - hosts, - use_ssl=use_ssl, - ca_certs=ca_certs, - verify_certs=verify_certs, - ) - elif username and password: - es = elasticsearch.Elasticsearch( - hosts, - use_ssl=use_ssl, - ca_certs=ca_certs, - verify_certs=verify_certs, - http_auth=(username, password) - ) - else: + if proxies: # Custom connection class to use requests module with proxies class ProxyConnection(RequestsHttpConnection): def __init__(self, *args, **kwargs): @@ -137,14 +128,25 @@ def _get_instance(hosts=None, profile=None): es = elasticsearch.Elasticsearch( hosts, connection_class=ProxyConnection, - proxies=_profile.get('proxies', {}), + proxies=proxies, use_ssl=use_ssl, ca_certs=ca_certs, verify_certs=verify_certs, + http_auth=http_auth, + timeout=timeout, ) + else: + es = elasticsearch.Elasticsearch( + hosts, + use_ssl=use_ssl, + ca_certs=ca_certs, + verify_certs=verify_certs, + http_auth=http_auth, + timeout=timeout, + ) - if not es.ping(): - raise CommandExecutionError('Could not connect to Elasticsearch host/ cluster {0}, is it unhealthy?'.format(hosts)) + # Try the connection + es.info() except elasticsearch.exceptions.TransportError as e: raise CommandExecutionError('Could not connect to Elasticsearch host/ cluster {0} due to {1}'.format(hosts, str(e))) return es @@ -261,7 +263,7 @@ def cluster_stats(nodes=None, hosts=None, profile=None): raise CommandExecutionError("Cannot retrieve cluster stats, server returned code {0} with message {1}".format(e.status_code, e.error)) -def alias_create(indices, alias, hosts=None, body=None, profile=None): +def alias_create(indices, alias, hosts=None, body=None, profile=None, source=None): ''' Create an alias for a specific index/indices @@ -271,13 +273,21 @@ def alias_create(indices, alias, hosts=None, body=None, profile=None): Alias name body Optional definition such as routing or filter as defined in https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-aliases.html + source + URL of file specifying optional definition such as routing or filter. Cannot be used in combination with ``body``. CLI example:: salt myminion elasticsearch.alias_create testindex_v1 testindex ''' es = _get_instance(hosts, profile) - + if source and body: + message = 'Either body or source should be specified but not both.' + raise SaltInvocationError(message) + if source: + body = __salt__['cp.get_file_str']( + source, + saltenv=__opts__.get('saltenv', 'base')) try: result = es.indices.put_alias(index=indices, name=alias, body=body) return result.get('acknowledged', False) @@ -285,7 +295,7 @@ def alias_create(indices, alias, hosts=None, body=None, profile=None): raise CommandExecutionError("Cannot create alias {0} in index {1}, server returned code {2} with message {3}".format(alias, indices, e.status_code, e.error)) -def alias_delete(indices, aliases, hosts=None, body=None, profile=None): +def alias_delete(indices, aliases, hosts=None, body=None, profile=None, source=None): ''' Delete an alias of an index @@ -299,7 +309,13 @@ def alias_delete(indices, aliases, hosts=None, body=None, profile=None): salt myminion elasticsearch.alias_delete testindex_v1 testindex ''' es = _get_instance(hosts, profile) - + if source and body: + message = 'Either body or source should be specified but not both.' + raise SaltInvocationError(message) + if source: + body = __salt__['cp.get_file_str']( + source, + saltenv=__opts__.get('saltenv', 'base')) try: result = es.indices.delete_alias(index=indices, name=aliases) @@ -355,7 +371,7 @@ def alias_get(indices=None, aliases=None, hosts=None, profile=None): raise CommandExecutionError("Cannot get alias {0} in index {1}, server returned code {2} with message {3}".format(aliases, indices, e.status_code, e.error)) -def document_create(index, doc_type, body=None, id=None, hosts=None, profile=None): +def document_create(index, doc_type, body=None, id=None, hosts=None, profile=None, source=None): ''' Create a document in a specified index @@ -365,6 +381,8 @@ def document_create(index, doc_type, body=None, id=None, hosts=None, profile=Non Type of the document body Document to store + source + URL of file specifying document to store. Cannot be used in combination with ``body``. id Optional unique document identifier for specified doc_type (empty for random) @@ -373,7 +391,13 @@ def document_create(index, doc_type, body=None, id=None, hosts=None, profile=Non salt myminion elasticsearch.document_create testindex doctype1 '{}' ''' es = _get_instance(hosts, profile) - + if source and body: + message = 'Either body or source should be specified but not both.' + raise SaltInvocationError(message) + if source: + body = __salt__['cp.get_file_str']( + source, + saltenv=__opts__.get('saltenv', 'base')) try: return es.index(index=index, doc_type=doc_type, body=body, id=id) except elasticsearch.TransportError as e: @@ -455,7 +479,7 @@ def document_get(index, id, doc_type='_all', hosts=None, profile=None): raise CommandExecutionError("Cannot retrieve document {0} from index {1}, server returned code {2} with message {3}".format(id, index, e.status_code, e.error)) -def index_create(index, body=None, hosts=None, profile=None): +def index_create(index, body=None, hosts=None, profile=None, source=None): ''' Create an index @@ -463,6 +487,8 @@ def index_create(index, body=None, hosts=None, profile=None): Index name body Index definition, such as settings and mappings as defined in https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-create-index.html + source + URL to file specifying index definition. Cannot be used in combination with ``body``. CLI example:: @@ -470,7 +496,13 @@ def index_create(index, body=None, hosts=None, profile=None): salt myminion elasticsearch.index_create testindex2 '{"settings" : {"index" : {"number_of_shards" : 3, "number_of_replicas" : 2}}}' ''' es = _get_instance(hosts, profile) - + if source and body: + message = 'Either body or source should be specified but not both.' + raise SaltInvocationError(message) + if source: + body = __salt__['cp.get_file_str']( + source, + saltenv=__opts__.get('saltenv', 'base')) try: result = es.indices.create(index=index, body=body) return result.get('acknowledged', False) and result.get("shards_acknowledged", True) @@ -604,7 +636,7 @@ def index_close(index, allow_no_indices=True, expand_wildcards='open', ignore_un raise CommandExecutionError("Cannot close index {0}, server returned code {1} with message {2}".format(index, e.status_code, e.error)) -def mapping_create(index, doc_type, body, hosts=None, profile=None): +def mapping_create(index, doc_type, body=None, hosts=None, profile=None, source=None): ''' Create a mapping in a given index @@ -614,12 +646,21 @@ def mapping_create(index, doc_type, body, hosts=None, profile=None): Name of the document type body Mapping definition as specified in https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-put-mapping.html + source + URL to file specifying mapping definition. Cannot be used in combination with ``body``. CLI example:: salt myminion elasticsearch.mapping_create testindex user '{ "user" : { "properties" : { "message" : {"type" : "string", "store" : true } } } }' ''' es = _get_instance(hosts, profile) + if source and body: + message = 'Either body or source should be specified but not both.' + raise SaltInvocationError(message) + if source: + body = __salt__['cp.get_file_str']( + source, + saltenv=__opts__.get('saltenv', 'base')) try: result = es.indices.put_mapping(index=index, doc_type=doc_type, body=body) @@ -677,23 +718,33 @@ def mapping_get(index, doc_type, hosts=None, profile=None): raise CommandExecutionError("Cannot retrieve mapping {0}, server returned code {1} with message {2}".format(index, e.status_code, e.error)) -def index_template_create(name, body, hosts=None, profile=None): +def index_template_create(name, body=None, hosts=None, profile=None, source=None): ''' Create an index template name Index template name + body Template definition as specified in http://www.elastic.co/guide/en/elasticsearch/reference/current/indices-templates.html + source + URL to file specifying template definition. Cannot be used in combination with ``body``. + CLI example:: salt myminion elasticsearch.index_template_create testindex_templ '{ "template": "logstash-*", "order": 1, "settings": { "number_of_shards": 1 } }' ''' es = _get_instance(hosts, profile) + if source and body: + message = 'Either body or source should be specified but not both.' + raise SaltInvocationError(message) + if source: + body = __salt__['cp.get_file_str']( + source, + saltenv=__opts__.get('saltenv', 'base')) try: result = es.indices.put_template(name=name, body=body) - return result.get('acknowledged', False) except elasticsearch.TransportError as e: raise CommandExecutionError("Cannot create template {0}, server returned code {1} with message {2}".format(name, e.status_code, e.error)) diff --git a/salt/modules/environ.py b/salt/modules/environ.py index 112c176c3c9..26972f4ca9d 100644 --- a/salt/modules/environ.py +++ b/salt/modules/environ.py @@ -3,17 +3,16 @@ Support for getting and setting the environment variables of the current salt process. ''' -from __future__ import absolute_import - -# Import salt libs -import salt.utils as utils - # Import python libs +from __future__ import absolute_import import os import logging +# Import Salt libs +import salt.utils.platform + # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six log = logging.getLogger(__name__) @@ -61,7 +60,7 @@ def setval(key, val, false_unsets=False, permanent=False): salt '*' environ.setval baz bar permanent=True salt '*' environ.setval baz bar permanent=HKLM ''' - is_windows = utils.is_windows() + is_windows = salt.utils.platform.is_windows() if is_windows: permanent_hive = 'HKCU' permanent_key = 'Environment' diff --git a/salt/modules/eselect.py b/salt/modules/eselect.py index 09b04fd735c..67664622276 100644 --- a/salt/modules/eselect.py +++ b/salt/modules/eselect.py @@ -7,7 +7,7 @@ from __future__ import absolute_import import logging # Import salt libs -import salt.utils +import salt.utils.path log = logging.getLogger(__name__) @@ -16,7 +16,7 @@ def __virtual__(): ''' Only work on Gentoo systems with eselect installed ''' - if __grains__['os'] == 'Gentoo' and salt.utils.which('eselect'): + if __grains__['os'] == 'Gentoo' and salt.utils.path.which('eselect'): return 'eselect' return (False, 'The eselect execution module cannot be loaded: either the system is not Gentoo or the eselect binary is not in the path.') diff --git a/salt/modules/esxcluster.py b/salt/modules/esxcluster.py new file mode 100644 index 00000000000..fca68d775f6 --- /dev/null +++ b/salt/modules/esxcluster.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +''' +Module used to access the esxcluster proxy connection methods +''' +from __future__ import absolute_import + +# Import python libs +import logging +import salt.utils.platform + + +log = logging.getLogger(__name__) + +__proxyenabled__ = ['esxcluster'] +# Define the module's virtual name +__virtualname__ = 'esxcluster' + + +def __virtual__(): + ''' + Only work on proxy + ''' + if salt.utils.platform.is_proxy(): + return __virtualname__ + return (False, 'Must be run on a proxy minion') + + +def get_details(): + return __proxy__['esxcluster.get_details']() diff --git a/salt/modules/esxdatacenter.py b/salt/modules/esxdatacenter.py new file mode 100644 index 00000000000..f36ff279281 --- /dev/null +++ b/salt/modules/esxdatacenter.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +''' +Module used to access the esxdatacenter proxy connection methods +''' +from __future__ import absolute_import + +# Import python libs +import logging +import salt.utils.platform + + +log = logging.getLogger(__name__) + +__proxyenabled__ = ['esxdatacenter'] +# Define the module's virtual name +__virtualname__ = 'esxdatacenter' + + +def __virtual__(): + ''' + Only work on proxy + ''' + if salt.utils.platform.is_proxy(): + return __virtualname__ + return (False, 'Must be run on a proxy minion') + + +def get_details(): + return __proxy__['esxdatacenter.get_details']() diff --git a/salt/modules/esxi.py b/salt/modules/esxi.py index 144a84f2427..a4c1f8ddcc4 100644 --- a/salt/modules/esxi.py +++ b/salt/modules/esxi.py @@ -30,7 +30,9 @@ from __future__ import absolute_import # Import python libs import logging -import salt.utils + +# Import Salt libs +import salt.utils.platform log = logging.getLogger(__name__) @@ -43,7 +45,7 @@ def __virtual__(): ''' Only work on proxy ''' - if salt.utils.is_proxy(): + if salt.utils.platform.is_proxy(): return __virtualname__ return (False, 'The esxi execution module failed to load: ' 'only available on proxy minions.') diff --git a/salt/modules/etcd_mod.py b/salt/modules/etcd_mod.py index 78088c306af..5519aabd096 100644 --- a/salt/modules/etcd_mod.py +++ b/salt/modules/etcd_mod.py @@ -73,7 +73,7 @@ def __virtual__(): 'python etcd library not available.') -def get_(key, recurse=False, profile=None): +def get_(key, recurse=False, profile=None, **kwargs): ''' .. versionadded:: 2014.7.0 @@ -86,15 +86,16 @@ def get_(key, recurse=False, profile=None): salt myminion etcd.get /path/to/key salt myminion etcd.get /path/to/key profile=my_etcd_config salt myminion etcd.get /path/to/key recurse=True profile=my_etcd_config + salt myminion etcd.get /path/to/key host=127.0.0.1 port=2379 ''' - client = __utils__['etcd_util.get_conn'](__opts__, profile) + client = __utils__['etcd_util.get_conn'](__opts__, profile, **kwargs) if recurse: return client.tree(key) else: return client.get(key, recurse=recurse) -def set_(key, value, profile=None, ttl=None, directory=False): +def set_(key, value, profile=None, ttl=None, directory=False, **kwargs): ''' .. versionadded:: 2014.7.0 @@ -107,15 +108,16 @@ def set_(key, value, profile=None, ttl=None, directory=False): salt myminion etcd.set /path/to/key value salt myminion etcd.set /path/to/key value profile=my_etcd_config + salt myminion etcd.set /path/to/key value host=127.0.0.1 port=2379 salt myminion etcd.set /path/to/dir '' directory=True salt myminion etcd.set /path/to/key value ttl=5 ''' - client = __utils__['etcd_util.get_conn'](__opts__, profile) + client = __utils__['etcd_util.get_conn'](__opts__, profile, **kwargs) return client.set(key, value, ttl=ttl, directory=directory) -def update(fields, path='', profile=None): +def update(fields, path='', profile=None, **kwargs): ''' .. versionadded:: 2016.3.0 @@ -162,13 +164,14 @@ def update(fields, path='', profile=None): salt myminion etcd.update "{'/path/to/key': 'baz', '/another/key': 'bar'}" salt myminion etcd.update "{'/path/to/key': 'baz', '/another/key': 'bar'}" profile=my_etcd_config + salt myminion etcd.update "{'/path/to/key': 'baz', '/another/key': 'bar'}" host=127.0.0.1 port=2379 salt myminion etcd.update "{'/path/to/key': 'baz', '/another/key': 'bar'}" path='/some/root' ''' - client = __utils__['etcd_util.get_conn'](__opts__, profile) + client = __utils__['etcd_util.get_conn'](__opts__, profile, **kwargs) return client.update(fields, path) -def watch(key, recurse=False, profile=None, timeout=0, index=None): +def watch(key, recurse=False, profile=None, timeout=0, index=None, **kwargs): ''' .. versionadded:: 2016.3.0 @@ -186,13 +189,14 @@ def watch(key, recurse=False, profile=None, timeout=0, index=None): salt myminion etcd.watch /path/to/key salt myminion etcd.watch /path/to/key timeout=10 salt myminion etcd.watch /patch/to/key profile=my_etcd_config index=10 + salt myminion etcd.watch /patch/to/key host=127.0.0.1 port=2379 ''' - client = __utils__['etcd_util.get_conn'](__opts__, profile) + client = __utils__['etcd_util.get_conn'](__opts__, profile, **kwargs) return client.watch(key, recurse=recurse, timeout=timeout, index=index) -def ls_(path='/', profile=None): +def ls_(path='/', profile=None, **kwargs): ''' .. versionadded:: 2014.7.0 @@ -206,12 +210,13 @@ def ls_(path='/', profile=None): salt myminion etcd.ls /path/to/dir/ salt myminion etcd.ls /path/to/dir/ profile=my_etcd_config + salt myminion etcd.ls /path/to/dir/ host=127.0.0.1 port=2379 ''' - client = __utils__['etcd_util.get_conn'](__opts__, profile) + client = __utils__['etcd_util.get_conn'](__opts__, profile, **kwargs) return client.ls(path) -def rm_(key, recurse=False, profile=None): +def rm_(key, recurse=False, profile=None, **kwargs): ''' .. versionadded:: 2014.7.0 @@ -225,13 +230,14 @@ def rm_(key, recurse=False, profile=None): salt myminion etcd.rm /path/to/key salt myminion etcd.rm /path/to/key profile=my_etcd_config + salt myminion etcd.rm /path/to/key host=127.0.0.1 port=2379 salt myminion etcd.rm /path/to/dir recurse=True profile=my_etcd_config ''' - client = __utils__['etcd_util.get_conn'](__opts__, profile) + client = __utils__['etcd_util.get_conn'](__opts__, profile, **kwargs) return client.rm(key, recurse=recurse) -def tree(path='/', profile=None): +def tree(path='/', profile=None, **kwargs): ''' .. versionadded:: 2014.7.0 @@ -244,7 +250,8 @@ def tree(path='/', profile=None): salt myminion etcd.tree salt myminion etcd.tree profile=my_etcd_config + salt myminion etcd.tree host=127.0.0.1 port=2379 salt myminion etcd.tree /path/to/keys profile=my_etcd_config ''' - client = __utils__['etcd_util.get_conn'](__opts__, profile) + client = __utils__['etcd_util.get_conn'](__opts__, profile, **kwargs) return client.tree(path) diff --git a/salt/modules/event.py b/salt/modules/event.py index 08b551fc3ae..4c5259ec234 100644 --- a/salt/modules/event.py +++ b/salt/modules/event.py @@ -16,7 +16,7 @@ import salt.crypt import salt.utils.event import salt.payload import salt.transport -import salt.ext.six as six +from salt.ext import six __proxyenabled__ = ['*'] log = logging.getLogger(__name__) diff --git a/salt/modules/extfs.py b/salt/modules/extfs.py index eabcf5f116e..87ca37e53c1 100644 --- a/salt/modules/extfs.py +++ b/salt/modules/extfs.py @@ -8,7 +8,7 @@ from __future__ import absolute_import import logging # Import salt libs -import salt.utils +import salt.utils.platform log = logging.getLogger(__name__) @@ -17,8 +17,12 @@ def __virtual__(): ''' Only work on POSIX-like systems ''' - if salt.utils.is_windows(): - return (False, 'The extfs execution module cannot be loaded: only available on non-Windows systems.') + if salt.utils.platform.is_windows(): + return ( + False, + 'The extfs execution module cannot be loaded: only available on ' + 'non-Windows systems.' + ) return True diff --git a/salt/modules/file.py b/salt/modules/file.py index 317e6af889d..944736f7402 100644 --- a/salt/modules/file.py +++ b/salt/modules/file.py @@ -34,7 +34,7 @@ from collections import Iterable, Mapping from functools import reduce # pylint: disable=redefined-builtin # pylint: disable=import-error,no-name-in-module,redefined-builtin -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import range, zip from salt.ext.six.moves.urllib.parse import urlparse as _urlparse # pylint: enable=import-error,no-name-in-module,redefined-builtin @@ -47,11 +47,16 @@ except ImportError: # Import salt libs import salt.utils +import salt.utils.args import salt.utils.atomicfile -import salt.utils.find import salt.utils.filebuffer import salt.utils.files +import salt.utils.find +import salt.utils.itertools import salt.utils.locales +import salt.utils.path +import salt.utils.platform +import salt.utils.stringutils import salt.utils.templates import salt.utils.url from salt.exceptions import CommandExecutionError, MinionError, SaltInvocationError, get_error_message as _get_error_message @@ -78,8 +83,12 @@ def __virtual__(): Only work on POSIX-like systems ''' # win_file takes care of windows - if salt.utils.is_windows(): - return (False, 'The file execution module cannot be loaded: only available on non-Windows systems - use win_file instead.') + if salt.utils.platform.is_windows(): + return ( + False, + 'The file execution module cannot be loaded: only available on ' + 'non-Windows systems - use win_file instead.' + ) return True @@ -117,16 +126,16 @@ def _binary_replace(old, new): This function should only be run AFTER it has been determined that the files differ. ''' - old_isbin = not salt.utils.istextfile(old) - new_isbin = not salt.utils.istextfile(new) + old_isbin = not __utils__['files.is_text_file'](old) + new_isbin = not __utils__['files.is_text_file'](new) if any((old_isbin, new_isbin)): if all((old_isbin, new_isbin)): - return 'Replace binary file' + return u'Replace binary file' elif old_isbin: - return 'Replace binary file with text file' + return u'Replace binary file with text file' elif new_isbin: - return 'Replace text file with binary file' - return '' + return u'Replace text file with binary file' + return u'' def _get_bkroot(): @@ -549,17 +558,13 @@ def lsattr(path): if not os.path.exists(path): raise SaltInvocationError("File or directory does not exist.") - # Put quotes around path to ensure we can handle spaces in filename. - if not salt.utils.is_quoted(path): - path = '"{0}"'.format(path) - - cmd = 'lsattr {0}'.format(path) + cmd = ['lsattr', path] result = __salt__['cmd.run'](cmd, python_shell=False) results = {} for line in result.splitlines(): if not line.startswith('lsattr'): - vals = line.split() + vals = line.split(None, 1) results[vals[1]] = re.findall(r"[acdijstuADST]", vals[0]) return results @@ -598,7 +603,7 @@ def chattr(*args, **kwargs): salt '*' file.chattr foo1.txt foo2.txt operator=add attributes=ai salt '*' file.chattr foo3.txt operator=remove attributes=i version=2 ''' - args = [arg if salt.utils.is_quoted(arg) else '"{0}"'.format(arg) + args = [arg if salt.utils.stringutils.is_quoted(arg) else '"{0}"'.format(arg) for arg in args] operator = kwargs.pop('operator', None) @@ -844,18 +849,21 @@ def check_hash(path, file_hash): hash The hash to check against the file specified in the ``path`` argument. - For versions 2016.11.4 and newer, the hash can be specified without an + + .. versionchanged:: 2016.11.4 + + For this and newer versions the hash can be specified without an accompanying hash type (e.g. ``e138491e9d5b97023cea823fe17bac22``), but for earlier releases it is necessary to also specify the hash type - in the format ``:`` (e.g. - ``md5:e138491e9d5b97023cea823fe17bac22``). + in the format ``=`` (e.g. + ``md5=e138491e9d5b97023cea823fe17bac22``). CLI Example: .. code-block:: bash salt '*' file.check_hash /etc/fstab e138491e9d5b97023cea823fe17bac22 - salt '*' file.check_hash /etc/fstab md5:e138491e9d5b97023cea823fe17bac22 + salt '*' file.check_hash /etc/fstab md5=e138491e9d5b97023cea823fe17bac22 ''' path = os.path.expanduser(path) @@ -1099,7 +1107,7 @@ def sed(path, cmd = ['sed'] cmd.append('-i{0}'.format(backup) if backup else '-i') - cmd.extend(salt.utils.shlex_split(options)) + cmd.extend(salt.utils.args.shlex_split(options)) cmd.append( r'{limit}{negate_match}s/{before}/{after}/{flags}'.format( limit='/{0}/ '.format(limit) if limit else '', @@ -1146,7 +1154,7 @@ def sed_contains(path, options = options.replace('-r', '-E') cmd = ['sed'] - cmd.extend(salt.utils.shlex_split(options)) + cmd.extend(salt.utils.args.shlex_split(options)) cmd.append( r'{limit}s/{before}/$/{flags}'.format( limit='/{0}/ '.format(limit) if limit else '', @@ -1233,8 +1241,8 @@ def psed(path, shutil.copy2(path, '{0}{1}'.format(path, backup)) - with salt.utils.fopen(path, 'w') as ofile: - with salt.utils.fopen('{0}{1}'.format(path, backup), 'r') as ifile: + with salt.utils.files.fopen(path, 'w') as ofile: + with salt.utils.files.fopen('{0}{1}'.format(path, backup), 'r') as ifile: if multi is True: for line in ifile.readline(): ofile.write(_psed(line, before, after, limit, flags)) @@ -1428,7 +1436,7 @@ def comment_line(path, raise SaltInvocationError('File not found: {0}'.format(path)) # Make sure it is a text file - if not salt.utils.istextfile(path): + if not __utils__['files.is_text_file'](path): raise SaltInvocationError( 'Cannot perform string replacements on a binary file: {0}'.format(path)) @@ -1442,7 +1450,7 @@ def comment_line(path, bufsize = os.path.getsize(path) try: # Use a read-only handle to open the file - with salt.utils.fopen(path, + with salt.utils.files.fopen(path, mode='rb', buffering=bufsize) as r_file: # Loop through each line of the file and look for a match @@ -1468,7 +1476,7 @@ def comment_line(path, if not found: return False - if not salt.utils.is_windows(): + if not salt.utils.platform.is_windows(): pre_user = get_user(path) pre_group = get_group(path) pre_mode = salt.utils.normalize_mode(get_mode(path)) @@ -1481,12 +1489,12 @@ def comment_line(path, try: # Open the file in write mode - with salt.utils.fopen(path, + with salt.utils.files.fopen(path, mode='wb', buffering=bufsize) as w_file: try: # Open the temp file in read mode - with salt.utils.fopen(temp_file, + with salt.utils.files.fopen(temp_file, mode='rb', buffering=bufsize) as r_file: # Loop through each line of the file and look for a match @@ -1533,7 +1541,7 @@ def comment_line(path, else: os.remove(temp_file) - if not salt.utils.is_windows(): + if not salt.utils.platform.is_windows(): check_perms(path, None, pre_user, pre_group, pre_mode) # Return a diff using the two dictionaries @@ -1857,9 +1865,9 @@ def line(path, content=None, match=None, mode=None, location=None, if before is None and after is None and not match: match = content - with salt.utils.fopen(path, mode='r') as fp_: + with salt.utils.files.fopen(path, mode='r') as fp_: body = fp_.read() - body_before = hashlib.sha256(salt.utils.to_bytes(body)).hexdigest() + body_before = hashlib.sha256(salt.utils.stringutils.to_bytes(body)).hexdigest() after = _regex_to_static(body, after) before = _regex_to_static(body, before) match = _regex_to_static(body, match) @@ -1987,7 +1995,7 @@ def line(path, content=None, match=None, mode=None, location=None, "Unable to ensure line without knowing " "where to put it before and/or after.") - changed = body_before != hashlib.sha256(salt.utils.to_bytes(body)).hexdigest() + changed = body_before != hashlib.sha256(salt.utils.stringutils.to_bytes(body)).hexdigest() if backup and changed and __opts__['test'] is False: try: @@ -2000,7 +2008,7 @@ def line(path, content=None, match=None, mode=None, location=None, if changed: if show_changes: - with salt.utils.fopen(path, 'r') as fp_: + with salt.utils.files.fopen(path, 'r') as fp_: path_content = _splitlines_preserving_trailing_newline( fp_.read()) changes_diff = ''.join(difflib.unified_diff( @@ -2032,6 +2040,7 @@ def replace(path, show_changes=True, ignore_if_missing=False, preserve_inode=True, + backslash_literal=False, ): ''' .. versionadded:: 0.17.0 @@ -2132,6 +2141,14 @@ def replace(path, filename. Hard links will then share an inode with the backup, instead (if using ``backup`` to create a backup copy). + backslash_literal : False + .. versionadded:: 2016.11.7 + + Interpret backslashes as literal backslashes for the repl and not + escape characters. This will help when using append/prepend so that + the backslashes are not interpreted for the repl on the second run of + the state. + If an equal sign (``=``) appears in an argument to a Salt command it is interpreted as a keyword argument in the format ``key=val``. That processing can be bypassed in order to pass an equal sign through to the @@ -2163,7 +2180,7 @@ def replace(path, else: raise SaltInvocationError('File not found: {0}'.format(path)) - if not salt.utils.istextfile(path): + if not __utils__['files.is_text_file'](path): raise SaltInvocationError( 'Cannot perform string replacements on a binary file: {0}' .format(path) @@ -2180,7 +2197,7 @@ def replace(path, ) flags_num = _get_flags(flags) - cpattern = re.compile(salt.utils.to_bytes(pattern), flags_num) + cpattern = re.compile(salt.utils.stringutils.to_bytes(pattern), flags_num) filesize = os.path.getsize(path) if bufsize == 'file': bufsize = filesize @@ -2189,30 +2206,30 @@ def replace(path, has_changes = False orig_file = [] # used for show_changes and change detection new_file = [] # used for show_changes and change detection - if not salt.utils.is_windows(): + if not salt.utils.platform.is_windows(): pre_user = get_user(path) pre_group = get_group(path) pre_mode = salt.utils.normalize_mode(get_mode(path)) # Avoid TypeErrors by forcing repl to be bytearray related to mmap # Replacement text may contains integer: 123 for example - repl = salt.utils.to_bytes(str(repl)) + repl = salt.utils.stringutils.to_bytes(str(repl)) if not_found_content: - not_found_content = salt.utils.to_bytes(not_found_content) + not_found_content = salt.utils.stringutils.to_bytes(not_found_content) found = False temp_file = None - content = salt.utils.to_str(not_found_content) if not_found_content and \ + content = salt.utils.stringutils.to_str(not_found_content) if not_found_content and \ (prepend_if_not_found or append_if_not_found) \ - else salt.utils.to_str(repl) + else salt.utils.stringutils.to_str(repl) try: # First check the whole file, determine whether to make the replacement # Searching first avoids modifying the time stamp if there are no changes r_data = None # Use a read-only handle to open the file - with salt.utils.fopen(path, + with salt.utils.files.fopen(path, mode='rb', buffering=bufsize) as r_file: try: @@ -2222,13 +2239,16 @@ def replace(path, access=mmap.ACCESS_READ) except (ValueError, mmap.error): # size of file in /proc is 0, but contains data - r_data = salt.utils.to_bytes("".join(r_file)) + r_data = salt.utils.stringutils.to_bytes("".join(r_file)) if search_only: # Just search; bail as early as a match is found if re.search(cpattern, r_data): return True # `with` block handles file closure else: - result, nrepl = re.subn(cpattern, repl, r_data, count) + result, nrepl = re.subn(cpattern, + repl.replace('\\', '\\\\') if backslash_literal else repl, + r_data, + count) # found anything? (even if no change) if nrepl > 0: @@ -2239,7 +2259,7 @@ def replace(path, if prepend_if_not_found or append_if_not_found: # Search for content, to avoid pre/appending the # content if it was pre/appended in a previous run. - if re.search(salt.utils.to_bytes('^{0}$'.format(re.escape(content))), + if re.search(salt.utils.stringutils.to_bytes('^{0}$'.format(re.escape(content))), r_data, flags=flags_num): # Content was found, so set found. @@ -2271,21 +2291,23 @@ def replace(path, r_data = None try: # Open the file in write mode - with salt.utils.fopen(path, + with salt.utils.files.fopen(path, mode='w', buffering=bufsize) as w_file: try: # Open the temp file in read mode - with salt.utils.fopen(temp_file, + with salt.utils.files.fopen(temp_file, mode='r', buffering=bufsize) as r_file: r_data = mmap.mmap(r_file.fileno(), 0, access=mmap.ACCESS_READ) - result, nrepl = re.subn(cpattern, repl, - r_data, count) + result, nrepl = re.subn(cpattern, + repl.replace('\\', '\\\\') if backslash_literal else repl, + r_data, + count) try: - w_file.write(salt.utils.to_str(result)) + w_file.write(salt.utils.stringutils.to_str(result)) except (OSError, IOError) as exc: raise CommandExecutionError( "Unable to write file '{0}'. Contents may " @@ -2325,7 +2347,7 @@ def replace(path, try: fh_ = salt.utils.atomicfile.atomic_open(path, 'w') for line in new_file: - fh_.write(salt.utils.to_str(line)) + fh_.write(salt.utils.stringutils.to_str(line)) finally: fh_.close() @@ -2367,12 +2389,12 @@ def replace(path, "Exception: {1}".format(temp_file, exc) ) - if not dry_run and not salt.utils.is_windows(): + if not dry_run and not salt.utils.platform.is_windows(): check_perms(path, None, pre_user, pre_group, pre_mode) def get_changes(): - orig_file_as_str = [salt.utils.to_str(x) for x in orig_file] - new_file_as_str = [salt.utils.to_str(x) for x in new_file] + orig_file_as_str = [salt.utils.stringutils.to_str(x) for x in orig_file] + new_file_as_str = [salt.utils.stringutils.to_str(x) for x in new_file] return ''.join(difflib.unified_diff(orig_file_as_str, new_file_as_str)) if show_changes: @@ -2475,7 +2497,7 @@ def blockreplace(path, 'Only one of append and prepend_if_not_found is permitted' ) - if not salt.utils.istextfile(path): + if not __utils__['files.is_text_file'](path): raise SaltInvocationError( 'Cannot perform string replacements on a binary file: {0}' .format(path) @@ -2689,7 +2711,7 @@ def patch(originalfile, patchfile, options='', dry_run=False): salt '*' file.patch /opt/file.txt /tmp/file.txt.patch ''' - patchpath = salt.utils.which('patch') + patchpath = salt.utils.path.which('patch') if not patchpath: raise CommandExecutionError( 'patch executable not found. Is the distribution\'s patch ' @@ -2697,7 +2719,7 @@ def patch(originalfile, patchfile, options='', dry_run=False): ) cmd = [patchpath] - cmd.extend(salt.utils.shlex_split(options)) + cmd.extend(salt.utils.args.shlex_split(options)) if dry_run: if __grains__['kernel'] in ('FreeBSD', 'OpenBSD'): cmd.append('-C') @@ -2792,7 +2814,7 @@ def contains_regex(path, regex, lchar=''): return False try: - with salt.utils.fopen(path, 'r') as target: + with salt.utils.files.fopen(path, 'r') as target: for line in target: if lchar: line = line.lstrip(lchar) @@ -2876,8 +2898,8 @@ def append(path, *args, **kwargs): # Make sure we have a newline at the end of the file. Do this in binary # mode so SEEK_END with nonzero offset will work. - with salt.utils.fopen(path, 'rb+') as ofile: - linesep = salt.utils.to_bytes(os.linesep) + with salt.utils.files.fopen(path, 'rb+') as ofile: + linesep = salt.utils.stringutils.to_bytes(os.linesep) try: ofile.seek(-len(linesep), os.SEEK_END) except IOError as exc: @@ -2892,7 +2914,7 @@ def append(path, *args, **kwargs): ofile.write(linesep) # Append lines in text mode - with salt.utils.fopen(path, 'a') as ofile: + with salt.utils.files.fopen(path, 'a') as ofile: for new_line in args: ofile.write('{0}{1}'.format(new_line, os.linesep)) @@ -2941,7 +2963,7 @@ def prepend(path, *args, **kwargs): args = [kwargs['args']] try: - with salt.utils.fopen(path) as fhr: + with salt.utils.files.fopen(path) as fhr: contents = fhr.readlines() except IOError: contents = [] @@ -2950,7 +2972,7 @@ def prepend(path, *args, **kwargs): for line in args: preface.append('{0}\n'.format(line)) - with salt.utils.fopen(path, "w") as ofile: + with salt.utils.files.fopen(path, "w") as ofile: contents = preface + contents ofile.write(''.join(contents)) return 'Prepended {0} lines to "{1}"'.format(len(args), path) @@ -2999,7 +3021,7 @@ def write(path, *args, **kwargs): contents = [] for line in args: contents.append('{0}\n'.format(line)) - with salt.utils.fopen(path, "w") as ofile: + with salt.utils.files.fopen(path, "w") as ofile: ofile.write(''.join(contents)) return 'Wrote {0} lines to "{1}"'.format(len(contents), path) @@ -3030,7 +3052,7 @@ def touch(name, atime=None, mtime=None): mtime = int(mtime) try: if not os.path.exists(name): - with salt.utils.fopen(name, 'a') as fhw: + with salt.utils.files.fopen(name, 'a') as fhw: fhw.write('') if not atime and not mtime: @@ -3133,7 +3155,7 @@ def truncate(path, length): salt '*' file.truncate /path/to/file 512 ''' path = os.path.expanduser(path) - with salt.utils.fopen(path, 'rb+') as seek_fh: + with salt.utils.files.fopen(path, 'rb+') as seek_fh: seek_fh.truncate(int(length)) @@ -3272,7 +3294,7 @@ def copy(src, dst, recurse=False, remove_existing=False): if not os.path.exists(src): raise CommandExecutionError('No such file or directory \'{0}\''.format(src)) - if not salt.utils.is_windows(): + if not salt.utils.platform.is_windows(): pre_user = get_user(src) pre_group = get_group(src) pre_mode = salt.utils.normalize_mode(get_mode(src)) @@ -3295,7 +3317,7 @@ def copy(src, dst, recurse=False, remove_existing=False): 'Could not copy \'{0}\' to \'{1}\''.format(src, dst) ) - if not salt.utils.is_windows(): + if not salt.utils.platform.is_windows(): check_perms(dst, None, pre_user, pre_group, pre_mode) return True @@ -3380,7 +3402,7 @@ def read(path, binary=False): access_mode = 'r' if binary is True: access_mode += 'b' - with salt.utils.fopen(path, access_mode) as file_obj: + with salt.utils.files.fopen(path, access_mode) as file_obj: return file_obj.read() @@ -4145,7 +4167,7 @@ def extract_hash(hash_fn, partial = None found = {} - with salt.utils.fopen(hash_fn, 'r') as fp_: + with salt.utils.files.fopen(hash_fn, 'r') as fp_: for line in fp_: line = line.strip() hash_re = r'(?i)(?' - else: - # Check to see if the files are bins - bdiff = _binary_replace(name, sfn) - if bdiff: - changes['diff'] = bdiff - else: - with salt.utils.fopen(sfn, 'r') as src: - slines = src.readlines() - with salt.utils.fopen(name, 'r') as name_: - nlines = name_.readlines() - changes['diff'] = \ - ''.join(difflib.unified_diff(nlines, slines)) + try: + changes['diff'] = get_diff( + sfn, name, template=True, show_filenames=False) + except CommandExecutionError as exc: + changes['diff'] = exc.strerror else: changes['sum'] = 'Checksum differs' @@ -4684,28 +4697,25 @@ def check_file_meta( # Write a tempfile with the static contents tmp = salt.utils.files.mkstemp(prefix=salt.utils.files.TEMPFILE_PREFIX, text=True) - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): contents = os.linesep.join( _splitlines_preserving_trailing_newline(contents)) - with salt.utils.fopen(tmp, 'w') as tmp_: - tmp_.write(str(contents)) + with salt.utils.files.fopen(tmp, 'w') as tmp_: + tmp_.write(salt.utils.stringutils.to_str(contents)) # Compare the static contents with the named file - with salt.utils.fopen(tmp, 'r') as src: - slines = src.readlines() - with salt.utils.fopen(name, 'r') as name_: - nlines = name_.readlines() + try: + differences = get_diff(name, tmp, show_filenames=False) + except CommandExecutionError as exc: + log.error('Failed to diff files: {0}'.format(exc)) + differences = exc.strerror __clean_tmp(tmp) - if ''.join(nlines) != ''.join(slines): + if differences: if __salt__['config.option']('obfuscate_templates'): changes['diff'] = '' else: - if salt.utils.istextfile(name): - changes['diff'] = \ - ''.join(difflib.unified_diff(nlines, slines)) - else: - changes['diff'] = 'Replace binary file with text file' + changes['diff'] = differences - if not salt.utils.is_windows(): + if not salt.utils.platform.is_windows(): # Check owner if (user is not None and user != lstats['user'] @@ -4735,44 +4745,113 @@ def check_file_meta( return changes -def get_diff( - minionfile, - masterfile, - saltenv='base'): +def get_diff(file1, + file2, + saltenv='base', + show_filenames=True, + show_changes=True, + template=False): ''' - Return unified diff of file compared to file on master + Return unified diff of two files - CLI Example: + file1 + The first file to feed into the diff utility + + .. versionchanged:: Oxygen + Can now be either a local or remote file. In earlier releases, + thuis had to be a file local to the minion. + + file2 + The second file to feed into the diff utility + + .. versionchanged:: Oxygen + Can now be either a local or remote file. In earlier releases, this + had to be a file on the salt fileserver (i.e. + ``salt://somefile.txt``) + + show_filenames : True + Set to ``False`` to hide the filenames in the top two lines of the + diff. + + show_changes : True + If set to ``False``, and there are differences, then instead of a diff + a simple message stating that show_changes is set to ``False`` will be + returned. + + template : False + Set to ``True`` if two templates are being compared. This is not useful + except for within states, with the ``obfuscate_templates`` option set + to ``True``. + + .. versionadded:: Oxygen + + CLI Examples: .. code-block:: bash salt '*' file.get_diff /home/fred/.vimrc salt://users/fred/.vimrc + salt '*' file.get_diff /tmp/foo.txt /tmp/bar.txt ''' - minionfile = os.path.expanduser(minionfile) + files = (file1, file2) + paths = [] + errors = [] - ret = '' + for filename in files: + try: + # Local file paths will just return the same path back when passed + # to cp.cache_file. + cached_path = __salt__['cp.cache_file'](filename, saltenv) + if cached_path is False: + errors.append( + u'File {0} not found'.format( + salt.utils.stringutils.to_unicode(filename) + ) + ) + continue + paths.append(cached_path) + except MinionError as exc: + errors.append(salt.utils.stringutils.to_unicode(exc.__str__())) + continue - if not os.path.exists(minionfile): - ret = 'File {0} does not exist on the minion'.format(minionfile) + if errors: + raise CommandExecutionError( + 'Failed to cache one or more files', + info=errors + ) + + args = [] + for idx, filename in enumerate(files): + try: + with salt.utils.files.fopen(filename, 'r') as fp_: + args.append(fp_.readlines()) + except (IOError, OSError) as exc: + raise CommandExecutionError( + 'Failed to read {0}: {1}'.format( + salt.utils.stringutils.to_str(filename), + exc.strerror + ) + ) + + if args[0] != args[1]: + if template and __salt__['config.option']('obfuscate_templates'): + ret = u'' + elif not show_changes: + ret = u'' + else: + bdiff = _binary_replace(*files) + if bdiff: + ret = bdiff + else: + if show_filenames: + args.extend( + [salt.utils.stringutils.to_str(x) for x in files] + ) + ret = salt.utils.locales.sdecode( + ''.join(difflib.unified_diff(*args)) # pylint: disable=no-value-for-parameter + ) return ret - sfn = __salt__['cp.cache_file'](masterfile, saltenv) - if sfn: - with salt.utils.fopen(sfn, 'r') as src: - slines = src.readlines() - with salt.utils.fopen(minionfile, 'r') as name_: - nlines = name_.readlines() - if ''.join(nlines) != ''.join(slines): - bdiff = _binary_replace(minionfile, sfn) - if bdiff: - ret += bdiff - else: - ret += ''.join(difflib.unified_diff(nlines, slines, - minionfile, masterfile)) - else: - ret = 'Failed to copy file from master' - - return ret + return u'' def manage_file(name, @@ -4978,19 +5057,11 @@ def manage_file(name, elif not show_changes: ret['changes']['diff'] = '' else: - # Check to see if the files are bins - bdiff = _binary_replace(real_name, sfn) - if bdiff: - ret['changes']['diff'] = bdiff - else: - with salt.utils.fopen(sfn, 'r') as src: - slines = src.readlines() - with salt.utils.fopen(real_name, 'r') as name_: - nlines = name_.readlines() - - sndiff = ''.join(difflib.unified_diff(nlines, slines)) - if sndiff: - ret['changes']['diff'] = sndiff + try: + ret['changes']['diff'] = get_diff( + real_name, sfn, show_filenames=False) + except CommandExecutionError as exc: + ret['changes']['diff'] = exc.strerror # Pre requisites are met, and the file needs to be replaced, do it try: @@ -5007,35 +5078,29 @@ def manage_file(name, # Write the static contents to a temporary file tmp = salt.utils.files.mkstemp(prefix=salt.utils.files.TEMPFILE_PREFIX, text=True) - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): contents = os.linesep.join( _splitlines_preserving_trailing_newline(contents)) - with salt.utils.fopen(tmp, 'w') as tmp_: + with salt.utils.files.fopen(tmp, 'w') as tmp_: if encoding: log.debug('File will be encoded with {0}'.format(encoding)) tmp_.write(contents.encode(encoding=encoding, errors=encoding_errors)) else: - tmp_.write(str(contents)) + tmp_.write(salt.utils.stringutils.to_str(contents)) - # Compare contents of files to know if we need to replace - with salt.utils.fopen(tmp, 'r') as src: - slines = src.readlines() - with salt.utils.fopen(real_name, 'r') as name_: - nlines = name_.readlines() - different = ''.join(slines) != ''.join(nlines) + try: + differences = get_diff( + real_name, tmp, show_filenames=False, + show_changes=show_changes, template=True) - if different: - if __salt__['config.option']('obfuscate_templates'): - ret['changes']['diff'] = '' - elif not show_changes: - ret['changes']['diff'] = '' - else: - if salt.utils.istextfile(real_name): - ret['changes']['diff'] = \ - ''.join(difflib.unified_diff(nlines, slines)) - else: - ret['changes']['diff'] = \ - 'Replace binary file with text file' + except CommandExecutionError as exc: + ret.setdefault('warnings', []).append( + 'Failed to detect changes to file: {0}'.format(exc.strerror) + ) + differences = '' + + if differences: + ret['changes']['diff'] = differences # Pre requisites are met, the file needs to be replaced, do it try: @@ -5086,7 +5151,7 @@ def manage_file(name, ret['changes']['diff'] = \ 'Replace symbolic link with regular file' - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): ret = check_perms(name, ret, kwargs.get('win_owner'), @@ -5098,7 +5163,9 @@ def manage_file(name, ret, _ = check_perms(name, ret, user, group, mode, attrs, follow_symlinks) if ret['changes']: - ret['comment'] = 'File {0} updated'.format(name) + ret['comment'] = u'File {0} updated'.format( + salt.utils.locales.sdecode(name) + ) elif not ret['changes'] and ret['result']: ret['comment'] = u'File {0} is in the correct state'.format( @@ -5112,7 +5179,7 @@ def manage_file(name, def _set_mode_and_make_dirs(name, dir_mode, mode, user, group): # check for existence of windows drive letter - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): drive, _ = os.path.splitdrive(name) if drive and not os.path.exists(drive): __clean_tmp(sfn) @@ -5129,7 +5196,7 @@ def manage_file(name, mode_list[idx] = str(int(mode_list[idx]) | 1) dir_mode = ''.join(mode_list) - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): # This function resides in win_file.py and will be available # on Windows. The local function will be overridden # pylint: disable=E1121 @@ -5217,15 +5284,15 @@ def manage_file(name, # Write the static contents to a temporary file tmp = salt.utils.files.mkstemp(prefix=salt.utils.files.TEMPFILE_PREFIX, text=True) - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): contents = os.linesep.join( _splitlines_preserving_trailing_newline(contents)) - with salt.utils.fopen(tmp, 'w') as tmp_: + with salt.utils.files.fopen(tmp, 'w') as tmp_: if encoding: log.debug('File will be encoded with {0}'.format(encoding)) tmp_.write(contents.encode(encoding=encoding, errors=encoding_errors)) else: - tmp_.write(str(contents)) + tmp_.write(salt.utils.stringutils.to_str(contents)) # Copy into place salt.utils.files.copyfile(tmp, @@ -5243,14 +5310,14 @@ def manage_file(name, # This is a new file, if no mode specified, use the umask to figure # out what mode to use for the new file. - if mode is None and not salt.utils.is_windows(): + if mode is None and not salt.utils.platform.is_windows(): # Get current umask mask = os.umask(0) os.umask(mask) # Calculate the mode value that results from the umask mode = oct((0o777 ^ mask) & 0o666) - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): ret = check_perms(name, ret, kwargs.get('win_owner'), @@ -5710,7 +5777,7 @@ def list_backups(path, limit=None): bkroot = _get_bkroot() parent_dir, basename = os.path.split(path) - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): # ':' is an illegal filesystem path character on Windows src_dir = parent_dir.replace(':', '_') else: @@ -5724,7 +5791,7 @@ def list_backups(path, limit=None): files = {} for fname in [x for x in os.listdir(bkdir) if os.path.isfile(os.path.join(bkdir, x))]: - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): # ':' is an illegal filesystem path character on Windows strpfmt = '{0}_%a_%b_%d_%H-%M-%S_%f_%Y'.format(basename) else: @@ -5735,7 +5802,7 @@ def list_backups(path, limit=None): # File didn't match the strp format string, so it's not a backup # for this file. Move on to the next one. continue - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): str_format = '%a %b %d %Y %H-%M-%S.%f' else: str_format = '%a %b %d %Y %H:%M:%S.%f' @@ -5865,7 +5932,7 @@ def restore_backup(path, backup_id): '{1}'.format(backup['Location'], path) # Try to set proper ownership - if not salt.utils.is_windows(): + if not salt.utils.platform.is_windows(): try: fstat = os.stat(path) except (OSError, IOError): @@ -5969,9 +6036,9 @@ def grep(path, split_opts = [] for opt in opts: try: - split = salt.utils.shlex_split(opt) + split = salt.utils.args.shlex_split(opt) except AttributeError: - split = salt.utils.shlex_split(str(opt)) + split = salt.utils.args.shlex_split(str(opt)) if len(split) > 1: raise SaltInvocationError( 'Passing multiple command line arguments in a single string ' diff --git a/salt/modules/firewalld.py b/salt/modules/firewalld.py index 4cb0831b901..8cf75b9f88e 100644 --- a/salt/modules/firewalld.py +++ b/salt/modules/firewalld.py @@ -12,7 +12,7 @@ import re # Import Salt Libs from salt.exceptions import CommandExecutionError -import salt.utils +import salt.utils.path log = logging.getLogger(__name__) @@ -21,7 +21,7 @@ def __virtual__(): ''' Check to see if firewall-cmd exists ''' - if salt.utils.which('firewall-cmd'): + if salt.utils.path.which('firewall-cmd'): return True return (False, 'The firewalld execution module cannot be loaded: the firewall-cmd binary is not in the path.') @@ -31,7 +31,7 @@ def __firewall_cmd(cmd): ''' Return the firewall-cmd location ''' - firewall_cmd = '{0} {1}'.format(salt.utils.which('firewall-cmd'), cmd) + firewall_cmd = '{0} {1}'.format(salt.utils.path.which('firewall-cmd'), cmd) out = __salt__['cmd.run_all'](firewall_cmd) if out['retcode'] != 0: diff --git a/salt/modules/freebsd_sysctl.py b/salt/modules/freebsd_sysctl.py index 05119f40c4e..a7a535bd8a5 100644 --- a/salt/modules/freebsd_sysctl.py +++ b/salt/modules/freebsd_sysctl.py @@ -8,12 +8,9 @@ from __future__ import absolute_import import logging # Import salt libs -import salt.utils +import salt.utils.files from salt.exceptions import CommandExecutionError -import logging -log = logging.getLogger(__name__) - # Define the module's virtual name __virtualname__ = 'sysctl' @@ -69,7 +66,7 @@ def show(config_file=False): if config_file: try: - with salt.utils.fopen(config_file, 'r') as f: + with salt.utils.files.fopen(config_file, 'r') as f: for line in f.readlines(): l = line.strip() if l != "" and not l.startswith("#"): @@ -144,7 +141,7 @@ def persist(name, value, config='/etc/sysctl.conf'): edited = False value = str(value) - with salt.utils.fopen(config, 'r') as ifile: + with salt.utils.files.fopen(config, 'r') as ifile: for line in ifile: if not line.startswith('{0}='.format(name)): nlines.append(line) @@ -165,7 +162,7 @@ def persist(name, value, config='/etc/sysctl.conf'): edited = True if not edited: nlines.append("{0}\n".format(_formatfor(name, value, config))) - with salt.utils.fopen(config, 'w+') as ofile: + with salt.utils.files.fopen(config, 'w+') as ofile: ofile.writelines(nlines) if config != '/boot/loader.conf': assign(name, value) diff --git a/salt/modules/freebsd_update.py b/salt/modules/freebsd_update.py index e59d43a2882..9ffb2ac8ea7 100644 --- a/salt/modules/freebsd_update.py +++ b/salt/modules/freebsd_update.py @@ -16,8 +16,8 @@ from __future__ import absolute_import import logging # Import salt libs -import salt -import salt.ext.six as six +import salt.utils.path +from salt.ext import six from salt.exceptions import CommandNotFoundError log = logging.getLogger(__name__) @@ -49,7 +49,7 @@ def _cmd(**kwargs): executed. It checks if any arguments are given to freebsd-update and appends them accordingly. ''' - update_cmd = salt.utils.which('freebsd-update') + update_cmd = salt.utils.path.which('freebsd-update') if not update_cmd: raise CommandNotFoundError('"freebsd-update" command not found') diff --git a/salt/modules/freebsdjail.py b/salt/modules/freebsdjail.py index efd17903a63..9b2e749680f 100644 --- a/salt/modules/freebsdjail.py +++ b/salt/modules/freebsdjail.py @@ -10,7 +10,9 @@ import re import subprocess # Import salt libs -import salt.utils +import salt.utils.args +import salt.utils.files +import salt.utils.stringutils # Define the module's virtual name __virtualname__ = 'jail' @@ -99,7 +101,7 @@ def get_enabled(): ret = [] for rconf in ('/etc/rc.conf', '/etc/rc.conf.local'): if os.access(rconf, os.R_OK): - with salt.utils.fopen(rconf, 'r') as _fp: + with salt.utils.files.fopen(rconf, 'r') as _fp: for line in _fp: if not line.strip(): continue @@ -124,7 +126,7 @@ def show_config(jail): ret = {} if subprocess.call(["jls", "-nq", "-j", jail]) == 0: jls = subprocess.check_output(["jls", "-nq", "-j", jail]) # pylint: disable=minimum-python-version - jailopts = salt.utils.shlex_split(salt.utils.to_str(jls)) + jailopts = salt.utils.args.shlex_split(salt.utils.stringutils.to_str(jls)) for jailopt in jailopts: if '=' not in jailopt: ret[jailopt.strip().rstrip(";")] = '1' @@ -135,7 +137,7 @@ def show_config(jail): else: for rconf in ('/etc/rc.conf', '/etc/rc.conf.local'): if os.access(rconf, os.R_OK): - with salt.utils.fopen(rconf, 'r') as _fp: + with salt.utils.files.fopen(rconf, 'r') as _fp: for line in _fp: if not line.strip(): continue @@ -145,7 +147,7 @@ def show_config(jail): ret[key.split('_', 2)[2]] = value.split('"')[1] for jconf in ('/etc/jail.conf', '/usr/local/etc/jail.conf'): if os.access(jconf, os.R_OK): - with salt.utils.fopen(jconf, 'r') as _fp: + with salt.utils.files.fopen(jconf, 'r') as _fp: for line in _fp: line = line.partition('#')[0].strip() if line: @@ -186,7 +188,7 @@ def fstab(jail): c_fstab = config['mount.fstab'] if 'fstab' in config or 'mount.fstab' in config: if os.access(c_fstab, os.R_OK): - with salt.utils.fopen(c_fstab, 'r') as _fp: + with salt.utils.files.fopen(c_fstab, 'r') as _fp: for line in _fp: line = line.strip() if not line: diff --git a/salt/modules/freebsdkmod.py b/salt/modules/freebsdkmod.py index ae3f05cf3e4..370a65b5336 100644 --- a/salt/modules/freebsdkmod.py +++ b/salt/modules/freebsdkmod.py @@ -9,7 +9,7 @@ import os import re # Import salt libs -import salt.utils +import salt.utils.files # Define the module's virtual name __virtualname__ = 'kmod' @@ -70,7 +70,7 @@ def _get_persistent_modules(): Returns a list of modules in loader.conf that load on boot. ''' mods = set() - with salt.utils.fopen(_LOADER_CONF, 'r') as loader_conf: + with salt.utils.files.fopen(_LOADER_CONF, 'r') as loader_conf: for line in loader_conf: line = line.strip() mod_name = _get_module_name(line) diff --git a/salt/modules/freebsdpkg.py b/salt/modules/freebsdpkg.py index 8d9241a77a7..34d242864fb 100644 --- a/salt/modules/freebsdpkg.py +++ b/salt/modules/freebsdpkg.py @@ -83,7 +83,7 @@ import re import salt.utils import salt.utils.pkg from salt.exceptions import CommandExecutionError, MinionError -import salt.ext.six as six +from salt.ext import six log = logging.getLogger(__name__) diff --git a/salt/modules/freebsdports.py b/salt/modules/freebsdports.py index 36c46bf0d41..1f2c0c3e32f 100644 --- a/salt/modules/freebsdports.py +++ b/salt/modules/freebsdports.py @@ -24,9 +24,10 @@ import logging # Import salt libs import salt.utils +import salt.utils.files from salt.ext.six import string_types from salt.exceptions import SaltInvocationError, CommandExecutionError -import salt.ext.six as six +from salt.ext import six log = logging.getLogger(__name__) @@ -114,7 +115,7 @@ def _write_options(name, configuration): 'Unable to make {0}: {1}'.format(dirname, exc) ) - with salt.utils.fopen(os.path.join(dirname, 'options'), 'w') as fp_: + with salt.utils.files.fopen(os.path.join(dirname, 'options'), 'w') as fp_: sorted_options = list(conf_ptr.keys()) sorted_options.sort() fp_.write( diff --git a/salt/modules/freebsdservice.py b/salt/modules/freebsdservice.py index 6d2dd8e6270..f5d791b0d54 100644 --- a/salt/modules/freebsdservice.py +++ b/salt/modules/freebsdservice.py @@ -17,8 +17,9 @@ import fnmatch import re # Import salt libs -import salt.utils +import salt.utils.path import salt.utils.decorators as decorators +import salt.utils.files from salt.exceptions import CommandNotFoundError __func_alias__ = { @@ -50,11 +51,11 @@ def _cmd(jail=None): Support for jail (representing jid or jail name) keyword argument in kwargs ''' - service = salt.utils.which('service') + service = salt.utils.path.which('service') if not service: raise CommandNotFoundError('\'service\' command not found') if jail: - jexec = salt.utils.which('jexec') + jexec = salt.utils.path.which('jexec') if not jexec: raise CommandNotFoundError('\'jexec\' command not found') service = '{0} {1} {2}'.format(jexec, jail, service) @@ -70,7 +71,7 @@ def _get_jail_path(jail): jail The jid or jail name ''' - jls = salt.utils.which('jls') + jls = salt.utils.path.which('jls') if not jls: raise CommandNotFoundError('\'jls\' command not found') jails = __salt__['cmd.run_stdout']('{0} -n jid name path'.format(jls)) @@ -222,7 +223,7 @@ def _switch(name, # pylint: disable=C0103 val = 'NO' if os.path.exists(config): - with salt.utils.fopen(config, 'r') as ifile: + with salt.utils.files.fopen(config, 'r') as ifile: for line in ifile: if not line.startswith('{0}='.format(rcvar)): nlines.append(line) @@ -236,7 +237,7 @@ def _switch(name, # pylint: disable=C0103 nlines[-1] = '{0}\n'.format(nlines[-1]) nlines.append('{0}="{1}"\n'.format(rcvar, val)) - with salt.utils.fopen(config, 'w') as ofile: + with salt.utils.files.fopen(config, 'w') as ofile: ofile.writelines(nlines) return True diff --git a/salt/modules/gem.py b/salt/modules/gem.py index 32e1e4ba9cd..ade9fa18817 100644 --- a/salt/modules/gem.py +++ b/salt/modules/gem.py @@ -8,13 +8,11 @@ from __future__ import absolute_import import re import logging -# Import salt libs +# Import Salt libs import salt.utils.itertools +import salt.utils.platform from salt.exceptions import CommandExecutionError -# Import salt libs -import salt.utils - __func_alias__ = { 'list_': 'list' } @@ -49,7 +47,7 @@ def _gem(command, ruby=None, runas=None, gem_bin=None): if __salt__['rvm.is_installed'](runas=runas): return __salt__['rvm.do'](ruby, cmdline, runas=runas) - if not salt.utils.is_windows() \ + if not salt.utils.platform.is_windows() \ and __salt__['rbenv.is_installed'](runas=runas): if ruby is None: return __salt__['rbenv.do'](cmdline, runas=runas) diff --git a/salt/modules/genesis.py b/salt/modules/genesis.py index 5680f00829b..ab0eed51384 100644 --- a/salt/modules/genesis.py +++ b/salt/modules/genesis.py @@ -17,13 +17,16 @@ except ImportError: from pipes import quote as _cmd_quote # Import salt libs -import salt.utils -import salt.utils.yast -import salt.utils.preseed -import salt.utils.kickstart import salt.syspaths +import salt.utils.kickstart +import salt.utils.path +import salt.utils.preseed +import salt.utils.validate.path +import salt.utils.yast from salt.exceptions import SaltInvocationError +# Import 3rd-party libs +from salt.ext import six log = logging.getLogger(__name__) @@ -325,6 +328,8 @@ def _bootstrap_yum( ''' if pkgs is None: pkgs = [] + elif isinstance(pkgs, six.string_types): + pkgs = pkgs.split(',') default_pkgs = ('yum', 'centos-release', 'iputils') for pkg in default_pkgs: @@ -333,6 +338,8 @@ def _bootstrap_yum( if exclude_pkgs is None: exclude_pkgs = [] + elif isinstance(exclude_pkgs, six.string_types): + exclude_pkgs = exclude_pkgs.split(',') for pkg in exclude_pkgs: pkgs.remove(pkg) @@ -393,15 +400,32 @@ def _bootstrap_deb( if repo_url is None: repo_url = 'http://ftp.debian.org/debian/' + if not salt.utils.path.which('debootstrap'): + log.error('Required tool debootstrap is not installed.') + return False + + if static_qemu and not salt.utils.validate.path.is_executable(static_qemu): + log.error('Required tool qemu not ' + 'present/readable at: {0}'.format(static_qemu)) + return False + + if isinstance(pkgs, (list, tuple)): + pkgs = ','.join(pkgs) + if isinstance(exclude_pkgs, (list, tuple)): + exclude_pkgs = ','.join(exclude_pkgs) + deb_args = [ 'debootstrap', '--foreign', '--arch', - _cmd_quote(arch), - '--include', - ] + pkgs + [ - '--exclude', - ] + exclude_pkgs + [ + _cmd_quote(arch)] + + if pkgs: + deb_args += ['--include', _cmd_quote(pkgs)] + if exclude_pkgs: + deb_args += ['--exclude', _cmd_quote(exclude_pkgs)] + + deb_args += [ _cmd_quote(flavor), _cmd_quote(root), _cmd_quote(repo_url), @@ -409,11 +433,13 @@ def _bootstrap_deb( __salt__['cmd.run'](deb_args, python_shell=False) - __salt__['cmd.run']( - 'cp {qemu} {root}/usr/bin/'.format( - qemu=_cmd_quote(static_qemu), root=_cmd_quote(root) + if static_qemu: + __salt__['cmd.run']( + 'cp {qemu} {root}/usr/bin/'.format( + qemu=_cmd_quote(static_qemu), root=_cmd_quote(root) + ) ) - ) + env = {'DEBIAN_FRONTEND': 'noninteractive', 'DEBCONF_NONINTERACTIVE_SEEN': 'true', 'LC_ALL': 'C', @@ -469,6 +495,8 @@ def _bootstrap_pacman( if pkgs is None: pkgs = [] + elif isinstance(pkgs, six.string_types): + pkgs = pkgs.split(',') default_pkgs = ('pacman', 'linux', 'systemd-sysvcompat', 'grub') for pkg in default_pkgs: @@ -477,6 +505,8 @@ def _bootstrap_pacman( if exclude_pkgs is None: exclude_pkgs = [] + elif isinstance(exclude_pkgs, six.string_types): + exclude_pkgs = exclude_pkgs.split(',') for pkg in exclude_pkgs: pkgs.remove(pkg) @@ -549,7 +579,7 @@ def avail_platforms(): for platform in CMD_MAP: ret[platform] = True for cmd in CMD_MAP[platform]: - if not salt.utils.which(cmd): + if not salt.utils.path.which(cmd): ret[platform] = False return ret @@ -663,7 +693,7 @@ def ldd_deps(filename, ret=None): salt myminion genesis.ldd_deps /bin/bash ''' if not os.path.exists(filename): - filename = salt.utils.which(filename) + filename = salt.utils.path.which(filename) if ret is None: ret = [] diff --git a/salt/modules/git.py b/salt/modules/git.py index 38d86d4f1cc..d52922ea34d 100644 --- a/salt/modules/git.py +++ b/salt/modules/git.py @@ -12,8 +12,11 @@ import re # Import salt libs import salt.utils +import salt.utils.args import salt.utils.files import salt.utils.itertools +import salt.utils.path +import salt.utils.platform import salt.utils.url from salt.exceptions import SaltInvocationError, CommandExecutionError from salt.utils.versions import LooseVersion as _LooseVersion @@ -30,7 +33,7 @@ def __virtual__(): ''' Only load if git exists on the system ''' - if salt.utils.which('git') is None: + if salt.utils.path.which('git') is None: return (False, 'The git execution module cannot be loaded: git unavailable.') else: @@ -64,10 +67,10 @@ def _config_getter(get_opt, Common code for config.get_* functions, builds and runs the git CLI command and returns the result dict for the calling function to parse. ''' - kwargs = salt.utils.clean_kwargs(**kwargs) + kwargs = salt.utils.args.clean_kwargs(**kwargs) global_ = kwargs.pop('global', False) if kwargs: - salt.utils.invalid_kwargs(kwargs) + salt.utils.args.invalid_kwargs(kwargs) if cwd is None: if not global_: @@ -133,7 +136,7 @@ def _format_opts(opts): if not isinstance(opts, six.string_types): opts = [str(opts)] else: - opts = salt.utils.shlex_split(opts) + opts = salt.utils.args.shlex_split(opts) try: if opts[-1] == '--': # Strip the '--' if it was passed at the end of the opts string, @@ -185,15 +188,24 @@ def _git_run(command, cwd=None, user=None, password=None, identity=None, identity = [identity] # try each of the identities, independently + tmp_identity_file = None for id_file in identity: if 'salt://' in id_file: - _id_file = id_file - id_file = __salt__['cp.cache_file'](id_file, saltenv) + with salt.utils.files.set_umask(0o077): + tmp_identity_file = salt.utils.mkstemp() + _id_file = id_file + id_file = __salt__['cp.get_file'](id_file, + tmp_identity_file, + saltenv) if not id_file: log.error('identity {0} does not exist.'.format(_id_file)) + __salt__['file.remove'](tmp_identity_file) continue else: - __salt__['file.set_mode'](id_file, '0600') + if user: + os.chown(id_file, + __salt__['file.user_to_uid'](user), + -1) else: if not __salt__['file.file_exists'](id_file): missing_keys.append(id_file) @@ -210,7 +222,7 @@ def _git_run(command, cwd=None, user=None, password=None, identity=None, salt.utils.templates.TEMPLATE_DIRNAME, 'git/ssh-id-wrapper' ) - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): for suffix in ('', ' (x86)'): ssh_exe = ( 'C:\\Program Files{0}\\Git\\bin\\ssh.exe' @@ -261,9 +273,14 @@ def _git_run(command, cwd=None, user=None, password=None, identity=None, redirect_stderr=redirect_stderr, **kwargs) finally: - if not salt.utils.is_windows() and 'GIT_SSH' in env: + if not salt.utils.platform.is_windows() and 'GIT_SSH' in env: os.remove(env['GIT_SSH']) + # Cleanup the temporary identity file + if tmp_identity_file and os.path.exists(tmp_identity_file): + log.debug('Removing identity file {0}'.format(tmp_identity_file)) + __salt__['file.remove'](tmp_identity_file) + # If the command was successful, no need to try additional IDs if result['retcode'] == 0: return result @@ -560,10 +577,10 @@ def archive(cwd, # allows us to accept 'format' as an argument to this function without # shadowing the format() global, while also not allowing unwanted arguments # to be passed. - kwargs = salt.utils.clean_kwargs(**kwargs) + kwargs = salt.utils.args.clean_kwargs(**kwargs) format_ = kwargs.pop('format', None) if kwargs: - salt.utils.invalid_kwargs(kwargs) + salt.utils.args.invalid_kwargs(kwargs) command = ['git'] + _format_git_opts(git_opts) command.append('archive') @@ -921,7 +938,7 @@ def clone(cwd, # https://github.com/saltstack/salt/issues/15519#issuecomment-128531310 # On Windows, just fall back to None (runs git clone command using the # home directory as the cwd). - clone_cwd = '/tmp' if not salt.utils.is_windows() else None + clone_cwd = '/tmp' if not salt.utils.platform.is_windows() else None _git_run(command, cwd=clone_cwd, user=user, @@ -1267,11 +1284,11 @@ def config_set(key, salt myminion git.config_set user.email me@example.com cwd=/path/to/repo salt myminion git.config_set user.email foo@bar.com global=True ''' - kwargs = salt.utils.clean_kwargs(**kwargs) + kwargs = salt.utils.args.clean_kwargs(**kwargs) add_ = kwargs.pop('add', False) global_ = kwargs.pop('global', False) if kwargs: - salt.utils.invalid_kwargs(kwargs) + salt.utils.args.invalid_kwargs(kwargs) if cwd is None: if not global_: @@ -1394,11 +1411,11 @@ def config_unset(key, salt myminion git.config_unset /path/to/repo foo.bar salt myminion git.config_unset /path/to/repo foo.bar all=True ''' - kwargs = salt.utils.clean_kwargs(**kwargs) + kwargs = salt.utils.args.clean_kwargs(**kwargs) all_ = kwargs.pop('all', False) global_ = kwargs.pop('global', False) if kwargs: - salt.utils.invalid_kwargs(kwargs) + salt.utils.args.invalid_kwargs(kwargs) if cwd is None: if not global_: @@ -2039,7 +2056,7 @@ def is_worktree(cwd, return False gitdir = os.path.join(toplevel, '.git') try: - with salt.utils.fopen(gitdir, 'r') as fp_: + with salt.utils.files.fopen(gitdir, 'r') as fp_: for line in fp_: try: label, path = line.split(None, 1) @@ -2222,10 +2239,10 @@ def list_worktrees(cwd, if not _check_worktree_support(failhard=True): return {} cwd = _expand_path(cwd, user) - kwargs = salt.utils.clean_kwargs(**kwargs) + kwargs = salt.utils.args.clean_kwargs(**kwargs) all_ = kwargs.pop('all', False) if kwargs: - salt.utils.invalid_kwargs(kwargs) + salt.utils.args.invalid_kwargs(kwargs) if all_ and stale: raise CommandExecutionError( @@ -2381,7 +2398,7 @@ def list_worktrees(cwd, Return contents of a single line file with EOF newline stripped ''' try: - with salt.utils.fopen(path, 'r') as fp_: + with salt.utils.files.fopen(path, 'r') as fp_: for line in fp_: ret = line.strip() # Ignore other lines, if they exist (which they @@ -2679,9 +2696,9 @@ def merge(cwd, # .. or merge another rev salt myminion git.merge /path/to/repo rev=upstream/foo ''' - kwargs = salt.utils.clean_kwargs(**kwargs) + kwargs = salt.utils.args.clean_kwargs(**kwargs) if kwargs: - salt.utils.invalid_kwargs(kwargs) + salt.utils.args.invalid_kwargs(kwargs) cwd = _expand_path(cwd, user) command = ['git'] + _format_git_opts(git_opts) @@ -2811,10 +2828,10 @@ def merge_base(cwd, salt myminion git.merge_base /path/to/repo refs=mybranch fork_point=upstream/master ''' cwd = _expand_path(cwd, user) - kwargs = salt.utils.clean_kwargs(**kwargs) + kwargs = salt.utils.args.clean_kwargs(**kwargs) all_ = kwargs.pop('all', False) if kwargs: - salt.utils.invalid_kwargs(kwargs) + salt.utils.args.invalid_kwargs(kwargs) if all_ and (independent or is_ancestor or fork_point): raise SaltInvocationError( @@ -3177,9 +3194,9 @@ def push(cwd, # Delete remote branch 'upstream/temp' salt myminion git.push /path/to/repo upstream :temp ''' - kwargs = salt.utils.clean_kwargs(**kwargs) + kwargs = salt.utils.args.clean_kwargs(**kwargs) if kwargs: - salt.utils.invalid_kwargs(kwargs) + salt.utils.args.invalid_kwargs(kwargs) cwd = _expand_path(cwd, user) command = ['git'] + _format_git_opts(git_opts) @@ -3269,7 +3286,7 @@ def rebase(cwd, command.extend(opts) if not isinstance(rev, six.string_types): rev = str(rev) - command.extend(salt.utils.shlex_split(rev)) + command.extend(salt.utils.args.shlex_split(rev)) return _git_run(command, cwd=cwd, user=user, @@ -4189,10 +4206,10 @@ def submodule(cwd, # Unregister submodule (2015.8.0 and later) salt myminion git.submodule /path/to/repo/sub/repo deinit ''' - kwargs = salt.utils.clean_kwargs(**kwargs) + kwargs = salt.utils.args.clean_kwargs(**kwargs) init_ = kwargs.pop('init', False) if kwargs: - salt.utils.invalid_kwargs(kwargs) + salt.utils.args.invalid_kwargs(kwargs) cwd = _expand_path(cwd, user) if init_: @@ -4454,10 +4471,10 @@ def worktree_add(cwd, salt myminion git.worktree_add /path/to/repo/main ../hotfix branch=hotfix21 ref=v2.1.9.3 ''' _check_worktree_support() - kwargs = salt.utils.clean_kwargs(**kwargs) + kwargs = salt.utils.args.clean_kwargs(**kwargs) branch_ = kwargs.pop('branch', None) if kwargs: - salt.utils.invalid_kwargs(kwargs) + salt.utils.args.invalid_kwargs(kwargs) cwd = _expand_path(cwd, user) if branch_ and detach: @@ -4639,7 +4656,7 @@ def worktree_rm(cwd, user=None): elif not is_worktree(cwd): raise CommandExecutionError(cwd + ' is not a git worktree') try: - salt.utils.rm_rf(cwd) + salt.utils.files.rm_rf(cwd) except Exception as exc: raise CommandExecutionError( 'Unable to remove {0}: {1}'.format(cwd, exc) diff --git a/salt/modules/github.py b/salt/modules/github.py index b8ea1fcb779..8bbf155d6af 100644 --- a/salt/modules/github.py +++ b/salt/modules/github.py @@ -36,7 +36,7 @@ import logging # Import Salt Libs from salt.exceptions import CommandExecutionError -import salt.ext.six as six +from salt.ext import six import salt.utils.http # Import third party libs diff --git a/salt/modules/glusterfs.py b/salt/modules/glusterfs.py index fb26e1c6102..ce25ed63d8a 100644 --- a/salt/modules/glusterfs.py +++ b/salt/modules/glusterfs.py @@ -10,10 +10,13 @@ import sys import xml.etree.ElementTree as ET # Import salt libs -import salt.utils -import salt.utils.cloud as suc +import salt.utils.cloud +import salt.utils.path from salt.exceptions import SaltInvocationError, CommandExecutionError +# Import 3rd-party libs +from salt.ext import six + log = logging.getLogger(__name__) @@ -21,7 +24,7 @@ def __virtual__(): ''' Only load this module if the gluster command exists ''' - if salt.utils.which('gluster'): + if salt.utils.path.which('gluster'): return True return (False, 'glusterfs server is not installed') @@ -201,7 +204,7 @@ def peer(name): ''' - if suc.check_name(name, 'a-zA-Z0-9._-'): + if salt.utils.cloud.check_name(name, 'a-zA-Z0-9._-'): raise SaltInvocationError( 'Invalid characters in peer name "{0}"'.format(name)) @@ -254,7 +257,7 @@ def create_volume(name, bricks, stripe=False, replica=False, device_vg=False, "gluster2:/export/vol2/brick"]' replica=2 start=True ''' # If single brick given as a string, accept it - if isinstance(bricks, str): + if isinstance(bricks, six.string_types): bricks = [bricks] # Error for block devices with multiple bricks @@ -531,7 +534,7 @@ def add_volume_bricks(name, bricks): cmd = 'volume add-brick {0}'.format(name) - if isinstance(bricks, str): + if isinstance(bricks, six.string_types): bricks = [bricks] volume_bricks = [x['path'] for x in volinfo[name]['bricks'].values()] diff --git a/salt/modules/gpg.py b/salt/modules/gpg.py index ae76e6867d2..ac3427edd29 100644 --- a/salt/modules/gpg.py +++ b/salt/modules/gpg.py @@ -21,12 +21,13 @@ import re import time # Import salt libs -import salt.utils +import salt.utils.files +import salt.utils.path from salt.exceptions import SaltInvocationError from salt.utils.versions import LooseVersion as _LooseVersion # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six # Set up logging log = logging.getLogger(__name__) @@ -85,7 +86,7 @@ def _gpg(): Returns the path to the gpg binary ''' # Get the path to the gpg binary. - return salt.utils.which('gpg') + return salt.utils.path.which('gpg') def __virtual__(): @@ -726,7 +727,7 @@ def import_key(text=None, if filename: try: - with salt.utils.flopen(filename, 'rb') as _fp: + with salt.utils.files.flopen(filename, 'rb') as _fp: lines = _fp.readlines() text = ''.join(lines) except IOError: @@ -1019,13 +1020,13 @@ def sign(user=None, else: signed_data = gpg.sign(text, keyid=keyid, passphrase=gpg_passphrase) elif filename: - with salt.utils.flopen(filename, 'rb') as _fp: + with salt.utils.files.flopen(filename, 'rb') as _fp: if gnupg_version >= '1.3.1': signed_data = gpg.sign(text, default_key=keyid, passphrase=gpg_passphrase) else: signed_data = gpg.sign_file(_fp, keyid=keyid, passphrase=gpg_passphrase) if output: - with salt.utils.flopen(output, 'w') as fout: + with salt.utils.files.flopen(output, 'w') as fout: fout.write(signed_data.data) else: raise SaltInvocationError('filename or text must be passed.') @@ -1056,7 +1057,7 @@ def verify(text=None, signature Specify the filename of a detached signature. - .. versionadded:: Nitrogen + .. versionadded:: Oxygen CLI Example: @@ -1077,10 +1078,10 @@ def verify(text=None, if signature: # need to call with fopen instead of flopen due to: # https://bitbucket.org/vinay.sajip/python-gnupg/issues/76/verify_file-closes-passed-file-handle - with salt.utils.fopen(signature, 'rb') as _fp: + with salt.utils.files.fopen(signature, 'rb') as _fp: verified = gpg.verify_file(_fp, filename) else: - with salt.utils.flopen(filename, 'rb') as _fp: + with salt.utils.files.flopen(filename, 'rb') as _fp: verified = gpg.verify_file(_fp) else: raise SaltInvocationError('filename or text must be passed.') @@ -1173,12 +1174,12 @@ def encrypt(user=None, if GPG_1_3_1: # This version does not allow us to encrypt using the # file stream # have to read in the contents and encrypt. - with salt.utils.flopen(filename, 'rb') as _fp: + with salt.utils.files.flopen(filename, 'rb') as _fp: _contents = _fp.read() result = gpg.encrypt(_contents, recipients, passphrase=gpg_passphrase, output=output) else: # This version allows encrypting the file stream - with salt.utils.flopen(filename, 'rb') as _fp: + with salt.utils.files.flopen(filename, 'rb') as _fp: if output: result = gpg.encrypt_file(_fp, recipients, passphrase=gpg_passphrase, output=output, sign=sign) else: @@ -1264,7 +1265,7 @@ def decrypt(user=None, if text: result = gpg.decrypt(text, passphrase=gpg_passphrase) elif filename: - with salt.utils.flopen(filename, 'rb') as _fp: + with salt.utils.files.flopen(filename, 'rb') as _fp: if output: result = gpg.decrypt_file(_fp, passphrase=gpg_passphrase, output=output) else: diff --git a/salt/modules/grains.py b/salt/modules/grains.py index cf719f9492e..e59b4b10051 100644 --- a/salt/modules/grains.py +++ b/salt/modules/grains.py @@ -6,26 +6,24 @@ Return/control aspects of the grains data # Import python libs from __future__ import absolute_import, print_function import os -import copy -import math import random import logging import operator import collections import json +import math +import yaml from functools import reduce # pylint: disable=redefined-builtin -# Import 3rd-party libs -import yaml -import salt.utils.compat -from salt.utils.odict import OrderedDict -import salt.ext.six as six -from salt.ext.six.moves import range # pylint: disable=import-error,no-name-in-module,redefined-builtin - -# Import salt libs +# Import Salt libs +from salt.ext import six import salt.utils +import salt.utils.compat +import salt.utils.files +import salt.utils.yamldumper from salt.defaults import DEFAULT_TARGET_DELIM from salt.exceptions import SaltException +from salt.ext.six.moves import range __proxyenabled__ = ['*'] @@ -236,7 +234,7 @@ def setvals(grains, destructive=False): ) if os.path.isfile(gfn): - with salt.utils.fopen(gfn, 'rb') as fp_: + with salt.utils.files.fopen(gfn, 'rb') as fp_: try: grains = yaml.safe_load(fp_.read()) except yaml.YAMLError as exc: @@ -252,32 +250,16 @@ def setvals(grains, destructive=False): else: grains[key] = val __grains__[key] = val - # Cast defaultdict to dict; is there a more central place to put this? + cstr = salt.utils.yamldumper.safe_dump(grains, default_flow_style=False) try: - yaml_reps = copy.deepcopy(yaml.representer.SafeRepresenter.yaml_representers) - yaml_multi_reps = copy.deepcopy(yaml.representer.SafeRepresenter.yaml_multi_representers) - except (TypeError, NameError): - # This likely means we are running under Python 2.6 which cannot deepcopy - # bound methods. Fallback to a modification of deepcopy which can support - # this behavior. - yaml_reps = salt.utils.compat.deepcopy_bound(yaml.representer.SafeRepresenter.yaml_representers) - yaml_multi_reps = salt.utils.compat.deepcopy_bound(yaml.representer.SafeRepresenter.yaml_multi_representers) - yaml.representer.SafeRepresenter.add_representer(collections.defaultdict, - yaml.representer.SafeRepresenter.represent_dict) - yaml.representer.SafeRepresenter.add_representer(OrderedDict, - yaml.representer.SafeRepresenter.represent_dict) - cstr = yaml.safe_dump(grains, default_flow_style=False) - yaml.representer.SafeRepresenter.yaml_representers = yaml_reps - yaml.representer.SafeRepresenter.yaml_multi_representers = yaml_multi_reps - try: - with salt.utils.fopen(gfn, 'w+') as fp_: + with salt.utils.files.fopen(gfn, 'w+') as fp_: fp_.write(cstr) except (IOError, OSError): msg = 'Unable to write to grains file at {0}. Check permissions.' log.error(msg.format(gfn)) fn_ = os.path.join(__opts__['cachedir'], 'module_refresh') try: - with salt.utils.flopen(fn_, 'w+') as fp_: + with salt.utils.files.flopen(fn_, 'w+') as fp_: fp_.write('') except (IOError, OSError): msg = 'Unable to write to cache file {0}. Check permissions.' diff --git a/salt/modules/groupadd.py b/salt/modules/groupadd.py index 2f79c47dd90..f02a5811a7b 100644 --- a/salt/modules/groupadd.py +++ b/salt/modules/groupadd.py @@ -31,7 +31,7 @@ def __virtual__(): if __grains__['kernel'] in ('Linux', 'OpenBSD', 'NetBSD'): return __virtualname__ return (False, 'The groupadd execution module cannot be loaded: ' - ' only available on Linux, OpenBSD and NetBSD') + ' only available on Linux, OpenBSD and NetBSD') def add(name, gid=None, system=False, root=None): @@ -44,12 +44,12 @@ def add(name, gid=None, system=False, root=None): salt '*' group.add foo 3456 ''' - cmd = 'groupadd ' + cmd = ['groupadd'] if gid: - cmd += '-g {0} '.format(gid) + cmd.append('-g {0}'.format(gid)) if system and __grains__['kernel'] != 'OpenBSD': - cmd += '-r ' - cmd += name + cmd.append('-r') + cmd.append(name) if root is not None: cmd.extend(('-R', root)) @@ -69,7 +69,7 @@ def delete(name, root=None): salt '*' group.delete foo ''' - cmd = ('groupdel', name) + cmd = ['groupdel', name] if root is not None: cmd.extend(('-R', root)) @@ -140,7 +140,7 @@ def chgid(name, gid, root=None): pre_gid = __salt__['file.group_to_gid'](name) if gid == pre_gid: return True - cmd = ('groupmod', '-g', gid, name) + cmd = ['groupmod', '-g', gid, name] if root is not None: cmd.extend(('-R', root)) @@ -170,15 +170,15 @@ def adduser(name, username, root=None): if __grains__['kernel'] == 'Linux': if on_redhat_5: - cmd = ('gpasswd', '-a', username, name) + cmd = ['gpasswd', '-a', username, name] elif on_suse_11: - cmd = ('usermod', '-A', name, username) + cmd = ['usermod', '-A', name, username] else: - cmd = ('gpasswd', '--add', username, name) + cmd = ['gpasswd', '--add', username, name] if root is not None: cmd.extend(('-Q', root)) else: - cmd = ('usermod', '-G', name, username) + cmd = ['usermod', '-G', name, username] if root is not None: cmd.extend(('-R', root)) @@ -208,20 +208,20 @@ def deluser(name, username, root=None): if username in grp_info['members']: if __grains__['kernel'] == 'Linux': if on_redhat_5: - cmd = ('gpasswd', '-d', username, name) + cmd = ['gpasswd', '-d', username, name] elif on_suse_11: - cmd = ('usermod', '-R', name, username) + cmd = ['usermod', '-R', name, username] else: - cmd = ('gpasswd', '--del', username, name) + cmd = ['gpasswd', '--del', username, name] if root is not None: cmd.extend(('-R', root)) retcode = __salt__['cmd.retcode'](cmd, python_shell=False) elif __grains__['kernel'] == 'OpenBSD': out = __salt__['cmd.run_stdout']('id -Gn {0}'.format(username), python_shell=False) - cmd = 'usermod -S ' - cmd += ','.join([g for g in out.split() if g != str(name)]) - cmd += ' {0}'.format(username) + cmd = ['usermod', '-S'] + cmd.append(','.join([g for g in out.split() if g != str(name)])) + cmd.append('{0}'.format(username)) retcode = __salt__['cmd.retcode'](cmd, python_shell=False) else: log.error('group.deluser is not yet supported on this platform') @@ -249,13 +249,13 @@ def members(name, members_list, root=None): if __grains__['kernel'] == 'Linux': if on_redhat_5: - cmd = ('gpasswd', '-M', members_list, name) + cmd = ['gpasswd', '-M', members_list, name] elif on_suse_11: for old_member in __salt__['group.info'](name).get('members'): __salt__['cmd.run']('groupmod -R {0} {1}'.format(old_member, name), python_shell=False) - cmd = ('groupmod', '-A', members_list, name) + cmd = ['groupmod', '-A', members_list, name] else: - cmd = ('gpasswd', '--members', members_list, name) + cmd = ['gpasswd', '--members', members_list, name] if root is not None: cmd.extend(('-R', root)) retcode = __salt__['cmd.retcode'](cmd, python_shell=False) @@ -270,7 +270,7 @@ def members(name, members_list, root=None): for user in members_list.split(","): if user: retcode = __salt__['cmd.retcode']( - 'usermod -G {0} {1}'.format(name, user), + ['usermod', '-G', name, user], python_shell=False) if not retcode == 0: break diff --git a/salt/modules/grub_legacy.py b/salt/modules/grub_legacy.py index 4b5f9918bd9..d636a77d1b9 100644 --- a/salt/modules/grub_legacy.py +++ b/salt/modules/grub_legacy.py @@ -8,7 +8,7 @@ from __future__ import absolute_import import os # Import salt libs -import salt.utils +import salt.utils.files import salt.utils.decorators as decorators from salt.exceptions import CommandExecutionError @@ -68,7 +68,7 @@ def conf(): ret = {} pos = 0 try: - with salt.utils.fopen(_detect_conf(), 'r') as _fp: + with salt.utils.files.fopen(_detect_conf(), 'r') as _fp: for line in _fp: if line.startswith('#'): continue diff --git a/salt/modules/guestfs.py b/salt/modules/guestfs.py index d521a935e63..a9f80a693f2 100644 --- a/salt/modules/guestfs.py +++ b/salt/modules/guestfs.py @@ -13,7 +13,7 @@ import hashlib import logging # Import Salt libs -import salt.utils +import salt.utils.path log = logging.getLogger(__name__) @@ -22,7 +22,7 @@ def __virtual__(): ''' Only load if libguestfs python bindings are installed ''' - if salt.utils.which('guestmount'): + if salt.utils.path.which('guestmount'): return 'guestfs' return (False, 'The guestfs execution module cannot be loaded: guestmount binary not in path.') diff --git a/salt/modules/hadoop.py b/salt/modules/hadoop.py index 3a29c498214..65668619bb9 100644 --- a/salt/modules/hadoop.py +++ b/salt/modules/hadoop.py @@ -12,7 +12,7 @@ Support for hadoop from __future__ import absolute_import # Import salt libs -import salt.utils +import salt.utils.path __authorized_modules__ = ['version', 'namenode', 'dfsadmin', 'dfs', 'fs'] @@ -21,7 +21,7 @@ def __virtual__(): ''' Check if hadoop is present, then load the module ''' - if salt.utils.which('hadoop') or salt.utils.which('hdfs'): + if salt.utils.path.which('hadoop') or salt.utils.path.which('hdfs'): return 'hadoop' return (False, 'The hadoop execution module cannot be loaded: hadoop or hdfs binary not in path.') @@ -40,7 +40,7 @@ def _hadoop_cmd(module, command, *args): E.g.: hadoop dfs -ls / ''' tool = 'hadoop' - if salt.utils.which('hdfs'): + if salt.utils.path.which('hdfs'): tool = 'hdfs' out = None diff --git a/salt/modules/hashutil.py b/salt/modules/hashutil.py index 0c9b7effe34..f7fb98cc1d5 100644 --- a/salt/modules/hashutil.py +++ b/salt/modules/hashutil.py @@ -11,9 +11,11 @@ import hmac # Import Salt libs import salt.exceptions -import salt.ext.six as six +from salt.ext import six import salt.utils +import salt.utils.files import salt.utils.hashutils +import salt.utils.stringutils if six.PY2: import StringIO @@ -72,7 +74,7 @@ def digest_file(infile, checksum='md5'): raise salt.exceptions.CommandExecutionError( "File path '{0}' not found.".format(infile)) - with salt.utils.fopen(infile, 'rb') as f: + with salt.utils.files.fopen(infile, 'rb') as f: file_hash = __salt__['hashutil.digest'](f.read(), checksum) return file_hash @@ -156,7 +158,7 @@ def base64_encodefile(fname): ''' encoded_f = StringIO.StringIO() - with salt.utils.fopen(fname, 'rb') as f: + with salt.utils.files.fopen(fname, 'rb') as f: base64.encode(f, encoded_f) encoded_f.seek(0) @@ -193,7 +195,7 @@ def base64_decodefile(instr, outfile): ''' encoded_f = StringIO.StringIO(instr) - with salt.utils.fopen(outfile, 'wb') as f: + with salt.utils.files.fopen(outfile, 'wb') as f: base64.decode(encoded_f, f) return True @@ -280,7 +282,7 @@ def github_signature(string, shared_secret, challenge_hmac): key = shared_secret hashtype, challenge = challenge_hmac.split('=') if six.PY3: - msg = salt.utils.to_bytes(msg) - key = salt.utils.to_bytes(key) + msg = salt.utils.stringutils.to_bytes(msg) + key = salt.utils.stringutils.to_bytes(key) hmac_hash = hmac.new(key, msg, getattr(hashlib, hashtype)) return hmac_hash.hexdigest() == challenge diff --git a/salt/modules/heat.py b/salt/modules/heat.py index 1f94f2e605e..d3e01f8d9ae 100644 --- a/salt/modules/heat.py +++ b/salt/modules/heat.py @@ -45,11 +45,10 @@ from __future__ import absolute_import import time import json import logging -# Import third party libs import yaml -# Import salt libs -import salt.ext.six as six -import salt.utils + +# Import Salt libs +from salt.ext import six import salt.utils.files from salt.exceptions import SaltInvocationError @@ -543,9 +542,9 @@ def create_stack(name=None, template_file=None, enviroment=None, contents=None, dir_mode=None) if template_manage_result['result']: - with salt.utils.fopen(template_tmp_file, 'r') as tfp_: + with salt.utils.files.fopen(template_tmp_file, 'r') as tfp_: tpl = tfp_.read() - salt.utils.safe_rm(template_tmp_file) + salt.utils.files.safe_rm(template_tmp_file) try: if isinstance(tpl, six.binary_type): tpl = tpl.decode('utf-8') @@ -605,9 +604,9 @@ def create_stack(name=None, template_file=None, enviroment=None, contents=None, dir_mode=None) if enviroment_manage_result['result']: - with salt.utils.fopen(enviroment_tmp_file, 'r') as efp_: + with salt.utils.files.fopen(enviroment_tmp_file, 'r') as efp_: env_str = efp_.read() - salt.utils.safe_rm(enviroment_tmp_file) + salt.utils.files.safe_rm(enviroment_tmp_file) try: env = _parse_enviroment(env_str) except ValueError as ex: @@ -732,9 +731,9 @@ def update_stack(name=None, template_file=None, enviroment=None, contents=None, dir_mode=None) if template_manage_result['result']: - with salt.utils.fopen(template_tmp_file, 'r') as tfp_: + with salt.utils.files.fopen(template_tmp_file, 'r') as tfp_: tpl = tfp_.read() - salt.utils.safe_rm(template_tmp_file) + salt.utils.files.safe_rm(template_tmp_file) try: if isinstance(tpl, six.binary_type): tpl = tpl.decode('utf-8') @@ -794,9 +793,9 @@ def update_stack(name=None, template_file=None, enviroment=None, contents=None, dir_mode=None) if enviroment_manage_result['result']: - with salt.utils.fopen(enviroment_tmp_file, 'r') as efp_: + with salt.utils.files.fopen(enviroment_tmp_file, 'r') as efp_: env_str = efp_.read() - salt.utils.safe_rm(enviroment_tmp_file) + salt.utils.files.safe_rm(enviroment_tmp_file) try: env = _parse_enviroment(env_str) except ValueError as ex: diff --git a/salt/modules/hg.py b/salt/modules/hg.py index 3ff524cf32a..d4992eb34a5 100644 --- a/salt/modules/hg.py +++ b/salt/modules/hg.py @@ -10,6 +10,7 @@ import logging # Import salt libs from salt.exceptions import CommandExecutionError import salt.utils +import salt.utils.path log = logging.getLogger(__name__) @@ -18,7 +19,7 @@ def __virtual__(): ''' Only load if hg is installed ''' - if salt.utils.which('hg') is None: + if salt.utils.path.which('hg') is None: return (False, 'The hg execution module cannot be loaded: hg unavailable.') else: diff --git a/salt/modules/highstate_doc.py b/salt/modules/highstate_doc.py new file mode 100644 index 00000000000..8aed0a361c5 --- /dev/null +++ b/salt/modules/highstate_doc.py @@ -0,0 +1,712 @@ +# -*- coding: utf-8 -*- + +# pylint: disable=W1401 +''' +This module renders highstate configuration into a more human readable format. + +How it works: + + `highstate or lowstate` data is parsed with a `proccesser` this defaults to `highstate_doc.proccesser_markdown`. + The proccessed data is passed to a `jinja` template that builds up the document content. + + +configuration: Pillar + +.. code-block:: yaml + + # the following defaults can be overrided + highstate_doc.config: + + # list of regex of state names to ignore in `highstate_doc.proccess_lowstates` + filter_id_regex: + - '.*!doc_skip$' + + # list of regex of state functions to ignore in `highstate_doc.proccess_lowstates` + filter_state_function_regex: + - 'file.accumulated' + + # dict of regex to replace text after `highstate_doc.render`. (remove passwords) + text_replace_regex: + 'password:.*^': '[PASSWORD]' + + # limit size of files that can be included in doc (10000 bytes) + max_render_file_size: 10000 + + # advanced option to set a custom lowstate proccesser + proccesser: highstate_doc.proccesser_markdown + + +State example + +.. code-block:: yaml + + {{sls}} note: + highstate_doc.note: + - name: example + - order: 0 + - contents: | + example `highstate_doc.note` + ------------------ + This state does not do anything to the system! It is only used by a `proccesser` + you can use `requisites` and `order` to move your docs around the rendered file. + + {{sls}} a file we dont want in the doc !doc_skip: + file.managed: + - name: /root/passwords + - contents: 'password: sadefgq34y45h56q' + # also could use `highstate_doc.config: text_replace_regex` to replace + # password string. `password:.*^': '[PASSWORD]` + + +To create the help document build a State that uses `highstate_doc.render`. +For preformance it's advised to not included this state in your `top.sls` file. + +.. code-block:: yaml + + # example `salt://makereadme.sls` + make helpfile: + file.managed: + - name: /root/README.md + - contents: {{salt.highstate_doc.render()|json}} + - show_diff: {{opts['test']}} + - mode: '0640' + - order: last + +Run our `makereadme.sls` state to create `/root/README.md`. + +.. code-block:: bash + + # first ensure `highstate` return without errors or changes + salt-call state.highstate + salt-call state.apply makereadme + # or if you dont want the extra `make helpfile` state + salt-call --out=newline_values_only salt.highstate_doc.render > /root/README.md ; chmod 0600 /root/README.md + + +Creating a document collection +------------------------------ + +From the master we can run the following script to +creates a collection of all your minion documents. + +.. code-block:: bash + + salt '*' state.apply makereadme + +.. code-block:: python + + #!/bin/python + import os + import salt.client + s = salt.client.LocalClient() + # NOTE: because of issues with `cp.push` use `highstate_doc.read_file` + o = s.cmd('*', 'highstate_doc.read_file', ['/root/README.md']) + for m in o: + d = o.get(m) + if d and not d.endswith('is not available.'): + # mkdir m + #directory = os.path.dirname(file_path) + if not os.path.exists(m): + os.makedirs(m) + with open(m + '/README.md','wb') as f: + f.write(d) + print('ADDED: ' + m + '/README.md') + + +Once the master has a collection of all the README files. +You can use pandoc to create HTML versions of the markdown. + +.. code-block:: bash + + # proccess all the readme.md files to readme.html + if which pandoc; then echo "Found pandoc"; else echo "** Missing pandoc"; exit 1; fi + if which gs; then echo "Found gs"; else echo "** Missing gs(ghostscript)"; exit 1; fi + readme_files=$(find $dest -type f -path "*/README.md" -print) + for f in $readme_files ; do + ff=${f#$dest/} + minion=${ff%%/*} + echo "proccess: $dest/${minion}/$(basename $f)" + cat $dest/${minion}/$(basename $f) | \ + pandoc --standalone --from markdown_github --to html \ + --include-in-header $dest/style.html \ + > $dest/${minion}/$(basename $f).html + done + +It is also nice to put the help files in source control. + + # git init + git add -A + git commit -am 'updated docs' + git push -f + + +Other hints +----------- + +If you wish to customize the document format: + +.. code-block:: yaml + + # you could also create a new `proccesser` for perhaps reStructuredText + # highstate_doc.config: + # proccesser: doc_custom.proccesser_rst + + # example `salt://makereadme.jinja` + """ + {{opts['id']}} + ========================================== + + {# lowstates is set from highstate_doc.render() #} + {# if lowstates is missing use salt.highstate_doc.proccess_lowstates() #} + {% for s in lowstates %} + {{s.id}} + ----------------------------------------------------------------- + {{s.function}} + + {{s.markdown.requisite}} + {{s.markdown.details}} + + {%- endfor %} + """ + + # example `salt://makereadme.sls` + {% import_text "makereadme.jinja" as makereadme %} + {{sls}} or: + file.managed: + - name: /root/README_other.md + - contents: {{salt.highstate_doc.render(jinja_template_text=makereadme)|json}} + - mode: '0640' + + +Some `replace_text_regex` values that might be helpfull. + + ## CERTS + '-----BEGIN RSA PRIVATE KEY-----[\r\n\t\f\S]{0,2200}': 'XXXXXXX' + '-----BEGIN CERTIFICATE-----[\r\n\t\f\S]{0,2200}': 'XXXXXXX' + '-----BEGIN DH PARAMETERS-----[\r\n\t\f\S]{0,2200}': 'XXXXXXX' + '-----BEGIN PRIVATE KEY-----[\r\n\t\f\S]{0,2200}': 'XXXXXXX' + '-----BEGIN OPENSSH PRIVATE KEY-----[\r\n\t\f\S]{0,2200}': 'XXXXXXX' + 'ssh-rsa .* ': 'ssh-rsa XXXXXXX ' + 'ssh-dss .* ': 'ssh-dss XXXXXXX ' + ## DB + 'DB_PASS.*': 'DB_PASS = XXXXXXX' + '5432:*:*:.*': '5432:*:XXXXXXX' + "'PASSWORD': .*": "'PASSWORD': 'XXXXXXX'," + " PASSWORD '.*'": " PASSWORD 'XXXXXXX'" + 'PGPASSWORD=.* ': 'PGPASSWORD=XXXXXXX' + "_replication password '.*'": "_replication password 'XXXXXXX'" + ## OTHER + 'EMAIL_HOST_PASSWORD =.*': 'EMAIL_HOST_PASSWORD =XXXXXXX' + "net ads join -U '.*@MFCFADS.MATH.EXAMPLE.CA.* ": "net ads join -U '.*@MFCFADS.MATH.EXAMPLE.CA%XXXXXXX " + "net ads join -U '.*@NEXUS.EXAMPLE.CA.* ": "net ads join -U '.*@NEXUS.EXAMPLE.CA%XXXXXXX " + 'install-uptrack .* --autoinstall': 'install-uptrack XXXXXXX --autoinstall' + 'accesskey = .*': 'accesskey = XXXXXXX' + 'auth_pass .*': 'auth_pass XXXXXXX' + 'PSK "0x.*': 'PSK "0xXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' + 'SECRET_KEY.*': 'SECRET_KEY = XXXXXXX' + "password=.*": "password=XXXXXXX" + '.*': 'XXXXXXX' + '.*': 'XXXXXXX' + 'application.secret = ".*"': 'application.secret = "XXXXXXX"' + 'url = "postgres://.*"': 'url = "postgres://XXXXXXX"' + 'PASS_.*_PASS': 'PASS_XXXXXXX_PASS' + ## HTACCESS + ':{PLAIN}.*': ':{PLAIN}XXXXXXX' + +''' + +from __future__ import absolute_import + +import re +import yaml +import logging + +import salt.utils.files +import salt.utils.templates as tpl +from salt.utils.yamldumper import OrderedDumper + + +__virtualname__ = 'highstate_doc' +log = logging.getLogger(__name__) + + +markdown_basic_jinja_template_txt = """ +{% for s in lowstates %} +`{{s.id_full}}` +----------------------------------------------------------------- + * state: {{s.state_function}} + * name: `{{s.name}}` + +{{s.markdown.requisites}} +{{s.markdown.details}} + +{%- endfor %} +""" + +markdown_default_jinja_template_txt = """ +Configuration Managment +=============================================================================== + +``` +#################################################### +fqdn: {{grains.get('fqdn')}} +os: {{grains.get('os')}} +osfinger: {{grains.get('osfinger')}} +mem_total: {{grains.get('mem_total')}}MB +num_cpus: {{grains.get('num_cpus')}} +ipv4: {{grains.get('ipv4')}} +master: {{opts.get('master')}} +#################################################### +``` + +This system is fully or partly managed using Salt. + +The following sections are a rendered view of what the configuration management system +controlled on this system. Each item is handled in order from top to bottom unless some +requisites like `require` force other ordering. + +""" + markdown_basic_jinja_template_txt + + +markdown_advanced_jinja_template_txt = markdown_default_jinja_template_txt + """ + +{% if vars.get('doc_other', True) -%} +Other information +===================================================================================== + +``` + +salt grain: ip_interfaces +----------------------------------------------------------------- +{{grains['ip_interfaces']|dictsort}} + + +salt grain: hwaddr_interfaces +----------------------------------------------------------------- +{{grains['hwaddr_interfaces']|dictsort}} + +{% if not grains['os'] == 'Windows' %} + +{% if salt['cmd.has_exec']('ip') -%} +# ip address show +----------------------------------------------------------------- +{{salt['cmd.run']('ip address show | sed "/valid_lft/d"')}} + + +# ip route list table all +----------------------------------------------------------------- +{{salt['cmd.run']('ip route list table all')}} +{% endif %} + +{% if salt['cmd.has_exec']('iptables') %} +{%- if salt['cmd.has_exec']('iptables-save') -%} +# iptables-save +----------------------------------------------------------------- +{{salt['cmd.run']("iptables --list > /dev/null; iptables-save | \grep -v -F '#' | sed '/^:/s@\[[0-9]\{1,\}:[0-9]\{1,\}\]@[0:0]@g'")}} + + +# ip6tables-save +----------------------------------------------------------------- +{{salt['cmd.run']("ip6tables --list > /dev/null; ip6tables-save | \grep -v -F '#' | sed '/^:/s@\[[0-9]\{1,\}:[0-9]\{1,\}\]@[0:0]@g'")}} +{%- else -%} +# iptables --list-rules +----------------------------------------------------------------- +{{salt['cmd.run']('iptables --list-rules')}} + + +# ip6tables --list-rules +----------------------------------------------------------------- +{{salt['cmd.run']('ip6tables --list-rules')}} +{% endif %} +{% endif %} + +{% if salt['cmd.has_exec']('firewall-cmd') -%} +# firewall-cmd --list-all +----------------------------------------------------------------- +{{salt['cmd.run']('firewall-cmd --list-all')}} +{% endif %} + +# mount +----------------------------------------------------------------- +{{salt['cmd.run']('mount')}} + +{% endif %} +""" + + +def markdown_basic_jinja_template(**kwargs): + ''' + Return text for a simple markdown jinja template + + This function can be used from the `highstate_doc.render` modules `jinja_template_function` option. + ''' + return markdown_basic_jinja_template_txt + + +def markdown_default_jinja_template(**kwargs): + ''' + Return text for a markdown jinja template that included a header + + This function can be used from the `highstate_doc.render` modules `jinja_template_function` option. + ''' + return markdown_default_jinja_template_txt + + +def markdown_full_jinja_template(**kwargs): + ''' + Return text for an advanced markdown jinja template + + This function can be used from the `highstate_doc.render` modules `jinja_template_function` option. + ''' + return markdown_advanced_jinja_template_txt + + +def _get_config(**kwargs): + ''' + Return configuration + ''' + config = { + 'filter_id_regex': ['.*!doc_skip'], + 'filter_function_regex': [], + 'replace_text_regex': {}, + 'proccesser': 'highstate_doc.proccesser_markdown', + 'max_render_file_size': 10000, + 'note': None + } + if '__salt__' in globals(): + config_key = '{0}.config'.format(__virtualname__) + config.update(__salt__['config.get'](config_key, {})) + # pylint: disable=C0201 + for k in set(config.keys()) & set(kwargs.keys()): + config[k] = kwargs[k] + return config + + +def read_file(name): + ''' + output the contents of a file: + + this is a workaround if the cp.push module does not work. + https://github.com/saltstack/salt/issues/37133 + + help the master output the contents of a document + that might be saved on the minions filesystem. + + .. code-block:: python + + #!/bin/python + import os + import salt.client + s = salt.client.LocalClient() + o = s.cmd('*', 'highstate_doc.read_file', ['/root/README.md']) + for m in o: + d = o.get(m) + if d and not d.endswith('is not available.'): + # mkdir m + #directory = os.path.dirname(file_path) + if not os.path.exists(m): + os.makedirs(m) + with open(m + '/README.md','wb') as fin: + fin.write(d) + print('ADDED: ' + m + '/README.md') + ''' + out = '' + try: + with salt.utils.files.fopen(name, 'r') as f: + out = f.read() + except Exception as ex: + log.error(ex) + return None + return out + + +def render(jinja_template_text=None, jinja_template_function='highstate_doc.markdown_default_jinja_template', **kwargs): + ''' + Render highstate to a text format (default Markdown) + + if `jinja_template_text` is not set, `jinja_template_function` is used. + + jinja_template_text: jinja text that the render uses to create the document. + jinja_template_function: a salt module call that returns template text. + options: + highstate_doc.markdown_basic_jinja_template + highstate_doc.markdown_default_jinja_template + highstate_doc.markdown_full_jinja_template + ''' + config = _get_config(**kwargs) + lowstates = proccess_lowstates(**kwargs) + #saltenv = __env__ + #__opts__['enviroment'] + #TODO: __env__, + context = { + 'saltenv': None, + 'config': config, + 'lowstates': lowstates, + 'salt': __salt__, + 'pillar': __pillar__, + 'grains': __grains__, + 'opts': __opts__, + 'kwargs': kwargs, + } + template_text = jinja_template_text + if template_text is None and jinja_template_function: + template_text = __salt__[jinja_template_function](**kwargs) + if template_text is None: + raise Exception('No jinja template text') + + txt = tpl.render_jinja_tmpl(template_text, context, tmplpath=None) + # after proccessing the template replace passwords or other data. + rt = config.get('replace_text_regex') + for r in rt: + txt = re.sub(r, rt[r], txt) + return txt + + +def _blacklist_filter(s, config): + ss = s['state'] + sf = s['fun'] + state_function = '{0}.{1}'.format(s['state'], s['fun']) + for b in config['filter_function_regex']: + if re.match(b, state_function): + return True + for b in config['filter_id_regex']: + if re.match(b, s['__id__']): + return True + return False + + +def proccess_lowstates(**kwargs): + ''' + return proccessed lowstate data that was not blacklisted + + render_module_function is used to provide your own. + defaults to from_lowstate + ''' + states = [] + config = _get_config(**kwargs) + proccesser = config.get('proccesser') + ls = __salt__['state.show_lowstate']() + + if not isinstance(ls, list): + raise Exception('ERROR: to see details run: [salt-call state.show_lowstate] <-----***-SEE-***') + else: + if len(ls) > 0: + if not isinstance(ls[0], dict): + raise Exception('ERROR: to see details run: [salt-call state.show_lowstate] <-----***-SEE-***') + + for s in ls: + if _blacklist_filter(s, config): + continue + doc = __salt__[proccesser](s, config, **kwargs) + states.append(doc) + return states + + +def _state_data_to_yaml_string(data, whitelist=None, blacklist=None): + ''' + return a data dict in yaml string format. + ''' + y = {} + if blacklist is None: + # TODO: use salt defined STATE_REQUISITE_IN_KEYWORDS STATE_RUNTIME_KEYWORDS STATE_INTERNAL_KEYWORDS + blacklist = ['__env__', '__id__', '__sls__', 'fun', 'name', 'context', 'order', 'state', 'require', 'require_in', 'watch', 'watch_in'] + kset = set(data.keys()) + if blacklist: + kset -= set(blacklist) + if whitelist: + kset &= set(whitelist) + for k in kset: + y[k] = data[k] + if len(y) == 0: + return None + y = yaml.dump(y, Dumper=OrderedDumper, default_flow_style=False) + return y + + +def _md_fix(text): + ''' + sanitize text data that is to be displayed in a markdown code block + ''' + return text.replace('```', '``[`][markdown parse fix]') + + +def _format_markdown_system_file(filename, config): + ret = '' + file_stats = __salt__['file.stats'](filename) + y = _state_data_to_yaml_string(file_stats, whitelist=['user', 'group', 'mode', 'uid', 'gid', 'size']) + if y: + ret += 'file stat {1}\n```\n{0}```\n'.format(y, filename) + file_size = file_stats.get('size') + if file_size <= config.get('max_render_file_size'): + is_binary = True + try: + ## TODO: this is linux only should find somthing portable + file_type = __salt__['cmd.shell']('\\file -i \'{0}\''.format(filename)) + if 'charset=binary' not in file_type: + is_binary = False + except Exception as ex: + # likly on a windows system, set as not binary for now. + is_binary = False + if is_binary: + file_data = '[[skipped binary data]]' + else: + with salt.utils.files.fopen(filename, 'r') as f: + file_data = f.read() + #file_data = __salt__['cmd.shell']('\\file -i \'{0}\' | \\grep -q \'charset=binary\' && echo [[binary data]] || cat \'{0}\''.format(filename)) + file_data = _md_fix(file_data) + ret += 'file data {1}\n```\n{0}\n```\n'.format(file_data, filename) + else: + ret += '```\n{0}\n```\n'.format('SKIPPED LARGE FILE!\nSet {0}:max_render_file_size > {1} to render.'.format('{0}.config'.format(__virtualname__), file_size)) + return ret + + +def _format_markdown_link(name): + link = name + symbals = '~`!@#$%^&*()+={}[]:;"<>,.?/|\'\\' + for s in symbals: + link = link.replace(s, '') + link = link.replace(' ', '-') + return link + + +def _format_markdown_requisite(state, stateid, makelink=True): + ''' + format requisite as a link users can click + ''' + fmt_id = '{0}: {1}'.format(state, stateid) + if makelink: + return ' * [{0}](#{1})\n'.format(fmt_id, _format_markdown_link(fmt_id)) + else: + return ' * `{0}`\n'.format(fmt_id) + + +def proccesser_markdown(lowstate_item, config, **kwargs): + ''' + Takes low state data and returns a dict of proccessed data + that is by default used in a jinja template when rendering a markdown highstate_doc. + + This `lowstate_item_markdown` given a lowstate item, returns a dict like: + + .. code-block:: yaml + + vars: # the raw lowstate_item that was proccessed + id: # the 'id' of the state. + id_full: # combo of the state type and id "state: id" + state: # name of the salt state module + function: # name of the state function + name: # value of 'name:' passed to the salt state module + state_function: # the state name and function name + markdown: # text data to describe a state + requisites: # requisite like [watch_in, require_in] + details: # state name, parameters and other details like file contents + + ''' + ## TODO: switch or ... ext call. + s = lowstate_item + state_function = '{0}.{1}'.format(s['state'], s['fun']) + id_full = '{0}: {1}'.format(s['state'], s['__id__']) + + # requisites + # ------------ + # TODO: use salt defined STATE_REQUISITE_IN_KEYWORDS + requisites = '' + if s.get('watch'): + requisites += 'run or update after changes in:\n' + for w in s.get('watch', []): + requisites += _format_markdown_requisite(w.items()[0][0], w.items()[0][1]) + requisites += '\n' + if s.get('watch_in'): + requisites += 'after changes, run or update:\n' + for w in s.get('watch_in', []): + requisites += _format_markdown_requisite(w.items()[0][0], w.items()[0][1]) + requisites += '\n' + if s.get('require') and len(s.get('require')) > 0: + requisites += 'require:\n' + for w in s.get('require', []): + requisites += _format_markdown_requisite(w.items()[0][0], w.items()[0][1]) + requisites += '\n' + if s.get('require_in'): + requisites += 'required in:\n' + for w in s.get('require_in', []): + requisites += _format_markdown_requisite(w.items()[0][0], w.items()[0][1]) + requisites += '\n' + + # details + # ------------ + details = '' + + if state_function == 'highstate_doc.note': + if 'contents' in s: + details += '\n{0}\n'.format(s['contents']) + if 'source' in s: + text = __salt__['cp.get_file_str'](s['source']) + if text: + #details += '\n`file: {0}`\n{1}\n'.format(s['source'], text) + details += '\n{0}\n'.format(text) + else: + details += '\n{0}\n'.format('ERROR: opening {0}'.format(s['source'])) + + if state_function == 'pkg.installed': + pkgs = s.get('pkgs', s.get('name')) + #if isinstance(pkgs, list): + # pkgs = ' '.join(pkgs) + details += '\n```\ninstall: {0}\n```\n'.format(pkgs) + + #if state_function == 'cmd.run': + # details += 'run raw shell command\n```\n{0}\n```\n'.format(s['name']) + + #if state_function == 'cmd.wait': + # details += 'run raw shell command\n```\n{0}\n```\n'.format(s['name']) + + #if state_function == 'service.running': + #d['txt'] += 'name: {0}\n'.format(s['name']) + + if state_function == 'file.recurse': + details += '''recurse copy of files\n''' + y = _state_data_to_yaml_string(s) + if y: + details += '```\n{0}\n```\n'.format(y) + if '!doc_recurse' in id_full: + findfiles = __salt__['file.find'](path=s.get('name'), type='f') + if len(findfiles) < 10 or '!doc_recurse_force' in id_full: + for f in findfiles: + details += _format_markdown_system_file(f, config) + else: + details += ''' > Skipping because more than 10 files to display.\n''' + details += ''' > HINT: to force include !doc_recurse_force in state id.\n''' + else: + details += ''' > For more details review logs and Salt state files.\n\n''' + details += ''' > HINT: for improved docs use multiple file.managed states or file.archive, git.latest. etc.\n''' + details += ''' > HINT: to force doc to show all files in path add !doc_recurse .\n''' + + if state_function == 'file.blockreplace': + if s.get('content'): + details += 'ensure block of content is in file\n```\n{0}\n```\n'.format(_md_fix(s['content'])) + if s.get('source'): + text = '** source: ' + s.get('source') + details += 'ensure block of content is in file\n```\n{0}\n```\n'.format(_md_fix(text)) + + if state_function == 'file.managed': + details += _format_markdown_system_file(s['name'], config) + + # if no state doc is created use default state as yaml + if len(details) == 0: + y = _state_data_to_yaml_string(s) + if y: + details += '```\n{0}```\n'.format(y) + + # ------------ + r = { + 'vars': lowstate_item, + 'state': s['state'], + 'name': s['name'], + 'function': s['fun'], + 'id': s['__id__'], + 'id_full': id_full, + 'state_function': state_function, + 'markdown': { + 'requisites': requisites.decode('utf-8'), + 'details': details.decode('utf-8') + } + } + return r diff --git a/salt/modules/hosts.py b/salt/modules/hosts.py index 0d7162c6ad0..953345fa351 100644 --- a/salt/modules/hosts.py +++ b/salt/modules/hosts.py @@ -8,11 +8,11 @@ from __future__ import absolute_import import os # Import salt libs -import salt.utils +import salt.utils.files import salt.utils.odict as odict # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import range # pylint: disable=import-error,no-name-in-module,redefined-builtin @@ -33,7 +33,7 @@ def _get_or_create_hostfile(): if hfn is None: hfn = '' if not os.path.exists(hfn): - with salt.utils.fopen(hfn, 'w'): + with salt.utils.files.fopen(hfn, 'w'): pass return hfn @@ -47,7 +47,7 @@ def _list_hosts(): ret = odict.OrderedDict() if not os.path.isfile(hfn): return ret - with salt.utils.fopen(hfn) as ifile: + with salt.utils.files.fopen(hfn) as ifile: for line in ifile: line = line.strip() if not line: @@ -161,7 +161,7 @@ def set_host(ip, alias): if not alias.strip(): line_to_add = '' - with salt.utils.fopen(hfn) as fp_: + with salt.utils.files.fopen(hfn) as fp_: lines = fp_.readlines() for ind, line in enumerate(lines): tmpline = line.strip() @@ -182,7 +182,7 @@ def set_host(ip, alias): lines[-1] += os.linesep line = line_to_add lines.append(line) - with salt.utils.fopen(hfn, 'w+') as ofile: + with salt.utils.files.fopen(hfn, 'w+') as ofile: ofile.writelines(lines) return True @@ -200,7 +200,7 @@ def rm_host(ip, alias): if not has_pair(ip, alias): return True hfn = _get_or_create_hostfile() - with salt.utils.fopen(hfn) as fp_: + with salt.utils.files.fopen(hfn) as fp_: lines = fp_.readlines() for ind in range(len(lines)): tmpline = lines[ind].strip() @@ -221,7 +221,7 @@ def rm_host(ip, alias): else: # Only an alias was removed lines[ind] = newline + os.linesep - with salt.utils.fopen(hfn, 'w+') as ofile: + with salt.utils.files.fopen(hfn, 'w+') as ofile: ofile.writelines(lines) return True @@ -271,7 +271,7 @@ def _write_hosts(hosts): lines.append(line) hfn = _get_or_create_hostfile() - with salt.utils.fopen(hfn, 'w+') as ofile: + with salt.utils.files.fopen(hfn, 'w+') as ofile: for line in lines: if line.strip(): # /etc/hosts needs to end with EOL so that some utils that read diff --git a/salt/modules/htpasswd.py b/salt/modules/htpasswd.py index 02f48457c2d..60e2ad7c10b 100644 --- a/salt/modules/htpasswd.py +++ b/salt/modules/htpasswd.py @@ -14,7 +14,7 @@ import os import logging # Import salt libs -import salt.utils +import salt.utils.path log = logging.getLogger(__name__) @@ -25,7 +25,7 @@ def __virtual__(): ''' Only load the module if htpasswd is installed ''' - if salt.utils.which('htpasswd'): + if salt.utils.path.which('htpasswd'): return __virtualname__ return (False, 'The htpasswd execution mdule cannot be loaded: htpasswd binary not in path.') @@ -35,8 +35,6 @@ def useradd(pwfile, user, password, opts='', runas=None): Add a user to htpasswd file using the htpasswd command. If the htpasswd file does not exist, it will be created. - .. deprecated:: 2016.3.0 - pwfile Path to htpasswd file diff --git a/salt/modules/http.py b/salt/modules/http.py index 09378f22a83..77c764416a5 100644 --- a/salt/modules/http.py +++ b/salt/modules/http.py @@ -30,7 +30,12 @@ def query(url, **kwargs): salt '*' http.query http://somelink.com/ method=POST \ data='somecontent' ''' - return salt.utils.http.query(url=url, opts=__opts__, **kwargs) + opts = __opts__.copy() + if 'opts' in kwargs: + opts.update(kwargs['opts']) + del kwargs['opts'] + + return salt.utils.http.query(url=url, opts=opts, **kwargs) def wait_for_successful_query(url, wait_for=300, **kwargs): diff --git a/salt/modules/icinga2.py b/salt/modules/icinga2.py index 81f442dfc55..899c39f7cfb 100644 --- a/salt/modules/icinga2.py +++ b/salt/modules/icinga2.py @@ -13,7 +13,8 @@ import logging import subprocess # Import Salt libs -import salt.utils +import salt.utils.path +import salt.utils.platform log = logging.getLogger(__name__) @@ -23,10 +24,10 @@ def __virtual__(): Only load this module if the mysql libraries exist ''' # TODO: This could work on windows with some love - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): return (False, 'The module cannot be loaded on windows.') - if salt.utils.which('icinga2'): + if salt.utils.path.which('icinga2'): return True return (False, 'Icinga2 not installed.') diff --git a/salt/modules/ilo.py b/salt/modules/ilo.py index a740d9c0e59..6a4139afec7 100644 --- a/salt/modules/ilo.py +++ b/salt/modules/ilo.py @@ -7,7 +7,7 @@ Manage HP ILO from __future__ import absolute_import from salt._compat import ElementTree as ET -import salt.utils +import salt.utils.path import os import tempfile @@ -20,7 +20,7 @@ def __virtual__(): ''' Make sure hponcfg tool is accessible ''' - if salt.utils.which('hponcfg'): + if salt.utils.path.which('hponcfg'): return True return (False, 'ilo execution module not loaded: the hponcfg binary is not in the path.') diff --git a/salt/modules/incron.py b/salt/modules/incron.py index bec92b79b1b..5782943b933 100644 --- a/salt/modules/incron.py +++ b/salt/modules/incron.py @@ -102,7 +102,7 @@ def _write_incron_lines(user, lines): return ret else: path = salt.utils.files.mkstemp() - with salt.utils.fopen(path, 'w+') as fp_: + with salt.utils.files.fopen(path, 'w+') as fp_: fp_.writelines(lines) if __grains__['os_family'] == 'Solaris' and user != "root": __salt__['cmd.run']('chown {0} {1}'.format(user, path), python_shell=False) @@ -121,7 +121,7 @@ def _write_file(folder, filename, data): msg = msg.format(filename, folder) log.error(msg) raise AttributeError(msg) - with salt.utils.fopen(path, 'w') as fp_: + with salt.utils.files.fopen(path, 'w') as fp_: fp_.write(data) return 0 @@ -133,7 +133,7 @@ def _read_file(folder, filename): ''' path = os.path.join(folder, filename) try: - with salt.utils.fopen(path, 'rb') as contents: + with salt.utils.files.fopen(path, 'rb') as contents: return contents.readlines() except (OSError, IOError): return '' diff --git a/salt/modules/infoblox.py b/salt/modules/infoblox.py index f51fccd7745..939cdf3ca09 100644 --- a/salt/modules/infoblox.py +++ b/salt/modules/infoblox.py @@ -1,508 +1,611 @@ # -*- coding: utf-8 -*- ''' -Module for managing Infoblox +This module have been tested on infoblox API v1.2.1, +other versions of the API are likly workable. -Will look for pillar data infoblox:server, infoblox:user, infoblox:password if not passed to functions +:depends: libinfoblox, https://github.com/steverweber/libinfoblox -.. versionadded:: 2016.3.0 + libinfoblox can be installed using `pip install libinfoblox` + +API documents can be found on your infoblox server at: + + https://INFOBLOX/wapidoc + +:configuration: The following configuration defaults can be + defined (pillar or config files '/etc/salt/master.d/infoblox.conf'): + + .. code-block:: python + + infoblox.config: + api_sslverify: True + api_url: 'https://INFOBLOX/wapi/v1.2.1' + api_user: 'username' + api_key: 'password' + + Many of the functions accept `api_opts` to override the API config. + + .. code-block:: bash + + salt-call infoblox.get_host name=my.host.com \ + api_url: 'https://INFOBLOX/wapi/v1.2.1' \ + api_user=admin \ + api_key=passs -:depends: - - requests ''' from __future__ import absolute_import -# Import salt libs -from salt.exceptions import CommandExecutionError -from salt.exceptions import SaltInvocationError -import logging - -log = logging.getLogger(__name__) +import time +IMPORT_ERR = None try: - import json - import requests - HAS_IMPORTS = True -except ImportError: - HAS_IMPORTS = False + import libinfoblox +except Exception as ex: + IMPORT_ERR = str(ex) +__virtualname__ = 'infoblox' def __virtual__(): - if HAS_IMPORTS: - return True - return (False, 'The infoblox execution module cannot be loaded: ' - 'python requests and/or json libraries are not available.') + return (IMPORT_ERR is None, IMPORT_ERR) -def _conn_info_check(infoblox_server=None, - infoblox_user=None, - infoblox_password=None): +cache = {} + + +def _get_config(**api_opts): ''' - get infoblox stuff from pillar if not passed + Return configuration + user passed api_opts override salt config.get vars ''' - - if infoblox_server is None: - infoblox_server = __salt__['pillar.get']('infoblox:server', None) - if infoblox_user is None: - infoblox_user = __salt__['pillar.get']('infoblox:user', None) - log.debug('Infoblox username is "{0}"'.format(infoblox_user)) - if infoblox_password is None: - infoblox_password = __salt__['pillar.get']('infoblox:password', None) - - return infoblox_server, infoblox_user, infoblox_password + config = { + 'api_sslverify': True, + 'api_url': 'https://INFOBLOX/wapi/v1.2.1', + 'api_user': '', + 'api_key': '', + } + if '__salt__' in globals(): + config_key = '{0}.config'.format(__virtualname__) + config.update(__salt__['config.get'](config_key, {})) + # pylint: disable=C0201 + for k in set(config.keys()) & set(api_opts.keys()): + config[k] = api_opts[k] + return config -def _process_return_data(retData): +def _get_infoblox(**api_opts): + config = _get_config(**api_opts) + # TODO: perhaps cache in __opts__ + cache_key = 'infoblox_session_{0},{1},{2}'.format( + config['api_url'], config['api_user'], config['api_key']) + if cache_key in cache: + timedelta = int(time.time()) - cache[cache_key]['time'] + if cache[cache_key]['obj'] and timedelta < 60: + return cache[cache_key]['obj'] + c = {} + c['time'] = int(time.time()) + c['obj'] = libinfoblox.Session(api_sslverify=config['api_sslverify'], api_url=config['api_url'], + api_user=config['api_user'], api_key=config['api_key']) + cache[cache_key] = c + return c['obj'] + + +def diff_objects(obja, objb): ''' - generic return processing + Diff two complex infoblox objects. + This is used from salt states to detect changes in objects. + + Using `func:nextavailableip` will not cause a diff if the ipaddres is in range ''' - if retData.status_code == 200: - if retData.json(): - return retData + return libinfoblox.diff_obj(obja, objb) + + +def is_ipaddr_in_ipfunc_range(ipaddr, ipfunc): + ''' + Return true if the ipaddress is in the range of the nextavailableip function + + CLI Example: + + salt-call infoblox.is_ipaddr_in_ipfunc_range \ + ipaddr="10.0.2.2" ipfunc="func:nextavailableip:10.0.0.0/8" + ''' + return libinfoblox.is_ipaddr_in_ipfunc_range(ipaddr, ipfunc) + + +def update_host(name, data, **api_opts): + ''' + Update host record. This is a helper call to update_object. + + Find a hosts `_ref` then call update_object with the record data. + + CLI Example: + + salt-call infoblox.update_host name=fqdn data={} + ''' + o = get_host(name=name, **api_opts) + return update_object(objref=o['_ref'], data=data, **api_opts) + + +def update_object(objref, data, **api_opts): + ''' + Update raw infoblox object. + This is a low level api call. + + CLI Example: + + .. code-block:: bash + + salt-call infoblox.update_object objref=[ref_of_object] data={} + ''' + if '__opts__' in globals() and __opts__['test']: + return {'Test': 'Would attempt to update object: {0}'.format(objref)} + infoblox = _get_infoblox(**api_opts) + return infoblox.update_object(objref, data) + + +def delete_object(objref, **api_opts): + ''' + Delete infoblox object. + This is a low level api call. + + CLI Example: + + salt-call infoblox.delete_object objref=[ref_of_object] + ''' + if '__opts__' in globals() and __opts__['test']: + return {'Test': 'Would attempt to delete object: {0}'.format(objref)} + infoblox = _get_infoblox(**api_opts) + return infoblox.delete_object(objref) + + +def create_object(object_type, data, **api_opts): + ''' + Create raw infoblox object + This is a low level api call. + + CLI Example: + + salt-call infoblox.update_object object_type=record:host data={} + ''' + if '__opts__' in globals() and __opts__['test']: + return {'Test': 'Would attempt to create object: {0}'.format(object_type)} + infoblox = _get_infoblox(**api_opts) + return infoblox.create_object(object_type, data) + + +def get_object(objref, data=None, return_fields=None, max_results=None, + ensure_none_or_one_result=False, **api_opts): + ''' + Get raw infoblox object. + This is a low level api call. + + CLI Example: + + salt-call infoblox.get_object objref=[_ref of object] + ''' + if not data: + data = {} + infoblox = _get_infoblox(**api_opts) + return infoblox.get_object(objref, data, return_fields, + max_results, ensure_none_or_one_result) + + +def create_cname(data, **api_opts): + ''' + Create a cname record. + + CLI Example: + + salt-call infoblox.create_cname data={ \ + "comment": "cname to example server", \ + "name": "example.example.com", \ + "zone": "example.com", \ + "view": "Internal", \ + "canonical": "example-ha-0.example.com" \ + } + ''' + infoblox = _get_infoblox(**api_opts) + host = infoblox.create_cname(data=data) + return host + + +def get_cname(name=None, canonical=None, return_fields=None, **api_opts): + ''' + Get CNAME information. + + CLI Example: + + salt-call infoblox.get_cname name=example.example.com + salt-call infoblox.get_cname canonical=example-ha-0.example.com + ''' + infoblox = _get_infoblox(**api_opts) + o = infoblox.get_cname(name=name, canonical=canonical, return_fields=return_fields) + return o + + +def update_cname(name, data, **api_opts): + ''' + Update CNAME. This is a helper call to update_object. + + Find a CNAME `_ref` then call update_object with the record data. + + CLI Example: + + salt-call infoblox.update_cname name=example.example.com data="{ + 'canonical':'example-ha-0.example.com', + 'use_ttl':true, + 'ttl':200, + 'comment':'Salt managed CNAME'}" + ''' + o = get_cname(name=name, **api_opts) + if not o: + raise Exception('CNAME record not found') + return update_object(objref=o['_ref'], data=data, **api_opts) + + +def delete_cname(name=None, canonical=None, **api_opts): + ''' + Delete CNAME. This is a helper call to delete_object. + + If record is not found, return True + + CLI Example: + + salt-call infoblox.delete_cname name=example.example.com + salt-call infoblox.delete_cname canonical=example-ha-0.example.com + ''' + cname = get_cname(name=name, canonical=canonical, **api_opts) + if cname: + return delete_object(cname['_ref'], **api_opts) + return True + + +def get_host(name=None, ipv4addr=None, mac=None, return_fields=None, **api_opts): + ''' + Get host information + + CLI Example: + + salt-call infoblox.get_host hostname.domain.ca + salt-call infoblox.get_host ipv4addr=123.123.122.12 + salt-call infoblox.get_host mac=00:50:56:84:6e:ae + + return_fields= + https://INFOBLOX/wapidoc/objects/record.host.html#fields-list + + return_fields='ipv4addrs,aliases,name,configure_for_dns,extattrs,disable,view,comment,zone' + ''' + infoblox = _get_infoblox(**api_opts) + host = infoblox.get_host(name=name, mac=mac, ipv4addr=ipv4addr, return_fields=return_fields) + return host + + +def get_host_advanced(name=None, ipv4addr=None, mac=None, **api_opts): + ''' + Get all host information + + CLI Example: + + salt-call infoblox.get_host_advanced hostname.domain.ca + ''' + infoblox = _get_infoblox(**api_opts) + host = infoblox.get_host_advanced(name=name, mac=mac, ipv4addr=ipv4addr) + return host + + +def get_host_domainname(name, domains=None, **api_opts): + ''' + Get host domain name + + If no domains are passed, the hostname is checked for a zone in infoblox, if no zone split on first dot. + + If domains are provided, the best match out of the list is returned. + + If none are found the return is None + + dots at end of names are ignored. + + CLI Example: + + salt-call uwl.get_host_domainname name=localhost.t.domain.com \ + domains=['domain.com', 't.domain.com.'] + + # returns: t.domain.com + ''' + name = name.lower().rstrip('.') + if not domains: + data = get_host(name=name, **api_opts) + if data and 'zone' in data: + return data['zone'].lower() else: - log.debug('no data returned from infoblox') - return None + if name.count('.') > 1: + return name[name.find('.')+1:] + return name + match = '' + for d in domains: + d = d.lower().rstrip('.') + if name.endswith(d) and len(d) > len(match): + match = d + if len(match) > 0: + return match + return None + + +def get_host_hostname(name, domains=None, **api_opts): + ''' + Get hostname + + If no domains are passed, the hostname is checked for a zone in infoblox, if no zone split on first dot. + + If domains are provided, the best match out of the list is truncated from the fqdn leaving the hostname. + + If no matching domains are found the fqdn is returned. + + dots at end of names are ignored. + + CLI Example: + + salt-call infoblox.get_host_hostname fqdn=localhost.xxx.t.domain.com \ + domains="['domain.com', 't.domain.com']" + #returns: localhost.xxx + + salt-call infoblox.get_host_hostname fqdn=localhost.xxx.t.domain.com + #returns: localhost + ''' + name = name.lower().rstrip('.') + if not domains: + return name.split('.')[0] + domain = get_host_domainname(name, domains, **api_opts) + if domain and domain in name: + return name.rsplit('.' + domain)[0] + return name + + +def get_host_mac(name=None, allow_array=False, **api_opts): + ''' + Get mac address from host record. + + Use `allow_array` to return possible mutiple values. + + CLI Example: + + salt-call infoblox.get_host_mac host=localhost.domain.com + ''' + data = get_host(name=name, **api_opts) + if data and 'ipv4addrs' in data: + l = [] + for a in data['ipv4addrs']: + if 'mac' in a: + l.append(a['mac']) + if allow_array: + return l + if l: + return l[0] + return None + + +def get_host_ipv4(name=None, mac=None, allow_array=False, **api_opts): + ''' + Get ipv4 address from host record. + + Use `allow_array` to return possible mutiple values. + + CLI Example: + + salt-call infoblox.get_host_ipv4 host=localhost.domain.com + salt-call infoblox.get_host_ipv4 mac=00:50:56:84:6e:ae + ''' + data = get_host(name=name, mac=mac, **api_opts) + if data and 'ipv4addrs' in data: + l = [] + for a in data['ipv4addrs']: + if 'ipv4addr' in a: + l.append(a['ipv4addr']) + if allow_array: + return l + if l: + return l[0] + return None + + +def get_host_ipv4addr_info(ipv4addr=None, mac=None, + discovered_data=None, + return_fields=None, **api_opts): + ''' + Get host ipv4addr information + + return_fields='mac,host,configure_for_dhcp,ipv4addr' + + CLI Example: + + salt-call infoblox.get_ipv4addr ipv4addr=123.123.122.12 + salt-call infoblox.get_ipv4addr mac=00:50:56:84:6e:ae + salt-call infoblox.get_ipv4addr mac=00:50:56:84:6e:ae return_fields=host + return_fields='mac,host,configure_for_dhcp,ipv4addr' + ''' + infoblox = _get_infoblox(**api_opts) + return infoblox.get_host_ipv4addr_object(ipv4addr, mac, discovered_data, return_fields) + + +def get_host_ipv6addr_info(ipv6addr=None, mac=None, + discovered_data=None, + return_fields=None, **api_opts): + ''' + Get host ipv6addr information + + CLI Example: + + salt-call infoblox.get_host_ipv6addr_info ipv6addr=2001:db8:85a3:8d3:1349:8a2e:370:7348 + ''' + infoblox = _get_infoblox(**api_opts) + return infoblox.get_host_ipv6addr_object(ipv6addr, mac, discovered_data, return_fields) + + +def get_network(ipv4addr=None, network=None, return_fields=None, **api_opts): + ''' + Get list of all networks. + This is helpfull when looking up subnets to + use with func:nextavailableip + + This call is offen slow and not cached! + + some return_fields + comment,network,network_view,ddns_domainname,disable,enable_ddns + + CLI Example: + + salt-call infoblox.get_network + ''' + infoblox = _get_infoblox(**api_opts) + return infoblox.get_network(ipv4addr=ipv4addr, network=network, return_fields=return_fields) + + +def delete_host(name=None, mac=None, ipv4addr=None, **api_opts): + ''' + Delete host + + CLI Example: + + salt-call infoblox.delete_host name=example.domain.com + salt-call infoblox.delete_host ipv4addr=123.123.122.12 + salt-call infoblox.delete_host ipv4addr=123.123.122.12 mac=00:50:56:84:6e:ae + ''' + if '__opts__' in globals() and __opts__['test']: + return {'Test': 'Would attempt to delete host'} + infoblox = _get_infoblox(**api_opts) + return infoblox.delete_host(name, mac, ipv4addr) + + +def create_host(data, **api_opts): + ''' + Add host record + + Avoid race conditions, use func:nextavailableip for ipv[4,6]addrs: + - func:nextavailableip:network/ZG54dfgsrDFEFfsfsLzA:10.0.0.0/8/default + - func:nextavailableip:10.0.0.0/8 + - func:nextavailableip:10.0.0.0/8,external + - func:nextavailableip:10.0.0.3-10.0.0.10 + + See your infoblox API for full `data` format. + + CLI Example: + + salt-call infoblox.create_host \ + data = + {'name': 'hostname.example.ca', + 'aliases': ['hostname.math.example.ca'], + 'extattrs': [{'Business Contact': {'value': 'example@example.ca'}}, + {'Pol8 Classification': {'value': 'Restricted'}}, + {'Primary OU': {'value': 'CS'}}, + {'Technical Contact': {'value': 'example@example.ca'}}], + 'ipv4addrs': [{'configure_for_dhcp': True, + 'ipv4addr': 'func:nextavailableip:129.97.139.0/24', + 'mac': '00:50:56:84:6e:ae'}], + 'ipv6addrs': [], } + ''' + return create_object('record:host', data, **api_opts) + + +def get_ipv4_range(start_addr=None, end_addr=None, return_fields=None, **api_opts): + ''' + Get ip range + + CLI Example: + + salt-call infoblox.get_ipv4_range start_addr=123.123.122.12 + ''' + infoblox = _get_infoblox(**api_opts) + return infoblox.get_range(start_addr, end_addr, return_fields) + + +def delete_ipv4_range(start_addr=None, end_addr=None, **api_opts): + ''' + Delete ip range. + + CLI Example: + + salt-call infoblox.delete_ipv4_range start_addr=123.123.122.12 + ''' + r = get_ipv4_range(start_addr, end_addr, **api_opts) + if r: + return delete_object(r['_ref'], **api_opts) else: - msg = 'Unsuccessful error code {0} returned'.format(retData.status_code) - raise CommandExecutionError(msg) - - -def delete_record(name, - dns_view, - record_type, - infoblox_server=None, - infoblox_user=None, - infoblox_password=None, - infoblox_api_version='v1.4.2', - sslVerify=True): - ''' - delete a record - - name - name of the record - - dns_view - the DNS view to remove the record from - - record_type - the record type (a, cname, host, etc) - - infoblox_server - the infoblox server hostname (can also use the infoblox:server pillar) - - infoblox_user - the infoblox user to connect with (can also use the infoblox:user pillar) - - infoblox_password - the infoblox user's password (can also use the infolblox:password pillar) - - infoblox_api_version - the infoblox api version to use - - sslVerify - should ssl verification be done on the connection to the Infoblox REST API - - CLI Example: - - .. code-block:: bash - - salt my-minion infoblox.delete_record some.dns.record MyInfobloxView A sslVerify=False - ''' - infoblox_server, infoblox_user, infoblox_password = _conn_info_check(infoblox_server, - infoblox_user, - infoblox_password) - if infoblox_server is None and infoblox_user is None and infoblox_password is None: - _throw_no_creds() - return None - - record_type = record_type.lower() - currentRecords = get_record(name, - record_type, - infoblox_server, - infoblox_user, - infoblox_password, - dns_view, - infoblox_api_version, - sslVerify) - if currentRecords: - for currentRecord in currentRecords: - url = 'https://{0}/wapi/{1}/{2}'.format(infoblox_server, - infoblox_api_version, - currentRecord['Record ID']) - ret = requests.delete(url, - auth=(infoblox_user, infoblox_password), - headers={'Content-Type': 'application/json'}, - verify=sslVerify) - if ret.status_code == 200: - return True - else: - msg = 'Unsuccessful error code {0} returned -- full json dump {1}'.format(ret.status_code, ret.json()) - raise CommandExecutionError(msg) - return False - - -def update_record(name, - value, - dns_view, - record_type, - infoblox_server=None, - infoblox_user=None, - infoblox_password=None, - infoblox_api_version='v1.4.2', - sslVerify=True): - ''' - update an entry to an infoblox dns view - - name - the dns name - - value - the value for the record - - record_type - the record type (a, cname, etc) - - dns_view - the DNS view to add the record to - - infoblox_server - the infoblox server hostname (can also use the infoblox:server pillar) - - infoblox_user - the infoblox user to connect with (can also use the infoblox:user pillar) - - infoblox_password - the infoblox user's password (can also use the infolblox:password pillar) - - infoblox_api_version - the infoblox api version to use - - sslVerify - should ssl verification be done on the connection to the Infoblox REST API - - CLI Example: - - .. code-block:: bash - - salt '*' infoblox.update_record alias.network.name canonical.network.name MyInfobloxView cname sslVerify=False - ''' - - infoblox_server, infoblox_user, infoblox_password = _conn_info_check(infoblox_server, - infoblox_user, - infoblox_password) - if infoblox_server is None and infoblox_user is None and infoblox_password is None: - _throw_no_creds() - return None - - record_type = record_type.lower() - currentRecords = get_record(name, - record_type, - infoblox_server, - infoblox_user, - infoblox_password, - dns_view, - infoblox_api_version, - sslVerify) - if currentRecords: - for currentRecord in currentRecords: - url = 'https://{0}/wapi/{1}/{2}'.format( - infoblox_server, - infoblox_api_version, - currentRecord['Record ID']) - data = None - if record_type == 'cname': - data = json.dumps({'canonical': value}) - elif record_type == 'a': - data = json.dumps({'ipv4addr': value}) - elif record_type == 'host': - data = {'ipv4addrs': []} - for i in value: - data['ipv4addrs'].append({'ipv4addr': i}) - data = json.dumps(data) - ret = requests.put(url, - data, - auth=(infoblox_user, infoblox_password), - headers={'Content-Type': 'application/json'}, - verify=sslVerify) - if ret.status_code == 200: - return True - else: - msg = 'Unsuccessful status code {0} returned.'.format(ret.status_code) - raise CommandExecutionError(msg) - else: - msg = 'Record {0} of type {1} was not found'.format(name, record_type) - log.error(msg) - return False - - -def add_record(name, - value, - record_type, - dns_view, - infoblox_server=None, - infoblox_user=None, - infoblox_password=None, - infoblox_api_version='v1.4.2', - sslVerify=True): - ''' - add a record to an infoblox dns view - - name - the record name - - value - the value for the entry - can make use of infoblox functions for next available IP, like 'func:nextavailableip:10.1.0.0/24' - - record_type - the record type (cname, a, host, etc) - - dns_view - the DNS view to add the record to - - infoblox_server - the infoblox server hostname (can also use the infoblox:server pillar) - - infoblox_user - the infoblox user to connect with (can also use the infoblox:user pillar) - - infoblox_password - the infoblox user's password (can also use the infolblox:password pillar) - - infoblox_api_version - the infoblox api version to use - - sslVerify - should ssl verification be done on the connection to the Infoblox REST API - - CLI Example: - - .. code-block:: bash - - salt 'myminion' infoblox.add_record alias.network.name canonical.network.name MyView - ''' - - infoblox_server, infoblox_user, infoblox_password = _conn_info_check(infoblox_server, - infoblox_user, - infoblox_password) - if infoblox_server is None and infoblox_user is None and infoblox_password is None: - _throw_no_creds() - return None - - record_type = record_type.lower() - - data = None - url = None - if record_type == 'cname': - data = json.dumps({'name': name, 'canonical': value, 'view': dns_view}) - log.debug('cname data {0}'.format(data)) - elif record_type == 'host': - data = json.dumps({'name': name, 'ipv4addrs': [{'ipv4addr': value}], 'view': dns_view}) - log.debug('host record data {0}'.format(data)) - elif record_type == 'a': - data = json.dumps({'name': name, 'ipv4addr': value, 'view': dns_view}) - log.debug('a record data {0}'.format(data)) - - url = 'https://{0}/wapi/{1}/record:{2}'.format(infoblox_server, - infoblox_api_version, - record_type) - - ret = requests.post(url, - data, - auth=(infoblox_user, infoblox_password), - headers={'Content-Type': 'application/json'}, - verify=sslVerify) - if ret.status_code == 201: return True - else: - msg = 'Unsuccessful error code {0} returned -- full json dump {1}'.format(ret.status_code, ret.json()) - raise CommandExecutionError(msg) -def _throw_no_creds(): +def create_ipv4_range(data, **api_opts): ''' - helper function to log no credentials found error - ''' - msg = 'An infoblox server, username, and password must be specified or configured via pillar' - raise SaltInvocationError(msg) + Create a ipv4 range - -def get_network(network_name, - network_view=None, - infoblox_server=None, - infoblox_user=None, - infoblox_password=None, - infoblox_api_version='v1.4.2', - sslVerify=True): - ''' - get a network from infoblox - - network_name - The name of the network in IPAM - - network_view - The name of the network view the network belongs to - - infoblox_server - the infoblox server hostname (can also use the infoblox:server pillar) - - infoblox_user - the infoblox user to connect with (can also use the infoblox:user pillar) - - infoblox_password - the infoblox user's password (can also use the infolblox:password pillar) - - infoblox_api_version - the infoblox api version to use - - sslVerify - should ssl verification be done on the connection to the Infoblox REST API + This is a helper function to `create_object` + See your infoblox API for full `data` format. CLI Example: - .. code-block:: bash - - salt myminion infoblox.get_network '10.0.0.0/8' + salt-call infoblox.create_ipv4_range data={ + start_addr: '129.97.150.160', + end_addr: '129.97.150.170'} ''' - - records = [] - infoblox_server, infoblox_user, infoblox_password = _conn_info_check(infoblox_server, - infoblox_user, - infoblox_password) - if infoblox_server is None and infoblox_user is None and infoblox_password is None: - _throw_no_creds() - return None - - url = 'https://{0}/wapi/{1}/network?network={2}{3}'.format( - infoblox_server, - infoblox_api_version, - network_name, - ('' if network_view is None else '&network_view=' + network_view)) - log.debug('Requst url is "{0}"'.format(url)) - ret = _process_return_data(requests.get(url, - auth=(infoblox_user, infoblox_password), - verify=sslVerify)) - if ret: - for entry in ret.json(): - log.debug('Infoblox record returned: {0}'.format(entry)) - tEntry = {} - data = _parse_record_data(entry) - for key in data: - tEntry[key] = data[key] - records.append(tEntry) - return records - else: - return False + return create_object('range', data, **api_opts) -def get_record(record_name, - record_type='host', - infoblox_server=None, - infoblox_user=None, - infoblox_password=None, - dns_view=None, - infoblox_api_version='v1.4.2', - sslVerify=True): +def create_a(data, **api_opts): ''' - get a record from infoblox + Create A record. - record_name - name of the record to search for - - record_type - type of reacord to search for (host, cname, a, etc...defaults to host) - - infoblox_server - the infoblox server hostname (can also use the infoblox:server pillar) - - infoblox_user - the infoblox user to connect with (can also use the infoblox:user pillar) - - infoblox_password - the infoblox user's password (can also use the infolblox:password pillar) - - dns_view - the infoblox DNS view to search, if not specified all views are searched - - infoblox_api_version - the infoblox api version to use - - sslVerify - should ssl verification be done on the connection to the Infoblox REST API + This is a helper function to `create_object`. + See your infoblox API for full `data` format. CLI Example: - .. code-block:: bash - - salt myminion infoblox.get_record some.host.com A sslVerify=False + salt-call infoblox.create_a \ + data = + name: 'fastlinux.math.example.ca' + ipv4addr: '127.0.0.1' + view: External ''' - - # TODO - verify record type (A, AAAA, CNAME< HOST, MX, PTR, SVR, TXT, host_ipv4addr, host_ipv6addr, naptr) - records = [] - - infoblox_server, infoblox_user, infoblox_password = _conn_info_check(infoblox_server, - infoblox_user, - infoblox_password) - - record_type = record_type.lower() - if infoblox_server is None and infoblox_user is None and infoblox_password is None: - _throw_no_creds() - return None - - url = 'https://{0}/wapi/{1}/record:{3}?name:={2}{4}{5}'.format( - infoblox_server, - infoblox_api_version, - record_name, - record_type, - ('' if dns_view is None else '&view=' + dns_view), - ('&_return_fields%2B=aliases' if record_type == 'host' else '') - ) - log.debug('Requst url is "{0}"'.format(url)) - ret = _process_return_data(requests.get(url, - auth=(infoblox_user, infoblox_password), - verify=sslVerify)) - if ret: - for entry in ret.json(): - log.debug('Infoblox record returned: {0}'.format(entry)) - tEntry = {} - data = _parse_record_data(entry) - for key in data: - tEntry[key] = data[key] - records.append(tEntry) - return records - else: - return False + return create_object('record:a', data, **api_opts) -def _parse_record_data(entry_data): - ''' - returns the right value data we'd be interested in for the specified record type +def get_a(name=None, ipv4addr=None, allow_array=True, **api_opts): ''' + Get A record - ret = {} - ipv4addrs = [] - aliases = [] - if 'canonical' in entry_data: - ret['Canonical Name'] = entry_data['canonical'] - if 'ipv4addrs' in entry_data: - for ipaddrs in entry_data['ipv4addrs']: - ipv4addrs.append(ipaddrs['ipv4addr']) - ret['IP Addresses'] = ipv4addrs - if 'ipv4addr' in entry_data: - ret['IP Address'] = entry_data['ipv4addr'] - if 'aliases' in entry_data: - for alias in entry_data['aliases']: - aliases.append(alias) - ret['Aliases'] = aliases - if 'name' in entry_data: - ret['Name'] = entry_data['name'] - if 'view' in entry_data: - ret['DNS View'] = entry_data['view'] - if 'network_view' in entry_data: - ret['Network View'] = entry_data['network_view'] - if 'comment' in entry_data: - ret['Comment'] = entry_data['comment'] - if 'network' in entry_data: - ret['Network'] = entry_data['network'] - if '_ref' in entry_data: - ret['Record ID'] = entry_data['_ref'] + CLI Example: + + salt-call infoblox.get_a name=abc.example.com + salt-call infoblox.get_a ipv4addr=192.168.3.5 + ''' + data = {} + if name: + data['name'] = name + if ipv4addr: + data['ipv4addr'] = ipv4addr + r = get_object('record:a', data=data, **api_opts) + if r and len(r) > 1 and not allow_array: + raise Exception('More than one result, use allow_array to return the data') + return r + + +def delete_a(name=None, ipv4addr=None, allow_array=False, **api_opts): + ''' + Delete A record + + If the A record is used as a round robin you can set + `allow_array=true to delete all records for the hostname. + + CLI Example: + + salt-call infoblox.delete_a name=abc.example.com + salt-call infoblox.delete_a ipv4addr=192.168.3.5 + salt-call infoblox.delete_a name=acname.example.com allow_array=true + ''' + r = get_a(name, ipv4addr, allow_array=False, **api_opts) + if not r: + return True + if len(r) == 0: + return True + if len(r) > 1 and not allow_array: + raise Exception('More than one result, use allow_array to override') + ret = [] + for ri in r: + ret.append(delete_object(ri['_ref'], **api_opts)) return ret diff --git a/salt/modules/ini_manage.py b/salt/modules/ini_manage.py index c553c97992b..9af24eda199 100644 --- a/salt/modules/ini_manage.py +++ b/salt/modules/ini_manage.py @@ -20,8 +20,8 @@ import re import json # Import Salt libs -import salt.ext.six as six -import salt.utils +from salt.ext import six +import salt.utils.files from salt.exceptions import CommandExecutionError from salt.utils.odict import OrderedDict @@ -318,17 +318,18 @@ class _Section(OrderedDict): yield '{0}[{1}]{0}'.format(os.linesep, self.name) sections_dict = OrderedDict() for name, value in six.iteritems(self): + # Handle Comment Lines if com_regx.match(name): yield '{0}{1}'.format(value, os.linesep) + # Handle Sections elif isinstance(value, _Section): sections_dict.update({name: value}) + # Key / Value pairs + # Adds spaces between the separator else: yield '{0}{1}{2}{3}'.format( name, - ( - ' {0} '.format(self.sep) if self.sep != ' ' - else self.sep - ), + ' {0} '.format(self.sep) if self.sep != ' ' else self.sep, value, os.linesep ) @@ -369,7 +370,7 @@ class _Ini(_Section): def refresh(self, inicontents=None): if inicontents is None: try: - with salt.utils.fopen(self.name) as rfh: + with salt.utils.files.fopen(self.name) as rfh: inicontents = rfh.read() except (OSError, IOError) as exc: raise CommandExecutionError( @@ -395,7 +396,7 @@ class _Ini(_Section): def flush(self): try: - with salt.utils.fopen(self.name, 'w') as outfile: + with salt.utils.files.fopen(self.name, 'w') as outfile: ini_gen = self.gen_ini() next(ini_gen) outfile.writelines(ini_gen) diff --git a/salt/modules/inspectlib/collector.py b/salt/modules/inspectlib/collector.py index 332c6efdec0..d67b2519bb4 100644 --- a/salt/modules/inspectlib/collector.py +++ b/salt/modules/inspectlib/collector.py @@ -28,9 +28,11 @@ from salt.modules.inspectlib import kiwiproc from salt.modules.inspectlib.entities import (AllowedDir, IgnoredDir, Package, PayloadFile, PackageCfgFile) -import salt.utils -from salt.utils import fsutils -from salt.utils import reinit_crypto +import salt.utils # Can be removed when reinit_crypto is moved +import salt.utils.files +import salt.utils.fsutils +import salt.utils.path +import salt.utils.stringutils from salt.exceptions import CommandExecutionError try: @@ -87,7 +89,7 @@ class Inspector(EnvLoader): ''' if self.grains_core.os_data().get('os_family') == 'Debian': return self.__get_cfg_pkgs_dpkg() - elif self.grains_core.os_data().get('os_family') in ['SUSE', 'redhat']: + elif self.grains_core.os_data().get('os_family') in ['Suse', 'redhat']: return self.__get_cfg_pkgs_rpm() else: return dict() @@ -100,13 +102,13 @@ class Inspector(EnvLoader): # Get list of all available packages data = dict() - for pkg_name in salt.utils.to_str(self._syscall('dpkg-query', None, None, + for pkg_name in salt.utils.stringutils.to_str(self._syscall('dpkg-query', None, None, '-Wf', "${binary:Package}\\n")[0]).split(os.linesep): pkg_name = pkg_name.strip() if not pkg_name: continue data[pkg_name] = list() - for pkg_cfg_item in salt.utils.to_str(self._syscall('dpkg-query', None, None, '-Wf', "${Conffiles}\\n", + for pkg_cfg_item in salt.utils.stringutils.to_str(self._syscall('dpkg-query', None, None, '-Wf', "${Conffiles}\\n", pkg_name)[0]).split(os.linesep): pkg_cfg_item = pkg_cfg_item.strip() if not pkg_cfg_item: @@ -131,7 +133,7 @@ class Inspector(EnvLoader): pkg_name = None pkg_configs = [] - out = salt.utils.to_str(out) + out = salt.utils.stringutils.to_str(out) for line in out.split(os.linesep): line = line.strip() if not line: @@ -161,10 +163,10 @@ class Inspector(EnvLoader): cfgs = list() cfg_data = list() if self.grains_core.os_data().get('os_family') == 'Debian': - cfg_data = salt.utils.to_str(self._syscall("dpkg", None, None, '--verify', + cfg_data = salt.utils.stringutils.to_str(self._syscall("dpkg", None, None, '--verify', pkg_name)[0]).split(os.linesep) - elif self.grains_core.os_data().get('os_family') in ['SUSE', 'redhat']: - cfg_data = salt.utils.to_str(self._syscall("rpm", None, None, '-V', '--nodeps', '--nodigest', + elif self.grains_core.os_data().get('os_family') in ['Suse', 'redhat']: + cfg_data = salt.utils.stringutils.to_str(self._syscall("rpm", None, None, '-V', '--nodeps', '--nodigest', '--nosignature', '--nomtime', '--nolinkto', pkg_name)[0]).split(os.linesep) for line in cfg_data: @@ -240,7 +242,7 @@ class Inspector(EnvLoader): ''' if self.grains_core.os_data().get('os_family') == 'Debian': return self.__get_managed_files_dpkg() - elif self.grains_core.os_data().get('os_family') in ['SUSE', 'redhat']: + elif self.grains_core.os_data().get('os_family') in ['Suse', 'redhat']: return self.__get_managed_files_rpm() return list(), list(), list() @@ -253,12 +255,12 @@ class Inspector(EnvLoader): links = set() files = set() - for pkg_name in salt.utils.to_str(self._syscall("dpkg-query", None, None, + for pkg_name in salt.utils.stringutils.to_str(self._syscall("dpkg-query", None, None, '-Wf', '${binary:Package}\\n')[0]).split(os.linesep): pkg_name = pkg_name.strip() if not pkg_name: continue - for resource in salt.utils.to_str(self._syscall("dpkg", None, None, '-L', pkg_name)[0]).split(os.linesep): + for resource in salt.utils.stringutils.to_str(self._syscall("dpkg", None, None, '-L', pkg_name)[0]).split(os.linesep): resource = resource.strip() if not resource or resource in ['/', './', '.']: continue @@ -279,7 +281,7 @@ class Inspector(EnvLoader): links = set() files = set() - for line in salt.utils.to_str(self._syscall("rpm", None, None, '-qlav')[0]).split(os.linesep): + for line in salt.utils.stringutils.to_str(self._syscall("rpm", None, None, '-qlav')[0]).split(os.linesep): line = line.strip() if not line: continue @@ -311,7 +313,7 @@ class Inspector(EnvLoader): continue if not valid or not os.path.exists(obj) or not os.access(obj, os.R_OK): continue - if os.path.islink(obj): + if salt.utils.path.islink(obj): links.append(obj) elif os.path.isdir(obj): dirs.append(obj) @@ -374,7 +376,7 @@ class Inspector(EnvLoader): # Add ignored filesystems ignored_fs = set() ignored_fs |= set(self.IGNORE_PATHS) - mounts = fsutils._get_mounts() + mounts = salt.utils.fsutils._get_mounts() for device, data in mounts.items(): if device in self.IGNORE_MOUNTS: for mpt in data: @@ -475,7 +477,7 @@ def is_alive(pidfile): Check if PID is still alive. ''' try: - with salt.utils.fopen(pidfile) as fp_: + with salt.utils.files.fopen(pidfile) as fp_: os.kill(int(fp_.read().strip()), 0) return True except Exception as ex: @@ -503,10 +505,10 @@ if __name__ == '__main__': # Double-fork stuff try: if os.fork() > 0: - reinit_crypto() + salt.utils.reinit_crypto() sys.exit(0) else: - reinit_crypto() + salt.utils.reinit_crypto() except OSError as ex: sys.exit(1) @@ -516,12 +518,12 @@ if __name__ == '__main__': try: pid = os.fork() if pid > 0: - reinit_crypto() - with salt.utils.fopen(os.path.join(pidfile, EnvLoader.PID_FILE), 'w') as fp_: + salt.utils.reinit_crypto() + with salt.utils.files.fopen(os.path.join(pidfile, EnvLoader.PID_FILE), 'w') as fp_: fp_.write('{0}\n'.format(pid)) sys.exit(0) except OSError as ex: sys.exit(1) - reinit_crypto() + salt.utils.reinit_crypto() main(dbfile, pidfile, mode) diff --git a/salt/modules/inspectlib/kiwiproc.py b/salt/modules/inspectlib/kiwiproc.py index 136cacf00cc..c9b42344daa 100644 --- a/salt/modules/inspectlib/kiwiproc.py +++ b/salt/modules/inspectlib/kiwiproc.py @@ -17,14 +17,17 @@ # Import python libs from __future__ import absolute_import import os -import grp -import pwd from xml.dom import minidom import platform import socket +try: + import grp + import pwd +except ImportError: + pass # Import salt libs -import salt.utils +import salt.utils.files from salt.modules.inspectlib.exceptions import InspectorKiwiProcessorException # Import third party libs @@ -145,14 +148,14 @@ class KiwiExporter(object): ''' # Get real local users with the local passwords shadow = {} - with salt.utils.fopen('/etc/shadow') as rfh: + with salt.utils.files.fopen('/etc/shadow') as rfh: for sh_line in rfh.read().split(os.linesep): if sh_line.strip(): login, pwd = sh_line.split(":")[:2] if pwd and pwd[0] not in '!*': shadow[login] = {'p': pwd} - with salt.utils.fopen('/etc/passwd') as rfh: + with salt.utils.files.fopen('/etc/passwd') as rfh: for ps_line in rfh.read().split(os.linesep): if ps_line.strip(): ps_line = ps_line.strip().split(':') diff --git a/salt/modules/inspectlib/query.py b/salt/modules/inspectlib/query.py index d5f7ae763c3..2ce33ce845b 100644 --- a/salt/modules/inspectlib/query.py +++ b/salt/modules/inspectlib/query.py @@ -21,6 +21,8 @@ import time import logging # Import Salt Libs +import salt.utils.files +import salt.utils.fsutils import salt.utils.network from salt.modules.inspectlib.exceptions import (InspectorQueryException, SIException) from salt.modules.inspectlib import EnvLoader @@ -214,7 +216,7 @@ class Query(EnvLoader): ''' users = dict() path = '/etc/passwd' - with salt.utils.fopen(path, 'r') as fp_: + with salt.utils.files.fopen(path, 'r') as fp_: for line in fp_: line = line.strip() if ':' not in line: @@ -239,7 +241,7 @@ class Query(EnvLoader): ''' groups = dict() path = '/etc/group' - with salt.utils.fopen(path, 'r') as fp_: + with salt.utils.files.fopen(path, 'r') as fp_: for line in fp_: line = line.strip() if ':' not in line: diff --git a/salt/modules/inspector.py b/salt/modules/inspector.py index df0004ed2e4..708ad144500 100644 --- a/salt/modules/inspector.py +++ b/salt/modules/inspector.py @@ -26,8 +26,8 @@ from salt.modules.inspectlib.exceptions import (InspectorQueryException, InspectorKiwiProcessorException) # Import Salt libs -import salt.utils import salt.utils.fsutils +import salt.utils.platform from salt.exceptions import CommandExecutionError from salt.exceptions import get_error_message as _get_error_message @@ -38,7 +38,7 @@ def __virtual__(): ''' Only work on POSIX-like systems ''' - return not salt.utils.is_windows() and 'inspector' + return not salt.utils.platform.is_windows() and 'inspector' def _(module): diff --git a/salt/modules/introspect.py b/salt/modules/introspect.py index 0d6a0d9ad99..2e434ad6430 100644 --- a/salt/modules/introspect.py +++ b/salt/modules/introspect.py @@ -9,7 +9,7 @@ from __future__ import absolute_import import os # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six def running_service_owners( diff --git a/salt/modules/ipset.py b/salt/modules/ipset.py index e202fb3ddc0..366d9665522 100644 --- a/salt/modules/ipset.py +++ b/salt/modules/ipset.py @@ -8,9 +8,9 @@ from __future__ import absolute_import import logging # Import Salt libs -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import map, range -import salt.utils +import salt.utils.path # Import third-party libs if six.PY3: @@ -108,7 +108,7 @@ def __virtual__(): ''' Only load the module if ipset is installed ''' - if salt.utils.which('ipset'): + if salt.utils.path.which('ipset'): return True return (False, 'The ipset execution modules cannot be loaded: ipset binary not in path.') @@ -117,7 +117,7 @@ def _ipset_cmd(): ''' Return correct command ''' - return salt.utils.which('ipset') + return salt.utils.path.which('ipset') def version(): diff --git a/salt/modules/iptables.py b/salt/modules/iptables.py index 2681768f61c..3123fb58fc9 100644 --- a/salt/modules/iptables.py +++ b/salt/modules/iptables.py @@ -36,7 +36,9 @@ import uuid import string # Import salt libs -import salt.utils +import salt.utils.args +import salt.utils.files +import salt.utils.path from salt.state import STATE_INTERNAL_KEYWORDS as _STATE_INTERNAL_KEYWORDS from salt.exceptions import SaltException from salt.ext import six @@ -49,7 +51,7 @@ def __virtual__(): ''' Only load the module if iptables is installed ''' - if not salt.utils.which('iptables'): + if not salt.utils.path.which('iptables'): return (False, 'The iptables execution module cannot be loaded: iptables not installed.') return True @@ -60,9 +62,9 @@ def _iptables_cmd(family='ipv4'): Return correct command based on the family, e.g. ipv4 or ipv6 ''' if family == 'ipv6': - return salt.utils.which('ip6tables') + return salt.utils.path.which('ip6tables') else: - return salt.utils.which('iptables') + return salt.utils.path.which('iptables') def _has_option(option, family='ipv4'): @@ -104,9 +106,14 @@ def _conf(family='ipv4'): return '/var/lib/ip6tables/rules-save' else: return '/var/lib/iptables/rules-save' - elif __grains__['os_family'] == 'SUSE': + elif __grains__['os_family'] == 'Suse': # SuSE does not seem to use separate files for IPv4 and IPv6 return '/etc/sysconfig/scripts/SuSEfirewall2-custom' + elif __grains__['os_family'] == 'Void': + if family == 'ipv6': + return '/etc/iptables/iptables.rules' + else: + return '/etc/iptables/ip6tables.rules' elif __grains__['os'] == 'Alpine': if family == 'ipv6': return '/etc/iptables/rules6-save' @@ -200,6 +207,7 @@ def build_rule(table='filter', chain=None, command=None, position='', full=None, If a position is required (as with `-I` or `-D`), it may be specified as `position`. This will only be useful if `full` is True. + If `state` is passed, it will be ignored, use `connstate`. If `connstate` is passed in, it will automatically be changed to `state`. To pass in jump options that doesn't take arguments, pass in an empty @@ -213,19 +221,19 @@ def build_rule(table='filter', chain=None, command=None, position='', full=None, connstate=RELATED,ESTABLISHED jump=ACCEPT salt '*' iptables.build_rule filter INPUT command=I position=3 \\ - full=True match=state state=RELATED,ESTABLISHED jump=ACCEPT + full=True match=state connstate=RELATED,ESTABLISHED jump=ACCEPT salt '*' iptables.build_rule filter INPUT command=A \\ - full=True match=state state=RELATED,ESTABLISHED \\ + full=True match=state connstate=RELATED,ESTABLISHED \\ source='127.0.0.1' jump=ACCEPT .. Invert Rules salt '*' iptables.build_rule filter INPUT command=A \\ - full=True match=state state=RELATED,ESTABLISHED \\ - source='! 127.0.0.1' jump=ACCEPT + full=True match=state connstate=RELATED,ESTABLISHED \\ + source='!127.0.0.1' jump=ACCEPT salt '*' iptables.build_rule filter INPUT command=A \\ - full=True match=state state=RELATED,ESTABLISHED \\ + full=True match=state connstate=RELATED,ESTABLISHED \\ destination='not 127.0.0.1' jump=ACCEPT IPv6: @@ -233,7 +241,7 @@ def build_rule(table='filter', chain=None, command=None, position='', full=None, connstate=RELATED,ESTABLISHED jump=ACCEPT \\ family=ipv6 salt '*' iptables.build_rule filter INPUT command=I position=3 \\ - full=True match=state state=RELATED,ESTABLISHED jump=ACCEPT \\ + full=True match=state connstate=RELATED,ESTABLISHED jump=ACCEPT \\ family=ipv6 ''' @@ -493,8 +501,11 @@ def build_rule(table='filter', chain=None, command=None, position='', full=None, after_jump.append('--{0} {1}'.format(after_jump_argument, value)) del kwargs[after_jump_argument] - for key, value in kwargs.items(): + for key in kwargs: negation = maybe_add_negation(key) + # don't use .items() since maybe_add_negation removes the prefix from + # the value in the kwargs, thus we need to fetch it after that has run + value = kwargs[key] flag = '-' if len(key) == 1 else '--' value = '' if value in (None, '') else ' {0}'.format(value) rule.append('{0}{1}{2}{3}'.format(negation, flag, key, value)) @@ -964,7 +975,7 @@ def _parse_conf(conf_file=None, in_mem=False, family='ipv4'): rules = '' if conf_file: - with salt.utils.fopen(conf_file, 'r') as ifile: + with salt.utils.files.fopen(conf_file, 'r') as ifile: rules = ifile.read() elif in_mem: cmd = '{0}-save' . format(_iptables_cmd(family)) @@ -991,7 +1002,7 @@ def _parse_conf(conf_file=None, in_mem=False, family='ipv4'): ret[table][chain]['rules'] = [] ret[table][chain]['rules_comment'] = {} elif line.startswith('-A'): - args = salt.utils.shlex_split(line) + args = salt.utils.args.shlex_split(line) index = 0 while index + 1 < len(args): swap = args[index] == '!' and args[index + 1].startswith('-') @@ -1452,6 +1463,8 @@ def _parser(): add_arg('--or-mark', dest='or-mark', action='append') add_arg('--xor-mark', dest='xor-mark', action='append') add_arg('--set-mark', dest='set-mark', action='append') + add_arg('--nfmask', dest='nfmask', action='append') + add_arg('--ctmask', dest='ctmask', action='append') ## CONNSECMARK add_arg('--save', dest='save', action='append') add_arg('--restore', dest='restore', action='append') diff --git a/salt/modules/iwtools.py b/salt/modules/iwtools.py index 2b782a4f84d..d766fabe4bc 100644 --- a/salt/modules/iwtools.py +++ b/salt/modules/iwtools.py @@ -5,7 +5,7 @@ Support for Wireless Tools for Linux from __future__ import absolute_import import logging -import salt.utils +import salt.utils.path from salt.exceptions import SaltInvocationError @@ -16,7 +16,7 @@ def __virtual__(): ''' Only load the module if iwconfig is installed ''' - if salt.utils.which('iwconfig'): + if salt.utils.path.which('iwconfig'): return True return (False, 'The iwtools execution module cannot be loaded: ' 'iwconfig is not installed.') diff --git a/salt/modules/jboss7.py b/salt/modules/jboss7.py index 45991628d85..a771057f702 100644 --- a/salt/modules/jboss7.py +++ b/salt/modules/jboss7.py @@ -28,11 +28,11 @@ import re import logging # Import Salt libs +import salt.utils.dictdiffer as dictdiffer from salt.exceptions import SaltInvocationError -from salt.utils import dictdiffer # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six log = logging.getLogger(__name__) diff --git a/salt/modules/jboss7_cli.py b/salt/modules/jboss7_cli.py index 89c7d203c42..df5b05d0de3 100644 --- a/salt/modules/jboss7_cli.py +++ b/salt/modules/jboss7_cli.py @@ -47,7 +47,7 @@ import time from salt.exceptions import CommandExecutionError # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six log = logging.getLogger(__name__) diff --git a/salt/modules/jenkins.py b/salt/modules/jenkins.py index d327d09292b..1041e46d312 100644 --- a/salt/modules/jenkins.py +++ b/salt/modules/jenkins.py @@ -33,11 +33,11 @@ try: except ImportError: HAS_JENKINS = False -import salt.utils +import salt.utils.files # Import 3rd-party libs # pylint: disable=import-error,no-name-in-module,redefined-builtin -from salt.exceptions import SaltInvocationError +from salt.exceptions import CommandExecutionError, SaltInvocationError # pylint: enable=import-error,no-name-in-module log = logging.getLogger(__name__) @@ -89,9 +89,22 @@ def _connect(): password=jenkins_password) +def _retrieve_config_xml(config_xml, saltenv): + ''' + Helper to cache the config XML and raise a CommandExecutionError if we fail + to do so. If we successfully cache the file, return the cached path. + ''' + ret = __salt__['cp.cache_file'](config_xml, saltenv) + + if not ret: + raise CommandExecutionError('Failed to retrieve {0}'.format(config_xml)) + + return ret + + def run(script): ''' - .. versionadded:: Carbon + .. versionadded:: 2017.7.0 Execute a groovy script on the jenkins master @@ -166,7 +179,7 @@ def job_exists(name=None): ''' if not name: - raise SaltInvocationError('Required parameter `name` is missing.') + raise SaltInvocationError('Required parameter \'name\' is missing') server = _connect() if server.job_exists(name): @@ -190,12 +203,12 @@ def get_job_info(name=None): ''' if not name: - raise SaltInvocationError('Required parameter `name` is missing.') + raise SaltInvocationError('Required parameter \'name\' is missing') server = _connect() if not job_exists(name): - raise SaltInvocationError('Job `{0}` does not exist.'.format(name)) + raise CommandExecutionError('Job \'{0}\' does not exist'.format(name)) job_info = server.get_job_info(name) if job_info: @@ -219,17 +232,19 @@ def build_job(name=None, parameters=None): ''' if not name: - raise SaltInvocationError('Required parameter `name` is missing.') + raise SaltInvocationError('Required parameter \'name\' is missing') server = _connect() if not job_exists(name): - raise SaltInvocationError('Job `{0}` does not exist.'.format(name)) + raise CommandExecutionError('Job \'{0}\' does not exist.'.format(name)) try: server.build_job(name, parameters) except jenkins.JenkinsException as err: - raise SaltInvocationError('Something went wrong {0}.'.format(err)) + raise CommandExecutionError( + 'Encountered error building job \'{0}\': {1}'.format(name, err) + ) return True @@ -254,24 +269,26 @@ def create_job(name=None, ''' if not name: - raise SaltInvocationError('Required parameter `name` is missing.') + raise SaltInvocationError('Required parameter \'name\' is missing') if job_exists(name): - raise SaltInvocationError('Job `{0}` already exists.'.format(name)) + raise CommandExecutionError('Job \'{0}\' already exists'.format(name)) if not config_xml: config_xml = jenkins.EMPTY_CONFIG_XML else: - config_xml_file = __salt__['cp.cache_file'](config_xml, saltenv) + config_xml_file = _retrieve_config_xml(config_xml, saltenv) - with salt.utils.fopen(config_xml_file) as _fp: + with salt.utils.files.fopen(config_xml_file) as _fp: config_xml = _fp.read() server = _connect() try: server.create_job(name, config_xml) except jenkins.JenkinsException as err: - raise SaltInvocationError('Something went wrong {0}.'.format(err)) + raise CommandExecutionError( + 'Encountered error creating job \'{0}\': {1}'.format(name, err) + ) return config_xml @@ -296,21 +313,23 @@ def update_job(name=None, ''' if not name: - raise SaltInvocationError('Required parameter `name` is missing.') + raise SaltInvocationError('Required parameter \'name\' is missing') if not config_xml: config_xml = jenkins.EMPTY_CONFIG_XML else: - config_xml_file = __salt__['cp.cache_file'](config_xml, saltenv) + config_xml_file = _retrieve_config_xml(config_xml, saltenv) - with salt.utils.fopen(config_xml_file) as _fp: + with salt.utils.files.fopen(config_xml_file) as _fp: config_xml = _fp.read() server = _connect() try: server.reconfig_job(name, config_xml) except jenkins.JenkinsException as err: - raise SaltInvocationError('Something went wrong {0}.'.format(err)) + raise CommandExecutionError( + 'Encountered error updating job \'{0}\': {1}'.format(name, err) + ) return config_xml @@ -329,17 +348,19 @@ def delete_job(name=None): ''' if not name: - raise SaltInvocationError('Required parameter `name` is missing.') + raise SaltInvocationError('Required parameter \'name\' is missing') server = _connect() if not job_exists(name): - raise SaltInvocationError('Job `{0}` does not exists.'.format(name)) + raise CommandExecutionError('Job \'{0}\' does not exist'.format(name)) try: server.delete_job(name) except jenkins.JenkinsException as err: - raise SaltInvocationError('Something went wrong {0}.'.format(err)) + raise CommandExecutionError( + 'Encountered error deleting job \'{0}\': {1}'.format(name, err) + ) return True @@ -358,17 +379,19 @@ def enable_job(name=None): ''' if not name: - raise SaltInvocationError('Required parameter `name` is missing.') + raise SaltInvocationError('Required parameter \'name\' is missing') server = _connect() if not job_exists(name): - raise SaltInvocationError('Job `{0}` does not exists.'.format(name)) + raise CommandExecutionError('Job \'{0}\' does not exist'.format(name)) try: server.enable_job(name) except jenkins.JenkinsException as err: - raise SaltInvocationError('Something went wrong {0}.'.format(err)) + raise CommandExecutionError( + 'Encountered error enabling job \'{0}\': {1}'.format(name, err) + ) return True @@ -388,17 +411,19 @@ def disable_job(name=None): ''' if not name: - raise SaltInvocationError('Required parameter `name` is missing.') + raise SaltInvocationError('Required parameter \'name\' is missing') server = _connect() if not job_exists(name): - raise SaltInvocationError('Job `{0}` does not exists.'.format(name)) + raise CommandExecutionError('Job \'{0}\' does not exist'.format(name)) try: server.disable_job(name) except jenkins.JenkinsException as err: - raise SaltInvocationError('Something went wrong {0}.'.format(err)) + raise CommandExecutionError( + 'Encountered error disabling job \'{0}\': {1}'.format(name, err) + ) return True @@ -418,12 +443,12 @@ def job_status(name=None): ''' if not name: - raise SaltInvocationError('Required parameter `name` is missing.') + raise SaltInvocationError('Required parameter \'name\' is missing') server = _connect() if not job_exists(name): - raise SaltInvocationError('Job `{0}` does not exists.'.format(name)) + raise CommandExecutionError('Job \'{0}\' does not exist'.format(name)) return server.get_job_info('empty')['buildable'] @@ -444,12 +469,12 @@ def get_job_config(name=None): ''' if not name: - raise SaltInvocationError('Required parameter `name` is missing.') + raise SaltInvocationError('Required parameter \'name\' is missing') server = _connect() if not job_exists(name): - raise SaltInvocationError('Job `{0}` does not exists.'.format(name)) + raise CommandExecutionError('Job \'{0}\' does not exist'.format(name)) job_info = server.get_job_config(name) return job_info diff --git a/salt/modules/junos.py b/salt/modules/junos.py index 9e41482bb5d..f6ae3e4dc42 100644 --- a/salt/modules/junos.py +++ b/salt/modules/junos.py @@ -25,6 +25,10 @@ try: except ImportError: from salt._compat import ElementTree as etree +# Import Salt libs +import salt.utils.files +from salt.ext import six + # Juniper interface libraries # https://github.com/Juniper/py-junos-eznc try: @@ -41,11 +45,6 @@ try: except ImportError: HAS_JUNOS = False -# Import salt libraries -from salt.utils import fopen -from salt.utils import files -from salt.utils import safe_rm - # Set up logging log = logging.getLogger(__name__) @@ -178,6 +177,10 @@ def rpc(cmd=None, dest=None, format='xml', **kwargs): if kwargs['__pub_arg']: if isinstance(kwargs['__pub_arg'][-1], dict): op.update(kwargs['__pub_arg'][-1]) + elif '__pub_schedule' in kwargs: + for key, value in six.iteritems(kwargs): + if not key.startswith('__pub_'): + op[key] = value else: op.update(kwargs) op['dev_timeout'] = str(op.pop('timeout', conn.timeout)) @@ -234,7 +237,7 @@ def rpc(cmd=None, dest=None, format='xml', **kwargs): write_response = json.dumps(reply, indent=1) else: write_response = etree.tostring(reply) - with fopen(dest, 'w') as fp: + with salt.utils.files.fopen(dest, 'w') as fp: fp.write(write_response) return ret @@ -460,7 +463,7 @@ def rollback(id=0, **kwargs): if 'diffs_file' in op and op['diffs_file'] is not None: diff = conn.cu.diff() if diff is not None: - with fopen(op['diffs_file'], 'w') as fp: + with salt.utils.files.fopen(op['diffs_file'], 'w') as fp: fp.write(diff) else: log.info( @@ -655,7 +658,7 @@ def cli(command=None, format='text', **kwargs): ret['message'] = jxmlease.parse(result) if 'dest' in op and op['dest'] is not None: - with fopen(op['dest'], 'w') as fp: + with salt.utils.files.fopen(op['dest'], 'w') as fp: fp.write(result) ret['out'] = True @@ -833,7 +836,7 @@ def install_config(path=None, **kwargs): if "template_vars" in op: template_vars = op["template_vars"] - template_cached_path = files.mkstemp() + template_cached_path = salt.utils.files.mkstemp() __salt__['cp.get_template']( path, template_cached_path, @@ -888,7 +891,7 @@ def install_config(path=None, **kwargs): return ret finally: - safe_rm(template_cached_path) + salt.utils.files.safe_rm(template_cached_path) config_diff = cu.diff() if config_diff is None: @@ -929,7 +932,7 @@ def install_config(path=None, **kwargs): try: if write_diff and config_diff is not None: - with fopen(write_diff, 'w') as fp: + with salt.utils.files.fopen(write_diff, 'w') as fp: fp.write(config_diff) except Exception as exception: ret['message'] = 'Could not write into diffs_file due to: "{0}"'.format( @@ -1004,7 +1007,7 @@ def install_os(path=None, **kwargs): ret['out'] = False return ret - image_cached_path = files.mkstemp() + image_cached_path = salt.utils.files.mkstemp() __salt__['cp.get_file'](path, image_cached_path) if not os.path.isfile(image_cached_path): @@ -1034,7 +1037,7 @@ def install_os(path=None, **kwargs): ret['out'] = False return ret finally: - safe_rm(image_cached_path) + salt.utils.files.safe_rm(image_cached_path) if 'reboot' in op and op['reboot'] is True: try: @@ -1231,7 +1234,7 @@ def load(path=None, **kwargs): if "template_vars" in op: template_vars = op["template_vars"] - template_cached_path = files.mkstemp() + template_cached_path = salt.utils.files.mkstemp() __salt__['cp.get_template']( path, template_cached_path, @@ -1278,7 +1281,7 @@ def load(path=None, **kwargs): ret['out'] = False return ret finally: - safe_rm(template_cached_path) + salt.utils.files.safe_rm(template_cached_path) return ret diff --git a/salt/modules/k8s.py b/salt/modules/k8s.py index 8b3c40bf6b9..f47b9f1cd15 100644 --- a/salt/modules/k8s.py +++ b/salt/modules/k8s.py @@ -21,12 +21,12 @@ import re import json import logging as logger import base64 -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves.urllib.parse import urlparse as _urlparse # pylint: disable=no-name-in-module # TODO Remove requests dependency -import salt.utils +import salt.utils.files import salt.utils.http as http __virtualname__ = 'k8s' @@ -55,7 +55,7 @@ def _guess_apiserver(apiserver_url=None): config = __salt__['config.get']('k8s:config', default_config) kubeapi_regex = re.compile("""KUBE_MASTER=['"]--master=(.*)['"]""", re.MULTILINE) - with salt.utils.fopen(config) as fh_k8s: + with salt.utils.files.fopen(config) as fh_k8s: for line in fh_k8s.readlines(): match_line = kubeapi_regex.match(line) if match_line: @@ -541,7 +541,7 @@ def _is_valid_secret_file(filename): def _file_encode(filename): log.trace("Encoding secret file: {0}".format(filename)) - with salt.utils.fopen(filename, "rb") as f: + with salt.utils.files.fopen(filename, "rb") as f: data = f.read() return base64.b64encode(data) diff --git a/salt/modules/kapacitor.py b/salt/modules/kapacitor.py index edfd424834f..aa09153608f 100644 --- a/salt/modules/kapacitor.py +++ b/salt/modules/kapacitor.py @@ -20,15 +20,15 @@ from __future__ import absolute_import import json import logging -import salt.utils import salt.utils.http +import salt.utils.path from salt.utils.decorators import memoize LOG = logging.getLogger(__name__) def __virtual__(): - return 'kapacitor' if salt.utils.which('kapacitor') else False + return 'kapacitor' if salt.utils.path.which('kapacitor') else False @memoize diff --git a/salt/modules/kerberos.py b/salt/modules/kerberos.py index a5a77a58912..1293e60c698 100644 --- a/salt/modules/kerberos.py +++ b/salt/modules/kerberos.py @@ -25,13 +25,13 @@ from __future__ import absolute_import import logging # Import Salt libs -import salt.utils +import salt.utils.path log = logging.getLogger(__name__) def __virtual__(): - if salt.utils.which('kadmin'): + if salt.utils.path.which('kadmin'): return True return (False, 'The kerberos execution module not loaded: kadmin not in path') @@ -262,7 +262,7 @@ def create_keytab(name, keytab, enctypes=None): .. code-block:: bash - salt 'kdc.example.com' host/host1.example.com host1.example.com.keytab + salt 'kdc.example.com' kerberos.create_keytab host/host1.example.com host1.example.com.keytab ''' ret = {} diff --git a/salt/modules/kernelpkg_linux_apt.py b/salt/modules/kernelpkg_linux_apt.py index 9319b00708d..34512a5f5c5 100644 --- a/salt/modules/kernelpkg_linux_apt.py +++ b/salt/modules/kernelpkg_linux_apt.py @@ -3,13 +3,16 @@ Manage Linux kernel packages on APT-based systems ''' from __future__ import absolute_import +import functools import logging import re -# Import 3rd-party libs try: + # Import Salt libs + from salt.ext import six from salt.utils.versions import LooseVersion as _LooseVersion from salt.ext.six.moves import filter # pylint: disable=import-error,redefined-builtin + from salt.exceptions import CommandExecutionError HAS_REQUIRED_LIBS = True except ImportError: HAS_REQUIRED_LIBS = False @@ -73,7 +76,11 @@ def list_installed(): return [] prefix_len = len(_package_prefix()) + 1 - return sorted([pkg[prefix_len:] for pkg in result], cmp=_cmp_version) + + if six.PY2: + return sorted([pkg[prefix_len:] for pkg in result], cmp=_cmp_version) + else: + return sorted([pkg[prefix_len:] for pkg in result], key=functools.cmp_to_key(_cmp_version)) def latest_available(): @@ -109,9 +116,9 @@ def latest_installed(): .. note:: This function may not return the same value as - :py:func:`~salt.modules.kernelpkg.active` if a new kernel + :py:func:`~salt.modules.kernelpkg_linux_apt.active` if a new kernel has been installed and the system has not yet been rebooted. - The :py:func:`~salt.modules.kernelpkg.needs_reboot` function + The :py:func:`~salt.modules.kernelpkg_linux_apt.needs_reboot` function exists to detect this condition. ''' pkgs = list_installed() @@ -193,6 +200,69 @@ def upgrade_available(): return _LooseVersion(latest_available()) > _LooseVersion(latest_installed()) +def remove(release): + ''' + Remove a specific version of the kernel. + + release + The release number of an installed kernel. This must be the entire release + number as returned by :py:func:`~salt.modules.kernelpkg_linux_apt.list_installed`, + not the package name. + + CLI Example: + + .. code-block:: bash + + salt '*' kernelpkg.remove 4.4.0-70-generic + ''' + if release not in list_installed(): + raise CommandExecutionError('Kernel release \'{0}\' is not installed'.format(release)) + + if release == active(): + raise CommandExecutionError('Active kernel cannot be removed') + + target = '{0}-{1}'.format(_package_prefix(), release) + log.info('Removing kernel package {0}'.format(target)) + + __salt__['pkg.purge'](target) + + return {'removed': [target]} + + +def cleanup(keep_latest=True): + ''' + Remove all unused kernel packages from the system. + + keep_latest : True + In the event that the active kernel is not the latest one installed, setting this to True + will retain the latest kernel package, in addition to the active one. If False, all kernel + packages other than the active one will be removed. + + CLI Example: + + .. code-block:: bash + + salt '*' kernelpkg.cleanup + ''' + removed = [] + + # Loop over all installed kernel packages + for kernel in list_installed(): + + # Keep the active kernel package + if kernel == active(): + continue + + # Optionally keep the latest kernel package + if keep_latest and kernel == latest_installed(): + continue + + # Remove the kernel package + removed.extend(remove(kernel)['removed']) + + return {'removed': removed} + + def _package_prefix(): ''' Return static string for the package prefix @@ -211,11 +281,11 @@ def _cmp_version(item1, item2): ''' Compare function for package version sorting ''' - v1 = _LooseVersion(item1) - v2 = _LooseVersion(item2) + vers1 = _LooseVersion(item1) + vers2 = _LooseVersion(item2) - if v1 < v2: + if vers1 < vers2: return -1 - if v1 > v2: + if vers1 > vers2: return 1 return 0 diff --git a/salt/modules/kernelpkg_linux_yum.py b/salt/modules/kernelpkg_linux_yum.py index 2fb12b35f0b..39110e5a164 100644 --- a/salt/modules/kernelpkg_linux_yum.py +++ b/salt/modules/kernelpkg_linux_yum.py @@ -3,11 +3,16 @@ Manage Linux kernel packages on YUM-based systems ''' from __future__ import absolute_import +import functools import logging -# Import 3rd-party libs try: + # Import Salt libs + from salt.ext import six from salt.utils.versions import LooseVersion as _LooseVersion + from salt.exceptions import CommandExecutionError + import salt.utils.systemd + import salt.modules.yumpkg HAS_REQUIRED_LIBS = True except ImportError: HAS_REQUIRED_LIBS = False @@ -17,6 +22,9 @@ log = logging.getLogger(__name__) # Define the module's virtual name __virtualname__ = 'kernelpkg' +# Import functions from yumpkg +_yum = salt.utils.namespaced_function(salt.modules.yumpkg._yum, globals()) # pylint: disable=invalid-name, protected-access + def __virtual__(): ''' @@ -65,7 +73,10 @@ def list_installed(): if result is None: return [] - return sorted(result, cmp=_cmp_version) + if six.PY2: + return sorted(result, cmp=_cmp_version) + else: + return sorted(result, key=functools.cmp_to_key(_cmp_version)) def latest_available(): @@ -97,9 +108,9 @@ def latest_installed(): .. note:: This function may not return the same value as - :py:func:`~salt.modules.kernelpkg.active` if a new kernel + :py:func:`~salt.modules.kernelpkg_linux_yum.active` if a new kernel has been installed and the system has not yet been rebooted. - The :py:func:`~salt.modules.kernelpkg.needs_reboot` function + The :py:func:`~salt.modules.kernelpkg_linux_yum.needs_reboot` function exists to detect this condition. ''' pkgs = list_installed() @@ -180,6 +191,94 @@ def upgrade_available(): return _LooseVersion(latest_available()) > _LooseVersion(latest_installed()) +def remove(release): + ''' + Remove a specific version of the kernel. + + release + The release number of an installed kernel. This must be the entire release + number as returned by :py:func:`~salt.modules.kernelpkg_linux_yum.list_installed`, + not the package name. + + CLI Example: + + .. code-block:: bash + + salt '*' kernelpkg.remove 3.10.0-327.el7 + ''' + if release not in list_installed(): + raise CommandExecutionError('Kernel release \'{0}\' is not installed'.format(release)) + + if release == active(): + raise CommandExecutionError('Active kernel cannot be removed') + + target = '{0}-{1}'.format(_package_name(), release) + log.info('Removing kernel package {0}'.format(target)) + old = __salt__['pkg.list_pkgs']() + + # Build the command string + cmd = [] + if salt.utils.systemd.has_scope(__context__) \ + and __salt__['config.get']('systemd.scope', True): + cmd.extend(['systemd-run', '--scope']) + cmd.extend([_yum(), '-y', 'remove', target]) + + # Execute the command + out = __salt__['cmd.run_all']( + cmd, + output_loglevel='trace', + python_shell=False + ) + + # Look for the changes in installed packages + __context__.pop('pkg.list_pkgs', None) + new = __salt__['pkg.list_pkgs']() + ret = salt.utils.compare_dicts(old, new) + + # Look for command execution errors + if out['retcode'] != 0: + raise CommandExecutionError( + 'Error occurred removing package(s)', + info={'errors': [out['stderr']], 'changes': ret} + ) + + return {'removed': [target]} + + +def cleanup(keep_latest=True): + ''' + Remove all unused kernel packages from the system. + + keep_latest : True + In the event that the active kernel is not the latest one installed, setting this to True + will retain the latest kernel package, in addition to the active one. If False, all kernel + packages other than the active one will be removed. + + CLI Example: + + .. code-block:: bash + + salt '*' kernelpkg.cleanup + ''' + removed = [] + + # Loop over all installed kernel packages + for kernel in list_installed(): + + # Keep the active kernel package + if kernel == active(): + continue + + # Optionally keep the latest kernel package + if keep_latest and kernel == latest_installed(): + continue + + # Remove the kernel package + removed.extend(remove(kernel)['removed']) + + return {'removed': removed} + + def _package_name(): ''' Return static string for the package name @@ -191,11 +290,11 @@ def _cmp_version(item1, item2): ''' Compare function for package version sorting ''' - v1 = _LooseVersion(item1) - v2 = _LooseVersion(item2) + vers1 = _LooseVersion(item1) + vers2 = _LooseVersion(item2) - if v1 < v2: + if vers1 < vers2: return -1 - if v1 > v2: + if vers1 > vers2: return 1 return 0 diff --git a/salt/modules/keyboard.py b/salt/modules/keyboard.py index c7168f56c40..2f5c504be97 100644 --- a/salt/modules/keyboard.py +++ b/salt/modules/keyboard.py @@ -9,7 +9,7 @@ from __future__ import absolute_import import logging # Import salt libs -import salt.utils +import salt.utils.path log = logging.getLogger(__name__) @@ -18,7 +18,7 @@ def __virtual__(): ''' Only works with systemd or on supported POSIX-like systems ''' - if salt.utils.which('localectl') \ + if salt.utils.path.which('localectl') \ or __grains__['os_family'] in ('RedHat', 'Debian', 'Gentoo'): return True return (False, 'The keyboard exeuction module cannot be loaded: ' @@ -36,7 +36,7 @@ def get_sys(): salt '*' keyboard.get_sys ''' cmd = '' - if salt.utils.which('localectl'): + if salt.utils.path.which('localectl'): cmd = 'localectl | grep Keymap | sed -e"s/: /=/" -e"s/^[ \t]*//"' elif 'RedHat' in __grains__['os_family']: cmd = 'grep LAYOUT /etc/sysconfig/keyboard | grep -vE "^#"' @@ -59,7 +59,7 @@ def set_sys(layout): salt '*' keyboard.set_sys dvorak ''' - if salt.utils.which('localectl'): + if salt.utils.path.which('localectl'): __salt__['cmd.run']('localectl set-keymap {0}'.format(layout)) elif 'RedHat' in __grains__['os_family']: __salt__['file.sed']('/etc/sysconfig/keyboard', diff --git a/salt/modules/keystone.py b/salt/modules/keystone.py index 02765f0546d..ecd061deafe 100644 --- a/salt/modules/keystone.py +++ b/salt/modules/keystone.py @@ -54,11 +54,10 @@ from __future__ import absolute_import import logging # Import Salt Libs -import salt.ext.six as six import salt.utils.http -from salt.ext import six -# Import third party libs +# Import 3rd-party libs +from salt.ext import six HAS_KEYSTONE = False try: # pylint: disable=import-error @@ -343,7 +342,7 @@ def endpoint_list(profile=None, **connection_args): for endpoint in kstone.endpoints.list(): ret[endpoint.id] = dict((value, getattr(endpoint, value)) for value in dir(endpoint) if not value.startswith('_') and - isinstance(getattr(endpoint, value), (six.text_type, dict, bool, str))) + isinstance(getattr(endpoint, value), (six.string_types, dict, bool))) return ret @@ -496,7 +495,7 @@ def role_list(profile=None, **connection_args): for role in kstone.roles.list(): ret[role.name] = dict((value, getattr(role, value)) for value in dir(role) if not value.startswith('_') and - isinstance(getattr(role, value), (six.text_type, dict, bool, str))) + isinstance(getattr(role, value), (six.string_types, dict, bool))) return ret @@ -560,7 +559,7 @@ def service_get(service_id=None, name=None, profile=None, **connection_args): service = kstone.services.get(service_id) ret[service.name] = dict((value, getattr(service, value)) for value in dir(service) if not value.startswith('_') and - isinstance(getattr(service, value), (six.text_type, dict, bool, str))) + isinstance(getattr(service, value), (six.string_types, dict, bool))) return ret @@ -579,7 +578,7 @@ def service_list(profile=None, **connection_args): for service in kstone.services.list(): ret[service.name] = dict((value, getattr(service, value)) for value in dir(service) if not value.startswith('_') and - isinstance(getattr(service, value), (six.text_type, dict, bool, str))) + isinstance(getattr(service, value), (six.string_types, dict, bool))) return ret @@ -722,7 +721,7 @@ def tenant_get(tenant_id=None, name=None, profile=None, tenant = getattr(kstone, _TENANTS, None).get(tenant_id) ret[tenant.name] = dict((value, getattr(tenant, value)) for value in dir(tenant) if not value.startswith('_') and - isinstance(getattr(tenant, value), (six.text_type, dict, bool, str))) + isinstance(getattr(tenant, value), (six.string_types, dict, bool))) return ret @@ -775,7 +774,7 @@ def tenant_list(profile=None, **connection_args): for tenant in getattr(kstone, _TENANTS, None).list(): ret[tenant.name] = dict((value, getattr(tenant, value)) for value in dir(tenant) if not value.startswith('_') and - isinstance(getattr(tenant, value), (six.text_type, dict, bool, str))) + isinstance(getattr(tenant, value), (six.string_types, dict, bool))) return ret @@ -839,7 +838,7 @@ def tenant_update(tenant_id=None, name=None, description=None, return dict((value, getattr(updated, value)) for value in dir(updated) if not value.startswith('_') and - isinstance(getattr(updated, value), (six.text_type, dict, bool, str))) + isinstance(getattr(updated, value), (six.string_types, dict, bool))) def project_update(project_id=None, name=None, description=None, @@ -918,7 +917,7 @@ def user_list(profile=None, **connection_args): for user in kstone.users.list(): ret[user.name] = dict((value, getattr(user, value, None)) for value in dir(user) if not value.startswith('_') and - isinstance(getattr(user, value, None), (six.text_type, dict, bool, str))) + isinstance(getattr(user, value, None), (six.string_types, dict, bool))) tenant_id = getattr(user, 'tenantId', None) if tenant_id: ret[user.name]['tenant_id'] = tenant_id @@ -955,7 +954,7 @@ def user_get(user_id=None, name=None, profile=None, **connection_args): ret[user.name] = dict((value, getattr(user, value, None)) for value in dir(user) if not value.startswith('_') and - isinstance(getattr(user, value, None), (six.text_type, dict, bool, str))) + isinstance(getattr(user, value, None), (six.string_types, dict, bool))) tenant_id = getattr(user, 'tenantId', None) if tenant_id: @@ -1317,7 +1316,7 @@ tenant_id=7167a092ece84bae8cead4bf9d15bb3b for role in kstone.roles.list(user=user_id, project=tenant_id): ret[role.name] = dict((value, getattr(role, value)) for value in dir(role) if not value.startswith('_') and - isinstance(getattr(role, value), (six.text_type, dict, bool, str))) + isinstance(getattr(role, value), (six.string_types, dict, bool))) else: for role in kstone.roles.roles_for_user(user=user_id, tenant=tenant_id): ret[role.name] = {'id': role.id, diff --git a/salt/modules/kmod.py b/salt/modules/kmod.py index 615b43e7691..677bd489640 100644 --- a/salt/modules/kmod.py +++ b/salt/modules/kmod.py @@ -10,7 +10,7 @@ import re import logging # Import salt libs -import salt.utils +import salt.utils.files log = logging.getLogger(__name__) @@ -201,7 +201,7 @@ def mod_list(only_persist=False): conf = _get_modules_conf() if os.path.exists(conf): try: - with salt.utils.fopen(conf, 'r') as modules_file: + with salt.utils.files.fopen(conf, 'r') as modules_file: for line in modules_file: line = line.strip() mod_name = _strip_module_name(line) diff --git a/salt/modules/kubernetes.py b/salt/modules/kubernetes.py new file mode 100644 index 00000000000..b44759d481f --- /dev/null +++ b/salt/modules/kubernetes.py @@ -0,0 +1,1473 @@ +# -*- coding: utf-8 -*- +''' +Module for handling kubernetes calls. + +:optdepends: - kubernetes Python client +:configuration: The k8s API settings are provided either in a pillar, in + the minion's config file, or in master's config file:: + + kubernetes.user: admin + kubernetes.password: verybadpass + kubernetes.api_url: 'http://127.0.0.1:8080' + kubernetes.certificate-authority-data: '...' + kubernetes.client-certificate-data: '....n + kubernetes.client-key-data: '...' + kubernetes.certificate-authority-file: '/path/to/ca.crt' + kubernetes.client-certificate-file: '/path/to/client.crt' + kubernetes.client-key-file: '/path/to/client.key' + + +These settings can be also overrided by adding `api_url`, `api_user`, +`api_password`, `api_certificate_authority_file`, `api_client_certificate_file` +or `api_client_key_file` parameters when calling a function: + +The data format for `kubernetes.*-data` values is the same as provided in `kubeconfig`. +It's base64 encoded certificates/keys in one line. + +For an item only one field should be provided. Either a `data` or a `file` entry. +In case both are provided the `file` entry is prefered. + +.. code-block:: bash + salt '*' kubernetes.nodes api_url=http://k8s-api-server:port api_user=myuser api_password=pass + +.. versionadded: 2017.7.0 +''' + +# Import Python Futures +from __future__ import absolute_import +import os.path +import base64 +import logging +import yaml +import tempfile + +from salt.exceptions import CommandExecutionError +from salt.ext.six import iteritems +import salt.utils.files +import salt.utils.templates + +try: + import kubernetes # pylint: disable=import-self + import kubernetes.client + from kubernetes.client.rest import ApiException + from urllib3.exceptions import HTTPError + try: + # There is an API change in Kubernetes >= 2.0.0. + from kubernetes.client import V1beta1Deployment as AppsV1beta1Deployment + from kubernetes.client import V1beta1DeploymentSpec as AppsV1beta1DeploymentSpec + except ImportError: + from kubernetes.client import AppsV1beta1Deployment + from kubernetes.client import AppsV1beta1DeploymentSpec + + HAS_LIBS = True +except ImportError: + HAS_LIBS = False + +log = logging.getLogger(__name__) + +__virtualname__ = 'kubernetes' + + +def __virtual__(): + ''' + Check dependencies + ''' + if HAS_LIBS: + return __virtualname__ + + return False, 'python kubernetes library not found' + + +# pylint: disable=no-member +def _setup_conn(**kwargs): + ''' + Setup kubernetes API connection singleton + ''' + host = __salt__['config.option']('kubernetes.api_url', + 'http://localhost:8080') + username = __salt__['config.option']('kubernetes.user') + password = __salt__['config.option']('kubernetes.password') + ca_cert = __salt__['config.option']('kubernetes.certificate-authority-data') + client_cert = __salt__['config.option']('kubernetes.client-certificate-data') + client_key = __salt__['config.option']('kubernetes.client-key-data') + ca_cert_file = __salt__['config.option']('kubernetes.certificate-authority-file') + client_cert_file = __salt__['config.option']('kubernetes.client-certificate-file') + client_key_file = __salt__['config.option']('kubernetes.client-key-file') + + # Override default API settings when settings are provided + if 'api_url' in kwargs: + host = kwargs.get('api_url') + + if 'api_user' in kwargs: + username = kwargs.get('api_user') + + if 'api_password' in kwargs: + password = kwargs.get('api_password') + + if 'api_certificate_authority_file' in kwargs: + ca_cert_file = kwargs.get('api_certificate_authority_file') + + if 'api_client_certificate_file' in kwargs: + client_cert_file = kwargs.get('api_client_certificate_file') + + if 'api_client_key_file' in kwargs: + client_key_file = kwargs.get('api_client_key_file') + + if ( + kubernetes.client.configuration.host != host or + kubernetes.client.configuration.user != username or + kubernetes.client.configuration.password != password): + # Recreates API connection if settings are changed + kubernetes.client.configuration.__init__() + + kubernetes.client.configuration.host = host + kubernetes.client.configuration.user = username + kubernetes.client.configuration.passwd = password + + if ca_cert_file: + kubernetes.client.configuration.ssl_ca_cert = ca_cert_file + elif ca_cert: + with tempfile.NamedTemporaryFile(prefix='salt-kube-', delete=False) as ca: + ca.write(base64.b64decode(ca_cert)) + kubernetes.client.configuration.ssl_ca_cert = ca.name + else: + kubernetes.client.configuration.ssl_ca_cert = None + + if client_cert_file: + kubernetes.client.configuration.cert_file = client_cert_file + elif client_cert: + with tempfile.NamedTemporaryFile(prefix='salt-kube-', delete=False) as c: + c.write(base64.b64decode(client_cert)) + kubernetes.client.configuration.cert_file = c.name + else: + kubernetes.client.configuration.cert_file = None + + if client_key_file: + kubernetes.client.configuration.key_file = client_key_file + if client_key: + with tempfile.NamedTemporaryFile(prefix='salt-kube-', delete=False) as k: + k.write(base64.b64decode(client_key)) + kubernetes.client.configuration.key_file = k.name + else: + kubernetes.client.configuration.key_file = None + + +def _cleanup(**kwargs): + ca = kubernetes.client.configuration.ssl_ca_cert + cert = kubernetes.client.configuration.cert_file + key = kubernetes.client.configuration.key_file + if cert and os.path.exists(cert) and os.path.basename(cert).startswith('salt-kube-'): + salt.utils.safe_rm(cert) + if key and os.path.exists(key) and os.path.basename(key).startswith('salt-kube-'): + salt.utils.safe_rm(key) + if ca and os.path.exists(ca) and os.path.basename(ca).startswith('salt-kube-'): + salt.utils.safe_rm(ca) + + +def ping(**kwargs): + ''' + Checks connections with the kubernetes API server. + Returns True if the connection can be established, False otherwise. + + CLI Example: + salt '*' kubernetes.ping + ''' + status = True + try: + nodes(**kwargs) + except CommandExecutionError: + status = False + + return status + + +def nodes(**kwargs): + ''' + Return the names of the nodes composing the kubernetes cluster + + CLI Examples:: + + salt '*' kubernetes.nodes + salt '*' kubernetes.nodes api_url=http://myhost:port api_user=my-user + ''' + _setup_conn(**kwargs) + try: + api_instance = kubernetes.client.CoreV1Api() + api_response = api_instance.list_node() + + return [k8s_node['metadata']['name'] for k8s_node in api_response.to_dict().get('items')] + except (ApiException, HTTPError) as exc: + if isinstance(exc, ApiException) and exc.status == 404: + return None + else: + log.exception( + 'Exception when calling CoreV1Api->list_node: {0}'.format(exc) + ) + raise CommandExecutionError(exc) + finally: + _cleanup() + + +def node(name, **kwargs): + ''' + Return the details of the node identified by the specified name + + CLI Examples:: + + salt '*' kubernetes.node name='minikube' + ''' + _setup_conn(**kwargs) + try: + api_instance = kubernetes.client.CoreV1Api() + api_response = api_instance.list_node() + except (ApiException, HTTPError) as exc: + if isinstance(exc, ApiException) and exc.status == 404: + return None + else: + log.exception( + 'Exception when calling CoreV1Api->list_node: {0}'.format(exc) + ) + raise CommandExecutionError(exc) + finally: + _cleanup() + + for k8s_node in api_response.items: + if k8s_node.metadata.name == name: + return k8s_node.to_dict() + + return None + + +def node_labels(name, **kwargs): + ''' + Return the labels of the node identified by the specified name + + CLI Examples:: + + salt '*' kubernetes.node_labels name="minikube" + ''' + match = node(name, **kwargs) + + if match is not None: + return match.metadata.labels + + return {} + + +def node_add_label(node_name, label_name, label_value, **kwargs): + ''' + Set the value of the label identified by `label_name` to `label_value` on + the node identified by the name `node_name`. + Creates the lable if not present. + + CLI Examples:: + + salt '*' kubernetes.node_add_label node_name="minikube" \ + label_name="foo" label_value="bar" + ''' + _setup_conn(**kwargs) + try: + api_instance = kubernetes.client.CoreV1Api() + body = { + 'metadata': { + 'labels': { + label_name: label_value} + } + } + api_response = api_instance.patch_node(node_name, body) + return api_response + except (ApiException, HTTPError) as exc: + if isinstance(exc, ApiException) and exc.status == 404: + return None + else: + log.exception( + 'Exception when calling CoreV1Api->patch_node: {0}'.format(exc) + ) + raise CommandExecutionError(exc) + finally: + _cleanup() + + return None + + +def node_remove_label(node_name, label_name, **kwargs): + ''' + Removes the label identified by `label_name` from + the node identified by the name `node_name`. + + CLI Examples:: + + salt '*' kubernetes.node_remove_label node_name="minikube" \ + label_name="foo" + ''' + _setup_conn(**kwargs) + try: + api_instance = kubernetes.client.CoreV1Api() + body = { + 'metadata': { + 'labels': { + label_name: None} + } + } + api_response = api_instance.patch_node(node_name, body) + return api_response + except (ApiException, HTTPError) as exc: + if isinstance(exc, ApiException) and exc.status == 404: + return None + else: + log.exception( + 'Exception when calling CoreV1Api->patch_node: {0}'.format(exc) + ) + raise CommandExecutionError(exc) + finally: + _cleanup() + + return None + + +def namespaces(**kwargs): + ''' + Return the names of the available namespaces + + CLI Examples:: + + salt '*' kubernetes.namespaces + salt '*' kubernetes.namespaces api_url=http://myhost:port api_user=my-user + ''' + _setup_conn(**kwargs) + try: + api_instance = kubernetes.client.CoreV1Api() + api_response = api_instance.list_namespace() + + return [nms['metadata']['name'] for nms in api_response.to_dict().get('items')] + except (ApiException, HTTPError) as exc: + if isinstance(exc, ApiException) and exc.status == 404: + return None + else: + log.exception( + 'Exception when calling CoreV1Api->list_namespace: ' + '{0}'.format(exc) + ) + raise CommandExecutionError(exc) + finally: + _cleanup() + + +def deployments(namespace='default', **kwargs): + ''' + Return a list of kubernetes deployments defined in the namespace + + CLI Examples:: + + salt '*' kubernetes.deployments + salt '*' kubernetes.deployments namespace=default + ''' + _setup_conn(**kwargs) + try: + api_instance = kubernetes.client.ExtensionsV1beta1Api() + api_response = api_instance.list_namespaced_deployment(namespace) + + return [dep['metadata']['name'] for dep in api_response.to_dict().get('items')] + except (ApiException, HTTPError) as exc: + if isinstance(exc, ApiException) and exc.status == 404: + return None + else: + log.exception( + 'Exception when calling ' + 'ExtensionsV1beta1Api->list_namespaced_deployment: ' + '{0}'.format(exc) + ) + raise CommandExecutionError(exc) + finally: + _cleanup() + + +def services(namespace='default', **kwargs): + ''' + Return a list of kubernetes services defined in the namespace + + CLI Examples:: + + salt '*' kubernetes.services + salt '*' kubernetes.services namespace=default + ''' + _setup_conn(**kwargs) + try: + api_instance = kubernetes.client.CoreV1Api() + api_response = api_instance.list_namespaced_service(namespace) + + return [srv['metadata']['name'] for srv in api_response.to_dict().get('items')] + except (ApiException, HTTPError) as exc: + if isinstance(exc, ApiException) and exc.status == 404: + return None + else: + log.exception( + 'Exception when calling ' + 'CoreV1Api->list_namespaced_service: {0}'.format(exc) + ) + raise CommandExecutionError(exc) + finally: + _cleanup() + + +def pods(namespace='default', **kwargs): + ''' + Return a list of kubernetes pods defined in the namespace + + CLI Examples:: + + salt '*' kubernetes.pods + salt '*' kubernetes.pods namespace=default + ''' + _setup_conn(**kwargs) + try: + api_instance = kubernetes.client.CoreV1Api() + api_response = api_instance.list_namespaced_pod(namespace) + + return [pod['metadata']['name'] for pod in api_response.to_dict().get('items')] + except (ApiException, HTTPError) as exc: + if isinstance(exc, ApiException) and exc.status == 404: + return None + else: + log.exception( + 'Exception when calling ' + 'CoreV1Api->list_namespaced_pod: {0}'.format(exc) + ) + raise CommandExecutionError(exc) + finally: + _cleanup() + + +def secrets(namespace='default', **kwargs): + ''' + Return a list of kubernetes secrets defined in the namespace + + CLI Examples:: + + salt '*' kubernetes.secrets + salt '*' kubernetes.secrets namespace=default + ''' + _setup_conn(**kwargs) + try: + api_instance = kubernetes.client.CoreV1Api() + api_response = api_instance.list_namespaced_secret(namespace) + + return [secret['metadata']['name'] for secret in api_response.to_dict().get('items')] + except (ApiException, HTTPError) as exc: + if isinstance(exc, ApiException) and exc.status == 404: + return None + else: + log.exception( + 'Exception when calling ' + 'CoreV1Api->list_namespaced_secret: {0}'.format(exc) + ) + raise CommandExecutionError(exc) + finally: + _cleanup() + + +def configmaps(namespace='default', **kwargs): + ''' + Return a list of kubernetes configmaps defined in the namespace + + CLI Examples:: + + salt '*' kubernetes.configmaps + salt '*' kubernetes.configmaps namespace=default + ''' + _setup_conn(**kwargs) + try: + api_instance = kubernetes.client.CoreV1Api() + api_response = api_instance.list_namespaced_config_map(namespace) + + return [secret['metadata']['name'] for secret in api_response.to_dict().get('items')] + except (ApiException, HTTPError) as exc: + if isinstance(exc, ApiException) and exc.status == 404: + return None + else: + log.exception( + 'Exception when calling ' + 'CoreV1Api->list_namespaced_config_map: {0}'.format(exc) + ) + raise CommandExecutionError(exc) + finally: + _cleanup() + + +def show_deployment(name, namespace='default', **kwargs): + ''' + Return the kubernetes deployment defined by name and namespace + + CLI Examples:: + + salt '*' kubernetes.show_deployment my-nginx default + salt '*' kubernetes.show_deployment name=my-nginx namespace=default + ''' + _setup_conn(**kwargs) + try: + api_instance = kubernetes.client.ExtensionsV1beta1Api() + api_response = api_instance.read_namespaced_deployment(name, namespace) + + return api_response.to_dict() + except (ApiException, HTTPError) as exc: + if isinstance(exc, ApiException) and exc.status == 404: + return None + else: + log.exception( + 'Exception when calling ' + 'ExtensionsV1beta1Api->read_namespaced_deployment: ' + '{0}'.format(exc) + ) + raise CommandExecutionError(exc) + finally: + _cleanup() + + +def show_service(name, namespace='default', **kwargs): + ''' + Return the kubernetes service defined by name and namespace + + CLI Examples:: + + salt '*' kubernetes.show_service my-nginx default + salt '*' kubernetes.show_service name=my-nginx namespace=default + ''' + _setup_conn(**kwargs) + try: + api_instance = kubernetes.client.CoreV1Api() + api_response = api_instance.read_namespaced_service(name, namespace) + + return api_response.to_dict() + except (ApiException, HTTPError) as exc: + if isinstance(exc, ApiException) and exc.status == 404: + return None + else: + log.exception( + 'Exception when calling ' + 'CoreV1Api->read_namespaced_service: {0}'.format(exc) + ) + raise CommandExecutionError(exc) + finally: + _cleanup() + + +def show_pod(name, namespace='default', **kwargs): + ''' + Return POD information for a given pod name defined in the namespace + + CLI Examples:: + + salt '*' kubernetes.show_pod guestbook-708336848-fqr2x + salt '*' kubernetes.show_pod guestbook-708336848-fqr2x namespace=default + ''' + _setup_conn(**kwargs) + try: + api_instance = kubernetes.client.CoreV1Api() + api_response = api_instance.read_namespaced_pod(name, namespace) + + return api_response.to_dict() + except (ApiException, HTTPError) as exc: + if isinstance(exc, ApiException) and exc.status == 404: + return None + else: + log.exception( + 'Exception when calling ' + 'CoreV1Api->read_namespaced_pod: {0}'.format(exc) + ) + raise CommandExecutionError(exc) + finally: + _cleanup() + + +def show_namespace(name, **kwargs): + ''' + Return information for a given namespace defined by the specified name + + CLI Examples:: + + salt '*' kubernetes.show_namespace kube-system + ''' + _setup_conn(**kwargs) + try: + api_instance = kubernetes.client.CoreV1Api() + api_response = api_instance.read_namespace(name) + + return api_response.to_dict() + except (ApiException, HTTPError) as exc: + if isinstance(exc, ApiException) and exc.status == 404: + return None + else: + log.exception( + 'Exception when calling ' + 'CoreV1Api->read_namespace: {0}'.format(exc) + ) + raise CommandExecutionError(exc) + finally: + _cleanup() + + +def show_secret(name, namespace='default', decode=False, **kwargs): + ''' + Return the kubernetes secret defined by name and namespace. + The secrets can be decoded if specified by the user. Warning: this has + security implications. + + CLI Examples:: + + salt '*' kubernetes.show_secret confidential default + salt '*' kubernetes.show_secret name=confidential namespace=default + salt '*' kubernetes.show_secret name=confidential decode=True + ''' + _setup_conn(**kwargs) + try: + api_instance = kubernetes.client.CoreV1Api() + api_response = api_instance.read_namespaced_secret(name, namespace) + + if api_response.data and (decode or decode == 'True'): + for key in api_response.data: + value = api_response.data[key] + api_response.data[key] = base64.b64decode(value) + + return api_response.to_dict() + except (ApiException, HTTPError) as exc: + if isinstance(exc, ApiException) and exc.status == 404: + return None + else: + log.exception( + 'Exception when calling ' + 'CoreV1Api->read_namespaced_secret: ' + '{0}'.format(exc) + ) + raise CommandExecutionError(exc) + finally: + _cleanup() + + +def show_configmap(name, namespace='default', **kwargs): + ''' + Return the kubernetes configmap defined by name and namespace. + + CLI Examples:: + + salt '*' kubernetes.show_configmap game-config default + salt '*' kubernetes.show_configmap name=game-config namespace=default + ''' + _setup_conn(**kwargs) + try: + api_instance = kubernetes.client.CoreV1Api() + api_response = api_instance.read_namespaced_config_map( + name, + namespace) + + return api_response.to_dict() + except (ApiException, HTTPError) as exc: + if isinstance(exc, ApiException) and exc.status == 404: + return None + else: + log.exception( + 'Exception when calling ' + 'CoreV1Api->read_namespaced_config_map: ' + '{0}'.format(exc) + ) + raise CommandExecutionError(exc) + finally: + _cleanup() + + +def delete_deployment(name, namespace='default', **kwargs): + ''' + Deletes the kubernetes deployment defined by name and namespace + + CLI Examples:: + + salt '*' kubernetes.delete_deployment my-nginx + salt '*' kubernetes.delete_deployment name=my-nginx namespace=default + ''' + _setup_conn(**kwargs) + body = kubernetes.client.V1DeleteOptions(orphan_dependents=True) + + try: + api_instance = kubernetes.client.ExtensionsV1beta1Api() + api_response = api_instance.delete_namespaced_deployment( + name=name, + namespace=namespace, + body=body) + return api_response.to_dict() + except (ApiException, HTTPError) as exc: + if isinstance(exc, ApiException) and exc.status == 404: + return None + else: + log.exception( + 'Exception when calling ' + 'ExtensionsV1beta1Api->delete_namespaced_deployment: ' + '{0}'.format(exc) + ) + raise CommandExecutionError(exc) + finally: + _cleanup() + + +def delete_service(name, namespace='default', **kwargs): + ''' + Deletes the kubernetes service defined by name and namespace + + CLI Examples:: + + salt '*' kubernetes.delete_service my-nginx default + salt '*' kubernetes.delete_service name=my-nginx namespace=default + ''' + _setup_conn(**kwargs) + + try: + api_instance = kubernetes.client.CoreV1Api() + api_response = api_instance.delete_namespaced_service( + name=name, + namespace=namespace) + + return api_response.to_dict() + except (ApiException, HTTPError) as exc: + if isinstance(exc, ApiException) and exc.status == 404: + return None + else: + log.exception( + 'Exception when calling CoreV1Api->delete_namespaced_service: ' + '{0}'.format(exc) + ) + raise CommandExecutionError(exc) + finally: + _cleanup() + + +def delete_pod(name, namespace='default', **kwargs): + ''' + Deletes the kubernetes pod defined by name and namespace + + CLI Examples:: + + salt '*' kubernetes.delete_pod guestbook-708336848-5nl8c default + salt '*' kubernetes.delete_pod name=guestbook-708336848-5nl8c namespace=default + ''' + _setup_conn(**kwargs) + body = kubernetes.client.V1DeleteOptions(orphan_dependents=True) + + try: + api_instance = kubernetes.client.CoreV1Api() + api_response = api_instance.delete_namespaced_pod( + name=name, + namespace=namespace, + body=body) + + return api_response.to_dict() + except (ApiException, HTTPError) as exc: + if isinstance(exc, ApiException) and exc.status == 404: + return None + else: + log.exception( + 'Exception when calling ' + 'CoreV1Api->delete_namespaced_pod: {0}'.format(exc) + ) + raise CommandExecutionError(exc) + finally: + _cleanup() + + +def delete_namespace(name, **kwargs): + ''' + Deletes the kubernetes namespace defined by name + + CLI Examples:: + + salt '*' kubernetes.delete_namespace salt + salt '*' kubernetes.delete_namespace name=salt + ''' + _setup_conn(**kwargs) + body = kubernetes.client.V1DeleteOptions(orphan_dependents=True) + + try: + api_instance = kubernetes.client.CoreV1Api() + api_response = api_instance.delete_namespace(name=name, body=body) + return api_response.to_dict() + except (ApiException, HTTPError) as exc: + if isinstance(exc, ApiException) and exc.status == 404: + return None + else: + log.exception( + 'Exception when calling ' + 'CoreV1Api->delete_namespace: ' + '{0}'.format(exc) + ) + raise CommandExecutionError(exc) + finally: + _cleanup() + + +def delete_secret(name, namespace='default', **kwargs): + ''' + Deletes the kubernetes secret defined by name and namespace + + CLI Examples:: + + salt '*' kubernetes.delete_secret confidential default + salt '*' kubernetes.delete_secret name=confidential namespace=default + ''' + _setup_conn(**kwargs) + body = kubernetes.client.V1DeleteOptions(orphan_dependents=True) + + try: + api_instance = kubernetes.client.CoreV1Api() + api_response = api_instance.delete_namespaced_secret( + name=name, + namespace=namespace, + body=body) + + return api_response.to_dict() + except (ApiException, HTTPError) as exc: + if isinstance(exc, ApiException) and exc.status == 404: + return None + else: + log.exception( + 'Exception when calling CoreV1Api->delete_namespaced_secret: ' + '{0}'.format(exc) + ) + raise CommandExecutionError(exc) + finally: + _cleanup() + + +def delete_configmap(name, namespace='default', **kwargs): + ''' + Deletes the kubernetes configmap defined by name and namespace + + CLI Examples:: + + salt '*' kubernetes.delete_configmap settings default + salt '*' kubernetes.delete_configmap name=settings namespace=default + ''' + _setup_conn(**kwargs) + body = kubernetes.client.V1DeleteOptions(orphan_dependents=True) + + try: + api_instance = kubernetes.client.CoreV1Api() + api_response = api_instance.delete_namespaced_config_map( + name=name, + namespace=namespace, + body=body) + + return api_response.to_dict() + except (ApiException, HTTPError) as exc: + if isinstance(exc, ApiException) and exc.status == 404: + return None + else: + log.exception( + 'Exception when calling ' + 'CoreV1Api->delete_namespaced_config_map: ' + '{0}'.format(exc) + ) + raise CommandExecutionError(exc) + finally: + _cleanup() + + +def create_deployment( + name, + namespace, + metadata, + spec, + source, + template, + saltenv, + **kwargs): + ''' + Creates the kubernetes deployment as defined by the user. + ''' + body = __create_object_body( + kind='Deployment', + obj_class=AppsV1beta1Deployment, + spec_creator=__dict_to_deployment_spec, + name=name, + namespace=namespace, + metadata=metadata, + spec=spec, + source=source, + template=template, + saltenv=saltenv) + + _setup_conn(**kwargs) + + try: + api_instance = kubernetes.client.ExtensionsV1beta1Api() + api_response = api_instance.create_namespaced_deployment( + namespace, body) + + return api_response.to_dict() + except (ApiException, HTTPError) as exc: + if isinstance(exc, ApiException) and exc.status == 404: + return None + else: + log.exception( + 'Exception when calling ' + 'ExtensionsV1beta1Api->create_namespaced_deployment: ' + '{0}'.format(exc) + ) + raise CommandExecutionError(exc) + finally: + _cleanup() + + +def create_pod( + name, + namespace, + metadata, + spec, + source, + template, + saltenv, + **kwargs): + ''' + Creates the kubernetes deployment as defined by the user. + ''' + body = __create_object_body( + kind='Pod', + obj_class=kubernetes.client.V1Pod, + spec_creator=__dict_to_pod_spec, + name=name, + namespace=namespace, + metadata=metadata, + spec=spec, + source=source, + template=template, + saltenv=saltenv) + + _setup_conn(**kwargs) + + try: + api_instance = kubernetes.client.CoreV1Api() + api_response = api_instance.create_namespaced_pod( + namespace, body) + + return api_response.to_dict() + except (ApiException, HTTPError) as exc: + if isinstance(exc, ApiException) and exc.status == 404: + return None + else: + log.exception( + 'Exception when calling ' + 'CoreV1Api->create_namespaced_pod: ' + '{0}'.format(exc) + ) + raise CommandExecutionError(exc) + finally: + _cleanup() + + +def create_service( + name, + namespace, + metadata, + spec, + source, + template, + saltenv, + **kwargs): + ''' + Creates the kubernetes service as defined by the user. + ''' + body = __create_object_body( + kind='Service', + obj_class=kubernetes.client.V1Service, + spec_creator=__dict_to_service_spec, + name=name, + namespace=namespace, + metadata=metadata, + spec=spec, + source=source, + template=template, + saltenv=saltenv) + + _setup_conn(**kwargs) + + try: + api_instance = kubernetes.client.CoreV1Api() + api_response = api_instance.create_namespaced_service( + namespace, body) + + return api_response.to_dict() + except (ApiException, HTTPError) as exc: + if isinstance(exc, ApiException) and exc.status == 404: + return None + else: + log.exception( + 'Exception when calling ' + 'CoreV1Api->create_namespaced_service: {0}'.format(exc) + ) + raise CommandExecutionError(exc) + finally: + _cleanup() + + +def create_secret( + name, + namespace, + data, + source, + template, + saltenv, + **kwargs): + ''' + Creates the kubernetes secret as defined by the user. + ''' + if source: + data = __read_and_render_yaml_file(source, template, saltenv) + elif data is None: + data = {} + + data = __enforce_only_strings_dict(data) + + # encode the secrets using base64 as required by kubernetes + for key in data: + data[key] = base64.b64encode(data[key]) + + body = kubernetes.client.V1Secret( + metadata=__dict_to_object_meta(name, namespace, {}), + data=data) + + _setup_conn(**kwargs) + + try: + api_instance = kubernetes.client.CoreV1Api() + api_response = api_instance.create_namespaced_secret( + namespace, body) + + return api_response.to_dict() + except (ApiException, HTTPError) as exc: + if isinstance(exc, ApiException) and exc.status == 404: + return None + else: + log.exception( + 'Exception when calling ' + 'CoreV1Api->create_namespaced_secret: {0}'.format(exc) + ) + raise CommandExecutionError(exc) + finally: + _cleanup() + + +def create_configmap( + name, + namespace, + data, + source, + template, + saltenv, + **kwargs): + ''' + Creates the kubernetes configmap as defined by the user. + ''' + if source: + data = __read_and_render_yaml_file(source, template, saltenv) + elif data is None: + data = {} + + data = __enforce_only_strings_dict(data) + + body = kubernetes.client.V1ConfigMap( + metadata=__dict_to_object_meta(name, namespace, {}), + data=data) + + _setup_conn(**kwargs) + + try: + api_instance = kubernetes.client.CoreV1Api() + api_response = api_instance.create_namespaced_config_map( + namespace, body) + + return api_response.to_dict() + except (ApiException, HTTPError) as exc: + if isinstance(exc, ApiException) and exc.status == 404: + return None + else: + log.exception( + 'Exception when calling ' + 'CoreV1Api->create_namespaced_config_map: {0}'.format(exc) + ) + raise CommandExecutionError(exc) + finally: + _cleanup() + + +def create_namespace( + name, + **kwargs): + ''' + Creates a namespace with the specified name. + + CLI Example: + salt '*' kubernetes.create_namespace salt + salt '*' kubernetes.create_namespace name=salt + ''' + + meta_obj = kubernetes.client.V1ObjectMeta(name=name) + body = kubernetes.client.V1Namespace(metadata=meta_obj) + body.metadata.name = name + + _setup_conn(**kwargs) + + try: + api_instance = kubernetes.client.CoreV1Api() + api_response = api_instance.create_namespace(body) + + return api_response.to_dict() + except (ApiException, HTTPError) as exc: + if isinstance(exc, ApiException) and exc.status == 404: + return None + else: + log.exception( + 'Exception when calling ' + 'CoreV1Api->create_namespace: ' + '{0}'.format(exc) + ) + raise CommandExecutionError(exc) + finally: + _cleanup() + + +def replace_deployment(name, + metadata, + spec, + source, + template, + saltenv, + namespace='default', + **kwargs): + ''' + Replaces an existing deployment with a new one defined by name and + namespace, having the specificed metadata and spec. + ''' + body = __create_object_body( + kind='Deployment', + obj_class=AppsV1beta1Deployment, + spec_creator=__dict_to_deployment_spec, + name=name, + namespace=namespace, + metadata=metadata, + spec=spec, + source=source, + template=template, + saltenv=saltenv) + + _setup_conn(**kwargs) + + try: + api_instance = kubernetes.client.ExtensionsV1beta1Api() + api_response = api_instance.replace_namespaced_deployment( + name, namespace, body) + + return api_response.to_dict() + except (ApiException, HTTPError) as exc: + if isinstance(exc, ApiException) and exc.status == 404: + return None + else: + log.exception( + 'Exception when calling ' + 'ExtensionsV1beta1Api->replace_namespaced_deployment: ' + '{0}'.format(exc) + ) + raise CommandExecutionError(exc) + finally: + _cleanup() + + +def replace_service(name, + metadata, + spec, + source, + template, + old_service, + saltenv, + namespace='default', + **kwargs): + ''' + Replaces an existing service with a new one defined by name and namespace, + having the specificed metadata and spec. + ''' + body = __create_object_body( + kind='Service', + obj_class=kubernetes.client.V1Service, + spec_creator=__dict_to_service_spec, + name=name, + namespace=namespace, + metadata=metadata, + spec=spec, + source=source, + template=template, + saltenv=saltenv) + + # Some attributes have to be preserved + # otherwise exceptions will be thrown + body.spec.cluster_ip = old_service['spec']['cluster_ip'] + body.metadata.resource_version = old_service['metadata']['resource_version'] + + _setup_conn(**kwargs) + + try: + api_instance = kubernetes.client.CoreV1Api() + api_response = api_instance.replace_namespaced_service( + name, namespace, body) + + return api_response.to_dict() + except (ApiException, HTTPError) as exc: + if isinstance(exc, ApiException) and exc.status == 404: + return None + else: + log.exception( + 'Exception when calling ' + 'CoreV1Api->replace_namespaced_service: {0}'.format(exc) + ) + raise CommandExecutionError(exc) + finally: + _cleanup() + + +def replace_secret(name, + data, + source, + template, + saltenv, + namespace='default', + **kwargs): + ''' + Replaces an existing secret with a new one defined by name and namespace, + having the specificed data. + ''' + if source: + data = __read_and_render_yaml_file(source, template, saltenv) + elif data is None: + data = {} + + data = __enforce_only_strings_dict(data) + + # encode the secrets using base64 as required by kubernetes + for key in data: + data[key] = base64.b64encode(data[key]) + + body = kubernetes.client.V1Secret( + metadata=__dict_to_object_meta(name, namespace, {}), + data=data) + + _setup_conn(**kwargs) + + try: + api_instance = kubernetes.client.CoreV1Api() + api_response = api_instance.replace_namespaced_secret( + name, namespace, body) + + return api_response.to_dict() + except (ApiException, HTTPError) as exc: + if isinstance(exc, ApiException) and exc.status == 404: + return None + else: + log.exception( + 'Exception when calling ' + 'CoreV1Api->replace_namespaced_secret: {0}'.format(exc) + ) + raise CommandExecutionError(exc) + finally: + _cleanup() + + +def replace_configmap(name, + data, + source, + template, + saltenv, + namespace='default', + **kwargs): + ''' + Replaces an existing configmap with a new one defined by name and + namespace, having the specificed data. + ''' + if source: + data = __read_and_render_yaml_file(source, template, saltenv) + + data = __enforce_only_strings_dict(data) + + body = kubernetes.client.V1ConfigMap( + metadata=__dict_to_object_meta(name, namespace, {}), + data=data) + + _setup_conn(**kwargs) + + try: + api_instance = kubernetes.client.CoreV1Api() + api_response = api_instance.replace_namespaced_config_map( + name, namespace, body) + + return api_response.to_dict() + except (ApiException, HTTPError) as exc: + if isinstance(exc, ApiException) and exc.status == 404: + return None + else: + log.exception( + 'Exception when calling ' + 'CoreV1Api->replace_namespaced_configmap: {0}'.format(exc) + ) + raise CommandExecutionError(exc) + finally: + _cleanup() + + +def __create_object_body(kind, + obj_class, + spec_creator, + name, + namespace, + metadata, + spec, + source, + template, + saltenv): + ''' + Create a Kubernetes Object body instance. + ''' + if source: + src_obj = __read_and_render_yaml_file(source, template, saltenv) + if ( + not isinstance(src_obj, dict) or + 'kind' not in src_obj or + src_obj['kind'] != kind): + raise CommandExecutionError( + 'The source file should define only ' + 'a {0} object'.format(kind)) + + if 'metadata' in src_obj: + metadata = src_obj['metadata'] + if 'spec' in src_obj: + spec = src_obj['spec'] + + return obj_class( + metadata=__dict_to_object_meta(name, namespace, metadata), + spec=spec_creator(spec)) + + +def __read_and_render_yaml_file(source, + template, + saltenv): + ''' + Read a yaml file and, if needed, renders that using the specifieds + templating. Returns the python objects defined inside of the file. + ''' + sfn = __salt__['cp.cache_file'](source, saltenv) + if not sfn: + raise CommandExecutionError( + 'Source file \'{0}\' not found'.format(source)) + + with salt.utils.files.fopen(sfn, 'r') as src: + contents = src.read() + + if template: + if template in salt.utils.templates.TEMPLATE_REGISTRY: + # TODO: should we allow user to set also `context` like # pylint: disable=fixme + # `file.managed` does? + # Apply templating + data = salt.utils.templates.TEMPLATE_REGISTRY[template]( + contents, + from_str=True, + to_str=True, + saltenv=saltenv, + grains=__grains__, + pillar=__pillar__, + salt=__salt__, + opts=__opts__) + + if not data['result']: + # Failed to render the template + raise CommandExecutionError( + 'Failed to render file path with error: ' + '{0}'.format(data['data']) + ) + + contents = data['data'].encode('utf-8') + else: + raise CommandExecutionError( + 'Unknown template specified: {0}'.format( + template)) + + return yaml.load(contents) + + +def __dict_to_object_meta(name, namespace, metadata): + ''' + Converts a dictionary into kubernetes ObjectMetaV1 instance. + ''' + meta_obj = kubernetes.client.V1ObjectMeta() + meta_obj.namespace = namespace + for key, value in iteritems(metadata): + if hasattr(meta_obj, key): + setattr(meta_obj, key, value) + + if meta_obj.name != name: + log.warning( + 'The object already has a name attribute, overwriting it with ' + 'the one defined inside of salt') + meta_obj.name = name + + return meta_obj + + +def __dict_to_deployment_spec(spec): + ''' + Converts a dictionary into kubernetes AppsV1beta1DeploymentSpec instance. + ''' + spec_obj = AppsV1beta1DeploymentSpec() + for key, value in iteritems(spec): + if hasattr(spec_obj, key): + setattr(spec_obj, key, value) + + return spec_obj + + +def __dict_to_pod_spec(spec): + ''' + Converts a dictionary into kubernetes V1PodSpec instance. + ''' + spec_obj = kubernetes.client.V1PodSpec() + for key, value in iteritems(spec): + if hasattr(spec_obj, key): + setattr(spec_obj, key, value) + + return spec_obj + + +def __dict_to_service_spec(spec): + ''' + Converts a dictionary into kubernetes V1ServiceSpec instance. + ''' + spec_obj = kubernetes.client.V1ServiceSpec() + for key, value in iteritems(spec): # pylint: disable=too-many-nested-blocks + if key == 'ports': + spec_obj.ports = [] + for port in value: + kube_port = kubernetes.client.V1ServicePort() + if isinstance(port, dict): + for port_key, port_value in iteritems(port): + if hasattr(kube_port, port_key): + setattr(kube_port, port_key, port_value) + else: + kube_port.port = port + spec_obj.ports.append(kube_port) + elif hasattr(spec_obj, key): + setattr(spec_obj, key, value) + + return spec_obj + + +def __enforce_only_strings_dict(dictionary): + ''' + Returns a dictionary that has string keys and values. + ''' + ret = {} + + for key, value in iteritems(dictionary): + ret[str(key)] = str(value) + + return ret diff --git a/salt/modules/launchctl.py b/salt/modules/launchctl.py index 62d8557d15b..fa951b2beef 100644 --- a/salt/modules/launchctl.py +++ b/salt/modules/launchctl.py @@ -21,9 +21,12 @@ import re # Import salt libs import salt.utils +import salt.utils.platform +import salt.utils.stringutils import salt.utils.decorators as decorators +import salt.utils.files from salt.utils.versions import LooseVersion as _LooseVersion -import salt.ext.six as six +from salt.ext import six # Set up logging log = logging.getLogger(__name__) @@ -38,7 +41,7 @@ def __virtual__(): ''' Only work on MacOS ''' - if not salt.utils.is_darwin(): + if not salt.utils.platform.is_darwin(): return (False, 'Failed to load the mac_service module:\n' 'Only available on macOS systems.') @@ -88,7 +91,7 @@ def _available_services(): try: # This assumes most of the plist files # will be already in XML format - with salt.utils.fopen(file_path): + with salt.utils.files.fopen(file_path): plist = plistlib.readPlist(true_path) except Exception: @@ -102,7 +105,7 @@ def _available_services(): plist = plistlib.readPlistFromString(plist_xml) else: plist = plistlib.readPlistFromBytes( - salt.utils.to_bytes(plist_xml)) + salt.utils.stringutils.to_bytes(plist_xml)) try: available_services[plist.Label.lower()] = { diff --git a/salt/modules/layman.py b/salt/modules/layman.py index 3e201ba4ce3..0cb322557f8 100644 --- a/salt/modules/layman.py +++ b/salt/modules/layman.py @@ -4,7 +4,7 @@ Support for Layman ''' from __future__ import absolute_import -import salt.utils +import salt.utils.path import salt.exceptions @@ -12,7 +12,7 @@ def __virtual__(): ''' Only work on Gentoo systems with layman installed ''' - if __grains__['os'] == 'Gentoo' and salt.utils.which('layman'): + if __grains__['os'] == 'Gentoo' and salt.utils.path.which('layman'): return 'layman' return (False, 'layman execution module cannot be loaded: only available on Gentoo with layman installed.') diff --git a/salt/modules/ldap3.py b/salt/modules/ldap3.py index 8b29a77af82..d4c9488ed3e 100644 --- a/salt/modules/ldap3.py +++ b/salt/modules/ldap3.py @@ -23,7 +23,7 @@ try: except ImportError: pass import logging -import salt.ext.six as six +from salt.ext import six import sys log = logging.getLogger(__name__) diff --git a/salt/modules/libcloud_compute.py b/salt/modules/libcloud_compute.py index 3e226072d70..8bab210a300 100644 --- a/salt/modules/libcloud_compute.py +++ b/salt/modules/libcloud_compute.py @@ -37,8 +37,8 @@ import logging import os.path # Import salt libs +import salt.utils.args import salt.utils.compat -from salt.utils import clean_kwargs from salt.utils.versions import LooseVersion as _LooseVersion log = logging.getLogger(__name__) @@ -106,7 +106,7 @@ def list_nodes(profile, **libcloud_kwargs): salt myminion libcloud_compute.list_nodes profile1 ''' conn = _get_driver(profile=profile) - libcloud_kwargs = clean_kwargs(**libcloud_kwargs) + libcloud_kwargs = salt.utils.args.clean_kwargs(**libcloud_kwargs) nodes = conn.list_nodes(**libcloud_kwargs) ret = [] for node in nodes: @@ -135,7 +135,7 @@ def list_sizes(profile, location_id=None, **libcloud_kwargs): salt myminion libcloud_compute.list_sizes profile1 us-east1 ''' conn = _get_driver(profile=profile) - libcloud_kwargs = clean_kwargs(**libcloud_kwargs) + libcloud_kwargs = salt.utils.args.clean_kwargs(**libcloud_kwargs) if location_id is not None: locations = [loc for loc in conn.list_locations() if loc.id == location_id] if len(locations) == 0: @@ -168,7 +168,7 @@ def list_locations(profile, **libcloud_kwargs): salt myminion libcloud_compute.list_locations profile1 ''' conn = _get_driver(profile=profile) - libcloud_kwargs = clean_kwargs(**libcloud_kwargs) + libcloud_kwargs = salt.utils.args.clean_kwargs(**libcloud_kwargs) locations = conn.list_locations(**libcloud_kwargs) ret = [] @@ -242,7 +242,7 @@ def list_volumes(profile, **libcloud_kwargs): salt myminion libcloud_compute.list_volumes profile1 ''' conn = _get_driver(profile=profile) - libcloud_kwargs = clean_kwargs(**libcloud_kwargs) + libcloud_kwargs = salt.utils.args.clean_kwargs(**libcloud_kwargs) volumes = conn.list_volumes(**libcloud_kwargs) ret = [] @@ -271,7 +271,7 @@ def list_volume_snapshots(volume_id, profile, **libcloud_kwargs): salt myminion libcloud_compute.list_volume_snapshots vol1 profile1 ''' conn = _get_driver(profile=profile) - libcloud_kwargs = clean_kwargs(**libcloud_kwargs) + libcloud_kwargs = salt.utils.args.clean_kwargs(**libcloud_kwargs) volume = _get_by_id(conn.list_volumes(), volume_id) snapshots = conn.list_volume_snapshots(volume, **libcloud_kwargs) @@ -309,7 +309,7 @@ def create_volume(size, name, profile, location_id=None, **libcloud_kwargs): salt myminion libcloud_compute.create_volume 1000 vol1 profile1 ''' conn = _get_driver(profile=profile) - libcloud_kwargs = clean_kwargs(**libcloud_kwargs) + libcloud_kwargs = salt.utils.args.clean_kwargs(**libcloud_kwargs) if location_id is not None: location = _get_by_id(conn.list_locations(), location_id) else: @@ -344,7 +344,7 @@ def create_volume_snapshot(volume_id, profile, name=None, **libcloud_kwargs): salt myminion libcloud_compute.create_volume_snapshot vol1 profile1 ''' conn = _get_driver(profile=profile) - libcloud_kwargs = clean_kwargs(**libcloud_kwargs) + libcloud_kwargs = salt.utils.args.clean_kwargs(**libcloud_kwargs) volume = _get_by_id(conn.list_volumes(), volume_id) snapshot = conn.create_volume_snapshot(volume, name=name, **libcloud_kwargs) @@ -377,7 +377,7 @@ def attach_volume(node_id, volume_id, profile, device=None, **libcloud_kwargs): salt myminion libcloud_compute.detach_volume vol1 profile1 ''' conn = _get_driver(profile=profile) - libcloud_kwargs = clean_kwargs(**libcloud_kwargs) + libcloud_kwargs = salt.utils.args.clean_kwargs(**libcloud_kwargs) volume = _get_by_id(conn.list_volumes(), volume_id) node = _get_by_id(conn.list_nodes(), node_id) return conn.attach_volume(node, volume, device=device, **libcloud_kwargs) @@ -403,7 +403,7 @@ def detach_volume(volume_id, profile, **libcloud_kwargs): salt myminion libcloud_compute.detach_volume vol1 profile1 ''' conn = _get_driver(profile=profile) - libcloud_kwargs = clean_kwargs(**libcloud_kwargs) + libcloud_kwargs = salt.utils.args.clean_kwargs(**libcloud_kwargs) volume = _get_by_id(conn.list_volumes(), volume_id) return conn.detach_volume(volume, **libcloud_kwargs) @@ -428,7 +428,7 @@ def destroy_volume(volume_id, profile, **libcloud_kwargs): salt myminion libcloud_compute.destroy_volume vol1 profile1 ''' conn = _get_driver(profile=profile) - libcloud_kwargs = clean_kwargs(**libcloud_kwargs) + libcloud_kwargs = salt.utils.args.clean_kwargs(**libcloud_kwargs) volume = _get_by_id(conn.list_volumes(), volume_id) return conn.destroy_volume(volume, **libcloud_kwargs) @@ -456,7 +456,7 @@ def destroy_volume_snapshot(volume_id, snapshot_id, profile, **libcloud_kwargs): salt myminion libcloud_compute.destroy_volume_snapshot snap1 profile1 ''' conn = _get_driver(profile=profile) - libcloud_kwargs = clean_kwargs(**libcloud_kwargs) + libcloud_kwargs = salt.utils.args.clean_kwargs(**libcloud_kwargs) volume = _get_by_id(conn.list_volumes(), volume_id) snapshot = _get_by_id(conn.list_volume_snapshots(volume), snapshot_id) return conn.destroy_volume_snapshot(snapshot, **libcloud_kwargs) @@ -482,7 +482,7 @@ def list_images(profile, location_id=None, **libcloud_kwargs): salt myminion libcloud_compute.list_images profile1 ''' conn = _get_driver(profile=profile) - libcloud_kwargs = clean_kwargs(**libcloud_kwargs) + libcloud_kwargs = salt.utils.args.clean_kwargs(**libcloud_kwargs) if location_id is not None: location = _get_by_id(conn.list_locations(), location_id) else: @@ -522,7 +522,7 @@ def create_image(node_id, name, profile, description=None, **libcloud_kwargs): salt myminion libcloud_compute.create_image server1 my_image profile1 description='test image' ''' conn = _get_driver(profile=profile) - libcloud_kwargs = clean_kwargs(**libcloud_kwargs) + libcloud_kwargs = salt.utils.args.clean_kwargs(**libcloud_kwargs) node = _get_by_id(conn.list_nodes(), node_id) return _simple_image(conn.create_image(node, name, description=description, **libcloud_kwargs)) @@ -547,7 +547,7 @@ def delete_image(image_id, profile, **libcloud_kwargs): salt myminion libcloud_compute.delete_image image1 profile1 ''' conn = _get_driver(profile=profile) - libcloud_kwargs = clean_kwargs(**libcloud_kwargs) + libcloud_kwargs = salt.utils.args.clean_kwargs(**libcloud_kwargs) image = _get_by_id(conn.list_images(), image_id) return conn.delete_image(image, **libcloud_kwargs) @@ -572,7 +572,7 @@ def get_image(image_id, profile, **libcloud_kwargs): salt myminion libcloud_compute.get_image image1 profile1 ''' conn = _get_driver(profile=profile) - libcloud_kwargs = clean_kwargs(**libcloud_kwargs) + libcloud_kwargs = salt.utils.args.clean_kwargs(**libcloud_kwargs) image = conn.get_image(image_id, **libcloud_kwargs) return _simple_image(image) @@ -606,7 +606,7 @@ def copy_image(source_region, image_id, name, profile, description=None, **libcl salt myminion libcloud_compute.copy_image us-east1 image1 'new image' profile1 ''' conn = _get_driver(profile=profile) - libcloud_kwargs = clean_kwargs(**libcloud_kwargs) + libcloud_kwargs = salt.utils.args.clean_kwargs(**libcloud_kwargs) image = conn.get_image(image_id, **libcloud_kwargs) new_image = conn.copy_image(source_region, image, name, description=description, **libcloud_kwargs) @@ -630,7 +630,7 @@ def list_key_pairs(profile, **libcloud_kwargs): salt myminion libcloud_compute.list_key_pairs profile1 ''' conn = _get_driver(profile=profile) - libcloud_kwargs = clean_kwargs(**libcloud_kwargs) + libcloud_kwargs = salt.utils.args.clean_kwargs(**libcloud_kwargs) keys = conn.list_key_pairs(**libcloud_kwargs) ret = [] @@ -659,7 +659,7 @@ def get_key_pair(name, profile, **libcloud_kwargs): salt myminion libcloud_compute.get_key_pair pair1 profile1 ''' conn = _get_driver(profile=profile) - libcloud_kwargs = clean_kwargs(**libcloud_kwargs) + libcloud_kwargs = salt.utils.args.clean_kwargs(**libcloud_kwargs) return _simple_key_pair(conn.get_key_pair(name, **libcloud_kwargs)) @@ -683,7 +683,7 @@ def create_key_pair(name, profile, **libcloud_kwargs): salt myminion libcloud_compute.create_key_pair pair1 profile1 ''' conn = _get_driver(profile=profile) - libcloud_kwargs = clean_kwargs(**libcloud_kwargs) + libcloud_kwargs = salt.utils.args.clean_kwargs(**libcloud_kwargs) return _simple_key_pair(conn.create_key_pair(name, **libcloud_kwargs)) @@ -715,7 +715,7 @@ def import_key_pair(name, key, profile, key_type=None, **libcloud_kwargs): salt myminion libcloud_compute.import_key_pair pair1 /path/to/key profile1 ''' conn = _get_driver(profile=profile) - libcloud_kwargs = clean_kwargs(**libcloud_kwargs) + libcloud_kwargs = salt.utils.args.clean_kwargs(**libcloud_kwargs) if os.path.exists(key) or key_type == 'FILE': return _simple_key_pair(conn.import_key_pair_from_file(name, key, @@ -746,7 +746,7 @@ def delete_key_pair(name, profile, **libcloud_kwargs): salt myminion libcloud_compute.delete_key_pair pair1 profile1 ''' conn = _get_driver(profile=profile) - libcloud_kwargs = clean_kwargs(**libcloud_kwargs) + libcloud_kwargs = salt.utils.args.clean_kwargs(**libcloud_kwargs) key = conn.get_key_pair(name) return conn.delete_key_pair(key, **libcloud_kwargs) @@ -770,7 +770,7 @@ def extra(method, profile, **libcloud_kwargs): salt myminion libcloud_compute.extra ex_get_permissions google container_name=my_container object_name=me.jpg --out=yaml ''' - libcloud_kwargs = clean_kwargs(**libcloud_kwargs) + libcloud_kwargs = salt.utils.args.clean_kwargs(**libcloud_kwargs) conn = _get_driver(profile=profile) connection_method = getattr(conn, method) return connection_method(**libcloud_kwargs) diff --git a/salt/modules/libcloud_loadbalancer.py b/salt/modules/libcloud_loadbalancer.py index 239ec056024..a181d137381 100644 --- a/salt/modules/libcloud_loadbalancer.py +++ b/salt/modules/libcloud_loadbalancer.py @@ -36,9 +36,9 @@ from __future__ import absolute_import import logging # Import salt libs +import salt.utils.args import salt.utils.compat -import salt.ext.six as six -from salt.utils import clean_kwargs +from salt.ext import six from salt.utils.versions import LooseVersion as _LooseVersion log = logging.getLogger(__name__) @@ -118,7 +118,7 @@ def list_balancers(profile, **libcloud_kwargs): salt myminion libcloud_storage.list_balancers profile1 ''' conn = _get_driver(profile=profile) - libcloud_kwargs = clean_kwargs(**libcloud_kwargs) + libcloud_kwargs = salt.utils.args.clean_kwargs(**libcloud_kwargs) balancers = conn.list_balancers(**libcloud_kwargs) ret = [] for balancer in balancers: @@ -146,7 +146,7 @@ def list_protocols(profile, **libcloud_kwargs): salt myminion libcloud_storage.list_protocols profile1 ''' conn = _get_driver(profile=profile) - libcloud_kwargs = clean_kwargs(**libcloud_kwargs) + libcloud_kwargs = salt.utils.args.clean_kwargs(**libcloud_kwargs) return conn.list_protocols(**libcloud_kwargs) @@ -194,7 +194,7 @@ def create_balancer(name, port, protocol, profile, algorithm=None, members=None, else: raise ValueError("members must be of type list") - libcloud_kwargs = clean_kwargs(**libcloud_kwargs) + libcloud_kwargs = salt.utils.args.clean_kwargs(**libcloud_kwargs) conn = _get_driver(profile=profile) balancer = conn.create_balancer(name, port, protocol, algorithm, starting_members, **libcloud_kwargs) return _simple_balancer(balancer) @@ -223,7 +223,7 @@ def destroy_balancer(balancer_id, profile, **libcloud_kwargs): salt myminion libcloud_storage.destroy_balancer balancer_1 profile1 ''' conn = _get_driver(profile=profile) - libcloud_kwargs = clean_kwargs(**libcloud_kwargs) + libcloud_kwargs = salt.utils.args.clean_kwargs(**libcloud_kwargs) balancer = conn.get_balancer(balancer_id) return conn.destroy_balancer(balancer, **libcloud_kwargs) @@ -250,7 +250,7 @@ def get_balancer_by_name(name, profile, **libcloud_kwargs): salt myminion libcloud_storage.get_balancer_by_name my_balancer profile1 ''' conn = _get_driver(profile=profile) - libcloud_kwargs = clean_kwargs(**libcloud_kwargs) + libcloud_kwargs = salt.utils.args.clean_kwargs(**libcloud_kwargs) balancers = conn.list_balancers(**libcloud_kwargs) match = [b for b in balancers if b.name == name] if len(match) == 1: @@ -283,7 +283,7 @@ def get_balancer(balancer_id, profile, **libcloud_kwargs): salt myminion libcloud_storage.get_balancer balancer123 profile1 ''' conn = _get_driver(profile=profile) - libcloud_kwargs = clean_kwargs(**libcloud_kwargs) + libcloud_kwargs = salt.utils.args.clean_kwargs(**libcloud_kwargs) balancer = conn.get_balancer(balancer_id, **libcloud_kwargs) return _simple_balancer(balancer) @@ -307,7 +307,7 @@ def list_supported_algorithms(profile, **libcloud_kwargs): salt myminion libcloud_storage.list_supported_algorithms profile1 ''' conn = _get_driver(profile=profile) - libcloud_kwargs = clean_kwargs(**libcloud_kwargs) + libcloud_kwargs = salt.utils.args.clean_kwargs(**libcloud_kwargs) return conn.list_supported_algorithms(**libcloud_kwargs) @@ -337,7 +337,7 @@ def balancer_attach_member(balancer_id, ip, port, profile, extra=None, **libclou salt myminion libcloud_storage.balancer_attach_member balancer123 1.2.3.4 80 profile1 ''' conn = _get_driver(profile=profile) - libcloud_kwargs = clean_kwargs(**libcloud_kwargs) + libcloud_kwargs = salt.utils.args.clean_kwargs(**libcloud_kwargs) member = Member(id=None, ip=ip, port=port, balancer=None, extra=extra) balancer = conn.get_balancer(balancer_id) member_saved = conn.balancer_attach_member(balancer, member, **libcloud_kwargs) @@ -379,7 +379,7 @@ def balancer_detach_member(balancer_id, member_id, profile, **libcloud_kwargs): raise ValueError("Bad argument, found no records") else: member = match[0] - libcloud_kwargs = clean_kwargs(**libcloud_kwargs) + libcloud_kwargs = salt.utils.args.clean_kwargs(**libcloud_kwargs) return conn.balancer_detach_member(balancer=balancer, member=member, **libcloud_kwargs) @@ -404,7 +404,7 @@ def list_balancer_members(balancer_id, profile, **libcloud_kwargs): ''' conn = _get_driver(profile=profile) balancer = conn.get_balancer(balancer_id) - libcloud_kwargs = clean_kwargs(**libcloud_kwargs) + libcloud_kwargs = salt.utils.args.clean_kwargs(**libcloud_kwargs) members = conn.balancer_list_members(balancer=balancer, **libcloud_kwargs) return [_simple_member(member) for member in members] @@ -428,7 +428,7 @@ def extra(method, profile, **libcloud_kwargs): salt myminion libcloud_loadbalancer.extra ex_get_permissions google container_name=my_container object_name=me.jpg --out=yaml ''' - libcloud_kwargs = clean_kwargs(**libcloud_kwargs) + libcloud_kwargs = salt.utils.args.clean_kwargs(**libcloud_kwargs) conn = _get_driver(profile=profile) connection_method = getattr(conn, method) return connection_method(**libcloud_kwargs) diff --git a/salt/modules/libcloud_storage.py b/salt/modules/libcloud_storage.py index b7696216427..9099c269916 100644 --- a/salt/modules/libcloud_storage.py +++ b/salt/modules/libcloud_storage.py @@ -36,8 +36,8 @@ from __future__ import absolute_import import logging # Import salt libs +import salt.utils.args import salt.utils.compat -from salt.utils import clean_kwargs from salt.utils.versions import LooseVersion as _LooseVersion log = logging.getLogger(__name__) @@ -102,7 +102,7 @@ def list_containers(profile, **libcloud_kwargs): salt myminion libcloud_storage.list_containers profile1 ''' conn = _get_driver(profile=profile) - libcloud_kwargs = clean_kwargs(**libcloud_kwargs) + libcloud_kwargs = salt.utils.args.clean_kwargs(**libcloud_kwargs) containers = conn.list_containers(**libcloud_kwargs) ret = [] for container in containers: @@ -134,7 +134,7 @@ def list_container_objects(container_name, profile, **libcloud_kwargs): ''' conn = _get_driver(profile=profile) container = conn.get_container(container_name) - libcloud_kwargs = clean_kwargs(**libcloud_kwargs) + libcloud_kwargs = salt.utils.args.clean_kwargs(**libcloud_kwargs) objects = conn.list_container_objects(container, **libcloud_kwargs) ret = [] for obj in objects: @@ -169,7 +169,7 @@ def create_container(container_name, profile, **libcloud_kwargs): salt myminion libcloud_storage.create_container MyFolder profile1 ''' conn = _get_driver(profile=profile) - libcloud_kwargs = clean_kwargs(**libcloud_kwargs) + libcloud_kwargs = salt.utils.args.clean_kwargs(**libcloud_kwargs) container = conn.create_container(container_name, **libcloud_kwargs) return { 'name': container.name, @@ -197,7 +197,7 @@ def get_container(container_name, profile, **libcloud_kwargs): salt myminion libcloud_storage.get_container MyFolder profile1 ''' conn = _get_driver(profile=profile) - libcloud_kwargs = clean_kwargs(**libcloud_kwargs) + libcloud_kwargs = salt.utils.args.clean_kwargs(**libcloud_kwargs) container = conn.get_container(container_name, **libcloud_kwargs) return { 'name': container.name, @@ -228,7 +228,7 @@ def get_container_object(container_name, object_name, profile, **libcloud_kwargs salt myminion libcloud_storage.get_container_object MyFolder MyFile.xyz profile1 ''' conn = _get_driver(profile=profile) - libcloud_kwargs = clean_kwargs(**libcloud_kwargs) + libcloud_kwargs = salt.utils.args.clean_kwargs(**libcloud_kwargs) obj = conn.get_container_object(container_name, object_name, **libcloud_kwargs) return { 'name': obj.name, @@ -282,7 +282,7 @@ def download_object(container_name, object_name, destination_path, profile, ''' conn = _get_driver(profile=profile) obj = conn.get_object(container_name, object_name) - libcloud_kwargs = clean_kwargs(**libcloud_kwargs) + libcloud_kwargs = salt.utils.args.clean_kwargs(**libcloud_kwargs) return conn.download_object(obj, destination_path, overwrite_existing, delete_on_failure, **libcloud_kwargs) @@ -328,7 +328,7 @@ def upload_object(file_path, container_name, object_name, profile, extra=None, ''' conn = _get_driver(profile=profile) - libcloud_kwargs = clean_kwargs(**libcloud_kwargs) + libcloud_kwargs = salt.utils.args.clean_kwargs(**libcloud_kwargs) container = conn.get_container(container_name) obj = conn.upload_object(file_path, container, object_name, extra, verify_hash, headers, **libcloud_kwargs) return obj.name @@ -361,7 +361,7 @@ def delete_object(container_name, object_name, profile, **libcloud_kwargs): salt myminion libcloud_storage.delete_object MyFolder me.jpg profile1 ''' conn = _get_driver(profile=profile) - libcloud_kwargs = clean_kwargs(**libcloud_kwargs) + libcloud_kwargs = salt.utils.args.clean_kwargs(**libcloud_kwargs) obj = conn.get_object(container_name, object_name, **libcloud_kwargs) return conn.delete_object(obj) @@ -390,7 +390,7 @@ def delete_container(container_name, profile, **libcloud_kwargs): salt myminion libcloud_storage.delete_container MyFolder profile1 ''' conn = _get_driver(profile=profile) - libcloud_kwargs = clean_kwargs(**libcloud_kwargs) + libcloud_kwargs = salt.utils.args.clean_kwargs(**libcloud_kwargs) container = conn.get_container(container_name) return conn.delete_container(container, **libcloud_kwargs) @@ -414,7 +414,7 @@ def extra(method, profile, **libcloud_kwargs): salt myminion libcloud_storage.extra ex_get_permissions google container_name=my_container object_name=me.jpg --out=yaml ''' - libcloud_kwargs = clean_kwargs(**libcloud_kwargs) + libcloud_kwargs = salt.utils.args.clean_kwargs(**libcloud_kwargs) conn = _get_driver(profile=profile) connection_method = getattr(conn, method) return connection_method(**libcloud_kwargs) diff --git a/salt/modules/linux_acl.py b/salt/modules/linux_acl.py index a7fa3cbd1cb..d9959b7cc11 100644 --- a/salt/modules/linux_acl.py +++ b/salt/modules/linux_acl.py @@ -5,7 +5,7 @@ Support for Linux File Access Control Lists from __future__ import absolute_import # Import salt libs -import salt.utils +import salt.utils.path from salt.exceptions import CommandExecutionError # Define the module's virtual name @@ -16,7 +16,7 @@ def __virtual__(): ''' Only load the module if getfacl is installed ''' - if salt.utils.which('getfacl'): + if salt.utils.path.which('getfacl'): return __virtualname__ return (False, 'The linux_acl execution module cannot be loaded: the getfacl binary is not in the path.') @@ -213,8 +213,10 @@ def modfacl(acl_type, acl_name='', perms='', *args, **kwargs): salt '*' acl.modfacl d:u myuser 7 /tmp/house/kitchen salt '*' acl.modfacl g mygroup 0 /tmp/house/kitchen /tmp/house/livingroom salt '*' acl.modfacl user myuser rwx /tmp/house/kitchen recursive=True + salt '*' acl.modfacl user myuser rwx /tmp/house/kitchen raise_err=True ''' recursive = kwargs.pop('recursive', False) + raise_err = kwargs.pop('raise_err', False) _raise_on_no_files(*args) @@ -228,7 +230,7 @@ def modfacl(acl_type, acl_name='', perms='', *args, **kwargs): for dentry in args: cmd += ' "{0}"'.format(dentry) - __salt__['cmd.run'](cmd, python_shell=False) + __salt__['cmd.run'](cmd, python_shell=False, raise_err=raise_err) return True diff --git a/salt/modules/linux_ip.py b/salt/modules/linux_ip.py index a94f373b47d..d848898f25a 100644 --- a/salt/modules/linux_ip.py +++ b/salt/modules/linux_ip.py @@ -2,8 +2,14 @@ ''' The networking module for Non-RH/Deb Linux distros ''' +# Import Python libs from __future__ import absolute_import -import salt.utils + +# Import Salt libs +import salt.utils.files +import salt.utils.path +import salt.utils.platform + from salt.ext.six.moves import zip __virtualname__ = 'ip' @@ -13,7 +19,7 @@ def __virtual__(): ''' Confine this module to Non-RH/Deb Linux distros ''' - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): return (False, 'Module linux_ip: Windows systems are not supported.') if __grains__['os_family'] == 'RedHat': return (False, 'Module linux_ip: RedHat systems are not supported.') @@ -21,7 +27,7 @@ def __virtual__(): return (False, 'Module linux_ip: Debian systems are not supported.') if __grains__['os_family'] == 'NILinuxRT': return (False, 'Module linux_ip: NILinuxRT systems are not supported.') - if not salt.utils.which('ip'): + if not salt.utils.path.which('ip'): return (False, 'The linux_ip execution module cannot be loaded: ' 'the ip binary is not in the path.') return __virtualname__ @@ -133,7 +139,7 @@ def _parse_routes(): ''' Parse the contents of ``/proc/net/route`` ''' - with salt.utils.fopen('/proc/net/route', 'r') as fp_: + with salt.utils.files.fopen('/proc/net/route', 'r') as fp_: out = fp_.read() ret = {} diff --git a/salt/modules/linux_lvm.py b/salt/modules/linux_lvm.py index be77285e252..1001879b342 100644 --- a/salt/modules/linux_lvm.py +++ b/salt/modules/linux_lvm.py @@ -8,8 +8,8 @@ from __future__ import absolute_import import os.path # Import salt libs -import salt.utils -import salt.ext.six as six +import salt.utils.path +from salt.ext import six from salt.exceptions import CommandExecutionError # Define the module's virtual name @@ -20,7 +20,7 @@ def __virtual__(): ''' Only load the module if lvm is installed ''' - if salt.utils.which('lvm'): + if salt.utils.path.which('lvm'): return __virtualname__ return (False, 'The linux_lvm execution module cannot be loaded: the lvm binary is not in the path.') @@ -159,7 +159,7 @@ def vgdisplay(vgname=''): return ret -def lvdisplay(lvname=''): +def lvdisplay(lvname='', quiet=False): ''' Return information about the logical volume(s) @@ -174,7 +174,10 @@ def lvdisplay(lvname=''): cmd = ['lvdisplay', '-c'] if lvname: cmd.append(lvname) - cmd_ret = __salt__['cmd.run_all'](cmd, python_shell=False) + if quiet: + cmd_ret = __salt__['cmd.run_all'](cmd, python_shell=False, output_loglevel='quiet') + else: + cmd_ret = __salt__['cmd.run_all'](cmd, python_shell=False) if cmd_ret['retcode'] != 0: return {} @@ -399,7 +402,7 @@ def lvcreate(lvname, elif k in valid: extra_arguments.extend(['--{0}'.format(k), '{0}'.format(v)]) - cmd = [salt.utils.which('lvcreate')] + cmd = [salt.utils.path.which('lvcreate')] if thinvolume: cmd.extend(['--thin', '-n', lvname]) @@ -430,7 +433,7 @@ def lvcreate(lvname, cmd.extend(extra_arguments) if force: - cmd.append('-yes') + cmd.append('--yes') out = __salt__['cmd.run'](cmd, python_shell=False).splitlines() lvdev = '/dev/{0}/{1}'.format(vgname, lvname) diff --git a/salt/modules/linux_sysctl.py b/salt/modules/linux_sysctl.py index 555cb862fce..11ef3591253 100644 --- a/salt/modules/linux_sysctl.py +++ b/salt/modules/linux_sysctl.py @@ -10,8 +10,8 @@ import os import re # Import salt libs -import salt.ext.six as six -import salt.utils +from salt.ext import six +import salt.utils.files from salt.ext.six import string_types from salt.exceptions import CommandExecutionError import salt.utils.systemd @@ -70,7 +70,7 @@ def show(config_file=False): ret = {} if config_file: try: - with salt.utils.fopen(config_file) as fp_: + with salt.utils.files.fopen(config_file) as fp_: for line in fp_: if not line.startswith('#') and '=' in line: # search if we have some '=' instead of ' = ' separators @@ -169,7 +169,7 @@ def persist(name, value, config=None): if not os.path.exists(sysctl_dir): os.makedirs(sysctl_dir) try: - with salt.utils.fopen(config, 'w+') as _fh: + with salt.utils.files.fopen(config, 'w+') as _fh: _fh.write('#\n# Kernel sysctl configuration\n#\n') except (IOError, OSError): msg = 'Could not write to file: {0}' @@ -178,7 +178,7 @@ def persist(name, value, config=None): # Read the existing sysctl.conf nlines = [] try: - with salt.utils.fopen(config, 'r') as _fh: + with salt.utils.files.fopen(config, 'r') as _fh: # Use readlines because this should be a small file # and it seems unnecessary to indent the below for # loop since it is a fairly large block of code. @@ -230,7 +230,7 @@ def persist(name, value, config=None): if not edited: nlines.append('{0} = {1}\n'.format(name, value)) try: - with salt.utils.fopen(config, 'w+') as _fh: + with salt.utils.files.fopen(config, 'w+') as _fh: _fh.writelines(nlines) except (IOError, OSError): msg = 'Could not write to file: {0}' diff --git a/salt/modules/localemod.py b/salt/modules/localemod.py index 3a9d0ec36a6..e998df1b834 100644 --- a/salt/modules/localemod.py +++ b/salt/modules/localemod.py @@ -15,11 +15,12 @@ try: except ImportError: pass -# Import salt libs -import salt.utils +# Import Salt libs import salt.utils.locales +import salt.utils.path +import salt.utils.platform import salt.utils.systemd -import salt.ext.six as six +from salt.ext import six from salt.exceptions import CommandExecutionError log = logging.getLogger(__name__) @@ -32,7 +33,7 @@ def __virtual__(): ''' Only work on POSIX-like systems ''' - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): return (False, 'Cannot load locale module: windows platforms are unsupported') return __virtualname__ @@ -185,7 +186,7 @@ def set_locale(locale): ) elif 'Debian' in __grains__['os_family']: # this block only applies to Debian without systemd - update_locale = salt.utils.which('update-locale') + update_locale = salt.utils.path.which('update-locale') if update_locale is None: raise CommandExecutionError( 'Cannot set locale: "update-locale" was not found.') @@ -318,7 +319,7 @@ def gen_locale(locale, **kwargs): append_if_not_found=True ) - if salt.utils.which('locale-gen'): + if salt.utils.path.which('locale-gen'): cmd = ['locale-gen'] if on_gentoo: cmd.append('--generate') @@ -326,7 +327,7 @@ def gen_locale(locale, **kwargs): cmd.append(salt.utils.locales.normalize_locale(locale)) else: cmd.append(locale) - elif salt.utils.which('localedef'): + elif salt.utils.path.which('localedef'): cmd = ['localedef', '--force', '-i', locale_search_str, '-f', locale_info['codeset'], '{0}.{1}'.format(locale_search_str, locale_info['codeset']), diff --git a/salt/modules/locate.py b/salt/modules/locate.py index 7dcfc08ee28..28474eb3442 100644 --- a/salt/modules/locate.py +++ b/salt/modules/locate.py @@ -8,7 +8,7 @@ from __future__ import absolute_import import logging # Import salt libs -import salt.utils +import salt.utils.platform log = logging.getLogger(__name__) @@ -17,8 +17,12 @@ def __virtual__(): ''' Only work on POSIX-like systems ''' - if salt.utils.is_windows(): - return (False, 'The locate execution module cannot be loaded: only available on non-Windows systems.') + if salt.utils.platform.is_windows(): + return ( + False, + 'The locate execution module cannot be loaded: only available on ' + 'non-Windows systems.' + ) return True diff --git a/salt/modules/logadm.py b/salt/modules/logadm.py index eaf72b72c48..9d1d4988181 100644 --- a/salt/modules/logadm.py +++ b/salt/modules/logadm.py @@ -14,7 +14,9 @@ except ImportError: # Import salt libs import salt.utils +import salt.utils.args import salt.utils.decorators as decorators +import salt.utils.files log = logging.getLogger(__name__) default_conf = '/etc/logadm.conf' @@ -69,7 +71,7 @@ def _parse_conf(conf_file=default_conf): Parse a logadm configuration file. ''' ret = {} - with salt.utils.fopen(conf_file, 'r') as ifile: + with salt.utils.files.fopen(conf_file, 'r') as ifile: for line in ifile: line = line.strip() if not line: @@ -153,8 +155,6 @@ def show_conf(conf_file=default_conf, name=None): ''' Show configuration - .. versionchanged:: Nitrogen - conf_file : string path to logadm.conf, defaults to /etc/logadm.conf name : string @@ -182,7 +182,7 @@ def list_conf(conf_file=default_conf, log_file=None, include_unset=False): ''' Show parsed configuration - .. versionadded:: Nitrogen + .. versionadded:: Oxygen conf_file : string path to logadm.conf, defaults to /etc/logadm.conf @@ -221,7 +221,7 @@ def show_args(): ''' Show which arguments map to which flags and options. - .. versionadded:: Nitrogen + .. versionadded:: Oxygen CLI Example: @@ -242,8 +242,6 @@ def rotate(name, pattern=None, conf_file=default_conf, **kwargs): ''' Set up pattern for logging. - .. versionchanged:: Nitrogen - name : string alias for entryname pattern : string @@ -271,7 +269,7 @@ def rotate(name, pattern=None, conf_file=default_conf, **kwargs): ''' ## cleanup kwargs - kwargs = salt.utils.clean_kwargs(**kwargs) + kwargs = salt.utils.args.clean_kwargs(**kwargs) ## inject name into kwargs if 'entryname' not in kwargs and name and not name.startswith('/'): diff --git a/salt/modules/logrotate.py b/salt/modules/logrotate.py index 4d2ffd0bc7d..f0e71dc25b2 100644 --- a/salt/modules/logrotate.py +++ b/salt/modules/logrotate.py @@ -9,8 +9,9 @@ import os import logging # Import salt libs +import salt.utils.files +import salt.utils.platform from salt.exceptions import SaltInvocationError -import salt.utils _LOG = logging.getLogger(__name__) _DEFAULT_CONF = '/etc/logrotate.conf' @@ -26,8 +27,12 @@ def __virtual__(): ''' Only work on POSIX-like systems ''' - if salt.utils.is_windows(): - return (False, 'The logrotate execution module cannot be loaded: only available on non-Windows systems.') + if salt.utils.platform.is_windows(): + return ( + False, + 'The logrotate execution module cannot be loaded: only available ' + 'on non-Windows systems.' + ) return True @@ -62,7 +67,7 @@ def _parse_conf(conf_file=_DEFAULT_CONF): multi = {} prev_comps = None - with salt.utils.fopen(conf_file, 'r') as ifile: + with salt.utils.files.fopen(conf_file, 'r') as ifile: for line in ifile: line = line.strip() if not line: diff --git a/salt/modules/lvs.py b/salt/modules/lvs.py index 11ad739627c..001280236ce 100644 --- a/salt/modules/lvs.py +++ b/salt/modules/lvs.py @@ -7,7 +7,7 @@ from __future__ import absolute_import # Import python libs # Import salt libs -import salt.utils +import salt.utils.path import salt.utils.decorators as decorators from salt.exceptions import SaltException @@ -20,7 +20,7 @@ __func_alias__ = { # Cache the output of running which('ipvsadm') @decorators.memoize def __detect_os(): - return salt.utils.which('ipvsadm') + return salt.utils.path.which('ipvsadm') def __virtual__(): diff --git a/salt/modules/lxc.py b/salt/modules/lxc.py index 0e369e5cc61..3a9b4bc9a20 100644 --- a/salt/modules/lxc.py +++ b/salt/modules/lxc.py @@ -25,18 +25,20 @@ import re import random # Import salt libs -import salt -import salt.utils.odict import salt.utils -import salt.utils.dictupdate -import salt.utils.network -from salt.exceptions import CommandExecutionError, SaltInvocationError +import salt.utils.args import salt.utils.cloud +import salt.utils.dictupdate +import salt.utils.files +import salt.utils.network +import salt.utils.odict +import salt.utils.path +from salt.exceptions import CommandExecutionError, SaltInvocationError import salt.config from salt.utils.versions import LooseVersion as _LooseVersion # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six # pylint: disable=import-error,no-name-in-module from salt.ext.six.moves import range # pylint: disable=redefined-builtin from salt.ext.six.moves.urllib.parse import urlparse as _urlparse @@ -61,7 +63,7 @@ _marker = object() def __virtual__(): - if salt.utils.which('lxc-start'): + if salt.utils.path.which('lxc-start'): return __virtualname__ # To speed up the whole thing, we decided to not use the # subshell way and assume things are in place for lxc @@ -70,7 +72,7 @@ def __virtual__(): # lxc-version presence is not sufficient, in lxc1.0 alpha # (precise backports), we have it and it is sufficient # for the module to execute. - # elif salt.utils.which('lxc-version'): + # elif salt.utils.path.which('lxc-version'): # passed = False # try: # passed = subprocess.check_output( @@ -288,10 +290,8 @@ def cloud_init_interface(name, vm_=None, **kwargs): any extra argument for the salt minion config dnsservers list of DNS servers to set inside the container - (Defaults to 8.8.8.8 and 4.4.4.4 until Oxygen release) dns_via_dhcp do not set the dns servers, let them be set by the dhcp. - (Defaults to False until Oxygen release) autostart autostart the container at boot time password @@ -404,13 +404,7 @@ def cloud_init_interface(name, vm_=None, **kwargs): snapshot = _cloud_get('snapshot', False) autostart = bool(_cloud_get('autostart', True)) dnsservers = _cloud_get('dnsservers', []) - dns_via_dhcp = _cloud_get('dns_via_dhcp', False) - if not dnsservers and not dns_via_dhcp: - salt.utils.warn_until('Oxygen', ( - 'dnsservers will stop defaulting to 8.8.8.8 and 4.4.4.4 ' - 'in Oxygen. Please set your dnsservers.' - )) - dnsservers = ['8.8.8.8', '4.4.4.4'] + dns_via_dhcp = _cloud_get('dns_via_dhcp', True) password = _cloud_get('password', 's3cr3t') password_encrypted = _cloud_get('password_encrypted', False) fstype = _cloud_get('fstype', None) @@ -577,7 +571,7 @@ def _get_profile(key, name, **kwargs): raise CommandExecutionError('lxc.{0} must be a dictionary'.format(key)) # Overlay the kwargs to override matched profile data - overrides = salt.utils.clean_kwargs(**copy.deepcopy(kwargs)) + overrides = salt.utils.args.clean_kwargs(**copy.deepcopy(kwargs)) profile_match = salt.utils.dictupdate.update( copy.deepcopy(profile_match), overrides @@ -901,7 +895,8 @@ def _get_lxc_default_data(**kwargs): ret['lxc.start.auto'] = '0' memory = kwargs.get('memory') if memory is not None: - ret['lxc.cgroup.memory.limit_in_bytes'] = memory * 1024 + # converting the config value from MB to bytes + ret['lxc.cgroup.memory.limit_in_bytes'] = memory * 1024 * 1024 cpuset = kwargs.get('cpuset') if cpuset: ret['lxc.cgroup.cpuset.cpus'] = cpuset @@ -1002,7 +997,7 @@ class _LXCConfig(object): if self.name: self.path = os.path.join(path, self.name, 'config') if os.path.isfile(self.path): - with salt.utils.fopen(self.path) as fhr: + with salt.utils.files.fopen(self.path) as fhr: for line in fhr.readlines(): match = self.pattern.findall((line.strip())) if match: @@ -1042,7 +1037,7 @@ class _LXCConfig(object): content = self.as_string() # 2 step rendering to be sure not to open/wipe the config # before as_string succeeds. - with salt.utils.fopen(self.path, 'w') as fic: + with salt.utils.files.fopen(self.path, 'w') as fic: fic.write(content) fic.flush() @@ -2768,7 +2763,7 @@ def info(name, path=None): ret = {} config = [] - with salt.utils.fopen(conf_file) as fp_: + with salt.utils.files.fopen(conf_file) as fp_: for line in fp_: comps = [x.strip() for x in line.split('#', 1)[0].strip().split('=', 1)] @@ -2978,8 +2973,8 @@ def update_lxc_conf(name, lxc_conf, lxc_conf_unset, path=None): changes = {'edited': [], 'added': [], 'removed': []} ret = {'changes': changes, 'result': True, 'comment': ''} - # do not use salt.utils.fopen ! - with salt.utils.fopen(lxc_conf_p, 'r') as fic: + # do not use salt.utils.files.fopen ! + with salt.utils.files.fopen(lxc_conf_p, 'r') as fic: filtered_lxc_conf = [] for row in lxc_conf: if not row: @@ -3036,11 +3031,11 @@ def update_lxc_conf(name, lxc_conf, lxc_conf_unset, path=None): conf_changed = conf != orig_config chrono = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S') if conf_changed: - # DO NOT USE salt.utils.fopen here, i got (kiorky) + # DO NOT USE salt.utils.files.fopen here, i got (kiorky) # problems with lxc configs which were wiped ! - with salt.utils.fopen('{0}.{1}'.format(lxc_conf_p, chrono), 'w') as wfic: + with salt.utils.files.fopen('{0}.{1}'.format(lxc_conf_p, chrono), 'w') as wfic: wfic.write(conf) - with salt.utils.fopen(lxc_conf_p, 'w') as wfic: + with salt.utils.files.fopen(lxc_conf_p, 'w') as wfic: wfic.write(conf) ret['comment'] = 'Updated' ret['result'] = True @@ -3105,7 +3100,7 @@ def set_dns(name, dnsservers=None, searchdomains=None, path=None): # operation. # - We also teach resolvconf to use the aforementioned dns. # - We finally also set /etc/resolv.conf in all cases - rstr = __salt__['test.rand_str']() + rstr = __salt__['test.random_hash']() # no tmp here, apparmor won't let us execute ! script = '/sbin/{0}_dns.sh'.format(rstr) DNS_SCRIPT = "\n".join([ @@ -3166,7 +3161,7 @@ def running_systemd(name, cache=True, path=None): k = 'lxc.systemd.test.{0}{1}'.format(name, path) ret = __context__.get(k, None) if ret is None or not cache: - rstr = __salt__['test.rand_str']() + rstr = __salt__['test.random_hash']() # no tmp here, apparmor won't let us execute ! script = '/sbin/{0}_testsystemd.sh'.format(rstr) # ubuntu already had since trusty some bits of systemd but was @@ -3503,7 +3498,7 @@ def bootstrap(name, pub_key=pub_key, priv_key=priv_key) if needs_install or force_install or unconditional_install: if install: - rstr = __salt__['test.rand_str']() + rstr = __salt__['test.random_hash']() configdir = '/var/tmp/.c_{0}'.format(rstr) cmd = 'install -m 0700 -d {0}'.format(configdir) @@ -4232,7 +4227,7 @@ def read_conf(conf_file, out_format='simple'): ''' ret_commented = [] ret_simple = {} - with salt.utils.fopen(conf_file, 'r') as fp_: + with salt.utils.files.fopen(conf_file, 'r') as fp_: for line in fp_.readlines(): if '=' not in line: ret_commented.append(line) @@ -4316,7 +4311,7 @@ def write_conf(conf_file, conf): if out_line: content += out_line content += '\n' - with salt.utils.fopen(conf_file, 'w') as fp_: + with salt.utils.files.fopen(conf_file, 'w') as fp_: fp_.write(content) return {} @@ -4646,7 +4641,7 @@ def apply_network_profile(name, network_profile, nic_opts=None, path=None): cfgpath = os.path.join(cpath, name, 'config') before = [] - with salt.utils.fopen(cfgpath, 'r') as fp_: + with salt.utils.files.fopen(cfgpath, 'r') as fp_: for line in fp_: before.append(line) @@ -4663,7 +4658,7 @@ def apply_network_profile(name, network_profile, nic_opts=None, path=None): edit_conf(cfgpath, out_format='commented', **network_params) after = [] - with salt.utils.fopen(cfgpath, 'r') as fp_: + with salt.utils.files.fopen(cfgpath, 'r') as fp_: for line in fp_: after.append(line) diff --git a/salt/modules/mac_assistive.py b/salt/modules/mac_assistive.py index 2ca69fd1133..f894587e51c 100644 --- a/salt/modules/mac_assistive.py +++ b/salt/modules/mac_assistive.py @@ -15,7 +15,7 @@ import re import logging # Import salt libs -import salt.utils +import salt.utils.platform from salt.exceptions import CommandExecutionError from salt.utils.versions import LooseVersion as _LooseVersion @@ -28,10 +28,14 @@ def __virtual__(): ''' Only work on Mac OS ''' - if salt.utils.is_darwin() and _LooseVersion(__grains__['osrelease']) >= '10.9': + if salt.utils.platform.is_darwin() \ + and _LooseVersion(__grains__['osrelease']) >= '10.9': return True - return False, 'The assistive module cannot be loaded: must be run on ' \ - 'macOS 10.9 or newer.' + return ( + False, + 'The assistive module cannot be loaded: must be run on ' + 'macOS 10.9 or newer.' + ) def install(app_id, enable=True): diff --git a/salt/modules/mac_brew.py b/salt/modules/mac_brew.py index 541d538b928..27252ebd895 100644 --- a/salt/modules/mac_brew.py +++ b/salt/modules/mac_brew.py @@ -18,9 +18,11 @@ import logging # Import salt libs import salt.utils +import salt.utils.path import salt.utils.pkg +import salt.utils.versions from salt.exceptions import CommandExecutionError, MinionError -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import zip # Import third party libs @@ -37,7 +39,7 @@ def __virtual__(): Confine this module to Mac OS with Homebrew. ''' - if salt.utils.which('brew') and __grains__['os'] == 'MacOS': + if salt.utils.path.which('brew') and __grains__['os'] == 'MacOS': return __virtualname__ return (False, 'The brew module could not be loaded: brew not found or grain os != MacOS') @@ -127,7 +129,7 @@ def list_pkgs(versions_as_list=False, **kwargs): name_and_versions = line.split(' ') name = name_and_versions[0] installed_versions = name_and_versions[1:] - key_func = functools.cmp_to_key(salt.utils.version_cmp) + key_func = functools.cmp_to_key(salt.utils.versions.version_cmp) newest_version = sorted(installed_versions, key=key_func).pop() except ValueError: continue diff --git a/salt/modules/mac_defaults.py b/salt/modules/mac_defaults.py index cda1f1b243c..b43ae6f48c3 100644 --- a/salt/modules/mac_defaults.py +++ b/salt/modules/mac_defaults.py @@ -8,8 +8,8 @@ Set defaults on Mac OS from __future__ import absolute_import import logging -# Import salt libs -import salt.utils +# Import Salt libs +import salt.utils.platform log = logging.getLogger(__name__) __virtualname__ = 'macdefaults' @@ -19,7 +19,7 @@ def __virtual__(): ''' Only work on Mac OS ''' - if salt.utils.is_darwin(): + if salt.utils.platform.is_darwin(): return __virtualname__ return False diff --git a/salt/modules/mac_desktop.py b/salt/modules/mac_desktop.py index d9c77845c6c..8bd2f409306 100644 --- a/salt/modules/mac_desktop.py +++ b/salt/modules/mac_desktop.py @@ -4,8 +4,8 @@ macOS implementations of various commands in the "desktop" interface ''' from __future__ import absolute_import -# Import salt libs -import salt.utils +# Import Salt libs +import salt.utils.platform from salt.exceptions import CommandExecutionError # Define the module's virtual name @@ -16,7 +16,7 @@ def __virtual__(): ''' Only load on Mac systems ''' - if salt.utils.is_darwin(): + if salt.utils.platform.is_darwin(): return __virtualname__ return False, 'Cannot load macOS desktop module: This is not a macOS host.' diff --git a/salt/modules/mac_group.py b/salt/modules/mac_group.py index 9a2f02ca85f..971aa74ed12 100644 --- a/salt/modules/mac_group.py +++ b/salt/modules/mac_group.py @@ -13,6 +13,7 @@ except ImportError: # Import Salt Libs import salt.utils import salt.utils.itertools +import salt.utils.stringutils from salt.exceptions import CommandExecutionError, SaltInvocationError from salt.modules.mac_user import _dscl, _flush_dscl_cache @@ -48,7 +49,7 @@ def add(name, gid=None, **kwargs): raise CommandExecutionError( 'Group \'{0}\' already exists'.format(name) ) - if salt.utils.contains_whitespace(name): + if salt.utils.stringutils.contains_whitespace(name): raise SaltInvocationError('Group name cannot contain whitespace') if name.startswith('_'): raise SaltInvocationError( @@ -96,7 +97,7 @@ def delete(name): salt '*' group.delete foo ''' - if salt.utils.contains_whitespace(name): + if salt.utils.stringutils.contains_whitespace(name): raise SaltInvocationError('Group name cannot contain whitespace') if name.startswith('_'): raise SaltInvocationError( @@ -183,7 +184,7 @@ def info(name): salt '*' group.info foo ''' - if salt.utils.contains_whitespace(name): + if salt.utils.stringutils.contains_whitespace(name): raise SaltInvocationError('Group name cannot contain whitespace') try: # getgrnam seems to cache weirdly, so don't use it diff --git a/salt/modules/mac_keychain.py b/salt/modules/mac_keychain.py index 652e593ace5..9e3977cf4ef 100644 --- a/salt/modules/mac_keychain.py +++ b/salt/modules/mac_keychain.py @@ -18,8 +18,8 @@ try: except ImportError: HAS_DEPS = False -# Import salt libs -import salt.utils +# Import Salt libs +import salt.utils.platform log = logging.getLogger(__name__) __virtualname__ = 'keychain' @@ -36,7 +36,7 @@ def __virtual__(): ''' Only work on Mac OS ''' - if salt.utils.is_darwin() and _quote is not None: + if salt.utils.platform.is_darwin() and _quote is not None: return __virtualname__ return False diff --git a/salt/modules/mac_package.py b/salt/modules/mac_package.py index 287689187f1..a18b71a4b6c 100644 --- a/salt/modules/mac_package.py +++ b/salt/modules/mac_package.py @@ -16,8 +16,8 @@ try: except ImportError: HAS_DEPS = False -# Import salt libs -import salt.utils +# Import Salt libs +import salt.utils.platform log = logging.getLogger(__name__) __virtualname__ = 'macpackage' @@ -35,7 +35,7 @@ def __virtual__(): ''' Only work on Mac OS ''' - if salt.utils.is_darwin() and _quote is not None: + if salt.utils.platform.is_darwin() and _quote is not None: return __virtualname__ return False diff --git a/salt/modules/mac_pkgutil.py b/salt/modules/mac_pkgutil.py index 1626e73644f..01adfd4231b 100644 --- a/salt/modules/mac_pkgutil.py +++ b/salt/modules/mac_pkgutil.py @@ -12,8 +12,9 @@ import os.path # Import 3rd-party libs from salt.ext.six.moves import urllib # pylint: disable=import-error -# Import salt libs -import salt.utils +# Import Salt libs +import salt.utils.path +import salt.utils.platform import salt.utils.itertools import salt.utils.mac_utils from salt.exceptions import SaltInvocationError @@ -26,10 +27,10 @@ __virtualname__ = 'pkgutil' def __virtual__(): - if not salt.utils.is_darwin(): + if not salt.utils.platform.is_darwin(): return (False, 'Only available on Mac OS systems') - if not salt.utils.which('pkgutil'): + if not salt.utils.path.which('pkgutil'): return (False, 'Missing pkgutil binary') return __virtualname__ diff --git a/salt/modules/mac_ports.py b/salt/modules/mac_ports.py index 8a842cdb8eb..4d720e5a031 100644 --- a/salt/modules/mac_ports.py +++ b/salt/modules/mac_ports.py @@ -38,12 +38,15 @@ import re # Import salt libs import salt.utils +import salt.utils.path import salt.utils.pkg +import salt.utils.platform import salt.utils.mac_utils +import salt.utils.versions from salt.exceptions import CommandExecutionError # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six log = logging.getLogger(__name__) @@ -55,10 +58,10 @@ def __virtual__(): ''' Confine this module to Mac OS with MacPorts. ''' - if not salt.utils.is_darwin(): + if not salt.utils.platform.is_darwin(): return False, 'mac_ports only available on MacOS' - if not salt.utils.which('port'): + if not salt.utils.path.which('port'): return False, 'mac_ports requires the "port" binary' return __virtualname__ @@ -168,7 +171,7 @@ def latest_version(*names, **kwargs): ret = {} for key, val in six.iteritems(available): - if key not in installed or salt.utils.compare_versions(ver1=installed[key], oper='<', ver2=val): + if key not in installed or salt.utils.versions.compare(ver1=installed[key], oper='<', ver2=val): ret[key] = val else: ret[key] = '{0} (installed)'.format(version(key)) diff --git a/salt/modules/mac_power.py b/salt/modules/mac_power.py index d7e09dce19c..705899ad325 100644 --- a/salt/modules/mac_power.py +++ b/salt/modules/mac_power.py @@ -7,10 +7,13 @@ Module for editing power settings on macOS # Import python libs from __future__ import absolute_import -# Import salt libs -import salt.utils +# Import Salt libs import salt.utils.mac_utils +import salt.utils.platform from salt.exceptions import SaltInvocationError + +# Import 3rd-party libs +from salt.ext import six from salt.ext.six.moves import range __virtualname__ = 'power' @@ -20,7 +23,7 @@ def __virtual__(): ''' Only for macOS ''' - if not salt.utils.is_darwin(): + if not salt.utils.platform.is_darwin(): return (False, 'The mac_power module could not be loaded: ' 'module only works on macOS systems.') @@ -38,7 +41,7 @@ def _validate_sleep(minutes): Returns: The value to be passed to the command ''' # Must be a value between 1 and 180 or Never/Off - if isinstance(minutes, str): + if isinstance(minutes, six.string_types): if minutes.lower() in ['never', 'off']: return 'Never' else: diff --git a/salt/modules/mac_service.py b/salt/modules/mac_service.py index f9c73b09a7c..54939102796 100644 --- a/salt/modules/mac_service.py +++ b/salt/modules/mac_service.py @@ -12,12 +12,16 @@ import plistlib # Import salt libs import salt.utils +import salt.utils.path +import salt.utils.platform +import salt.utils.stringutils import salt.utils.decorators as decorators +import salt.utils.files from salt.exceptions import CommandExecutionError from salt.utils.versions import LooseVersion as _LooseVersion # Import 3rd party libs -import salt.ext.six as six +from salt.ext import six # Define the module's virtual name __virtualname__ = 'service' @@ -31,15 +35,15 @@ def __virtual__(): ''' Only for macOS with launchctl ''' - if not salt.utils.is_darwin(): + if not salt.utils.platform.is_darwin(): return (False, 'Failed to load the mac_service module:\n' 'Only available on macOS systems.') - if not salt.utils.which('launchctl'): + if not salt.utils.path.which('launchctl'): return (False, 'Failed to load the mac_service module:\n' 'Required binary not found: "launchctl"') - if not salt.utils.which('plutil'): + if not salt.utils.path.which('plutil'): return (False, 'Failed to load the mac_service module:\n' 'Required binary not found: "plutil"') @@ -87,7 +91,7 @@ def _available_services(): try: # This assumes most of the plist files # will be already in XML format - with salt.utils.fopen(file_path): + with salt.utils.files.fopen(file_path): plist = plistlib.readPlist(true_path) except Exception: @@ -100,7 +104,7 @@ def _available_services(): plist = plistlib.readPlistFromString(plist_xml) else: plist = plistlib.readPlistFromBytes( - salt.utils.to_bytes(plist_xml)) + salt.utils.stringutils.to_bytes(plist_xml)) try: available_services[plist.Label.lower()] = { diff --git a/salt/modules/mac_shadow.py b/salt/modules/mac_shadow.py index d434f87e09f..6daaa14360a 100644 --- a/salt/modules/mac_shadow.py +++ b/salt/modules/mac_shadow.py @@ -21,9 +21,9 @@ except ImportError: HAS_PWD = False # Import salt libs -import salt.utils import logging import salt.utils.mac_utils +import salt.utils.platform from salt.exceptions import CommandExecutionError log = logging.getLogger(__name__) # Start logging @@ -33,7 +33,7 @@ __virtualname__ = 'shadow' def __virtual__(): # Is this macOS? - if not salt.utils.is_darwin(): + if not salt.utils.platform.is_darwin(): return False, 'Not macOS' if HAS_PWD: diff --git a/salt/modules/mac_softwareupdate.py b/salt/modules/mac_softwareupdate.py index 40308b8144e..0f598ed0bfb 100644 --- a/salt/modules/mac_softwareupdate.py +++ b/salt/modules/mac_softwareupdate.py @@ -11,7 +11,9 @@ import os # import salt libs import salt.utils +import salt.utils.files import salt.utils.mac_utils +import salt.utils.platform from salt.exceptions import CommandExecutionError, SaltInvocationError __virtualname__ = 'softwareupdate' @@ -21,7 +23,7 @@ def __virtual__(): ''' Only for MacOS ''' - if not salt.utils.is_darwin(): + if not salt.utils.platform.is_darwin(): return (False, 'The softwareupdate module could not be loaded: ' 'module only works on MacOS systems.') @@ -340,7 +342,7 @@ def list_downloads(): ret = [] for update in _get_available(): for f in dist_files: - with salt.utils.fopen(f) as fhr: + with salt.utils.files.fopen(f) as fhr: if update.rsplit('-', 1)[0] in fhr.read(): ret.append(update) diff --git a/salt/modules/mac_sysctl.py b/salt/modules/mac_sysctl.py index 1075dcc46d9..1b71e3cb127 100644 --- a/salt/modules/mac_sysctl.py +++ b/salt/modules/mac_sysctl.py @@ -8,7 +8,7 @@ from __future__ import absolute_import import os # Import salt libs -import salt.utils +import salt.utils.files from salt.exceptions import CommandExecutionError # Define the module's virtual name @@ -155,13 +155,13 @@ def persist(name, value, config='/etc/sysctl.conf', apply_change=False): # If the sysctl.conf is not present, add it if not os.path.isfile(config): try: - with salt.utils.fopen(config, 'w+') as _fh: + with salt.utils.files.fopen(config, 'w+') as _fh: _fh.write('#\n# Kernel sysctl configuration\n#\n') except (IOError, OSError): msg = 'Could not write to file: {0}' raise CommandExecutionError(msg.format(config)) - with salt.utils.fopen(config, 'r') as ifile: + with salt.utils.files.fopen(config, 'r') as ifile: for line in ifile: if not line.startswith('{0}='.format(name)): nlines.append(line) @@ -184,7 +184,7 @@ def persist(name, value, config='/etc/sysctl.conf', apply_change=False): if not edited: nlines.append('{0}={1}'.format(name, value)) nlines.append('\n') - with salt.utils.fopen(config, 'w+') as ofile: + with salt.utils.files.fopen(config, 'w+') as ofile: ofile.writelines(nlines) # If apply_change=True, apply edits to system if apply_change is True: diff --git a/salt/modules/mac_system.py b/salt/modules/mac_system.py index 889c6684866..9d92502573c 100644 --- a/salt/modules/mac_system.py +++ b/salt/modules/mac_system.py @@ -19,8 +19,8 @@ except ImportError: # python 2 import getpass # Import salt libs -import salt.utils import salt.utils.mac_utils +import salt.utils.platform from salt.exceptions import SaltInvocationError __virtualname__ = 'system' @@ -30,7 +30,7 @@ def __virtual__(): ''' Only for MacOS with atrun enabled ''' - if not salt.utils.is_darwin(): + if not salt.utils.platform.is_darwin(): return (False, 'The mac_system module could not be loaded: ' 'module only works on MacOS systems.') diff --git a/salt/modules/mac_timezone.py b/salt/modules/mac_timezone.py index ab5528a1206..1534a7cc4fd 100644 --- a/salt/modules/mac_timezone.py +++ b/salt/modules/mac_timezone.py @@ -9,9 +9,9 @@ from __future__ import absolute_import # Import python libs from datetime import datetime -# Import salt libs -import salt.utils +# Import Salt libs import salt.utils.mac_utils +import salt.utils.platform from salt.exceptions import SaltInvocationError __virtualname__ = 'timezone' @@ -21,7 +21,7 @@ def __virtual__(): ''' Only for macOS ''' - if not salt.utils.is_darwin(): + if not salt.utils.platform.is_darwin(): return (False, 'The mac_timezone module could not be loaded: ' 'module only works on macOS systems.') diff --git a/salt/modules/mac_user.py b/salt/modules/mac_user.py index 18194aada6d..9fe8d443bbf 100644 --- a/salt/modules/mac_user.py +++ b/salt/modules/mac_user.py @@ -24,7 +24,9 @@ from salt.ext.six import string_types # Import salt libs import salt.utils -import salt.utils.decorators as decorators +import salt.utils.args +import salt.utils.decorators.path +import salt.utils.stringutils from salt.utils.locales import sdecode as _sdecode from salt.exceptions import CommandExecutionError, SaltInvocationError @@ -95,7 +97,7 @@ def add(name, if info(name): raise CommandExecutionError('User \'{0}\' already exists'.format(name)) - if salt.utils.contains_whitespace(name): + if salt.utils.stringutils.contains_whitespace(name): raise SaltInvocationError('Username cannot contain whitespace') if uid is None: @@ -142,7 +144,7 @@ def delete(name, remove=False, force=False): salt '*' user.delete name remove=True force=True ''' - if salt.utils.contains_whitespace(name): + if salt.utils.stringutils.contains_whitespace(name): raise SaltInvocationError('Username cannot contain whitespace') if not info(name): return True @@ -271,10 +273,10 @@ def chhome(name, home, **kwargs): salt '*' user.chhome foo /Users/foo ''' - kwargs = salt.utils.clean_kwargs(**kwargs) + kwargs = salt.utils.args.clean_kwargs(**kwargs) persist = kwargs.pop('persist', False) if kwargs: - salt.utils.invalid_kwargs(kwargs) + salt.utils.args.invalid_kwargs(kwargs) if persist: log.info('Ignoring unsupported \'persist\' argument to user.chhome') @@ -358,7 +360,7 @@ def chgroups(name, groups, append=False): if isinstance(groups, string_types): groups = groups.split(',') - bad_groups = [x for x in groups if salt.utils.contains_whitespace(x)] + bad_groups = [x for x in groups if salt.utils.stringutils.contains_whitespace(x)] if bad_groups: raise SaltInvocationError( 'Invalid group name(s): {0}'.format(', '.join(bad_groups)) @@ -418,7 +420,7 @@ def _format_info(data): 'fullname': data.pw_gecos} -@decorators.which('id') +@salt.utils.decorators.path.which('id') def primary_group(name): ''' Return the primary group of the named user diff --git a/salt/modules/mac_xattr.py b/salt/modules/mac_xattr.py index c6d3e88ade5..88663c2e7eb 100644 --- a/salt/modules/mac_xattr.py +++ b/salt/modules/mac_xattr.py @@ -12,6 +12,7 @@ from __future__ import absolute_import import logging # Import salt libs +import salt.utils.args import salt.utils.mac_utils from salt.exceptions import CommandExecutionError @@ -53,10 +54,10 @@ def list_(path, **kwargs): salt '*' xattr.list /path/to/file salt '*' xattr.list /path/to/file hex=True ''' - kwargs = salt.utils.clean_kwargs(**kwargs) + kwargs = salt.utils.args.clean_kwargs(**kwargs) hex_ = kwargs.pop('hex', False) if kwargs: - salt.utils.invalid_kwargs(kwargs) + salt.utils.args.invalid_kwargs(kwargs) cmd = ['xattr', path] try: @@ -101,10 +102,10 @@ def read(path, attribute, **kwargs): salt '*' xattr.read /path/to/file com.test.attr salt '*' xattr.read /path/to/file com.test.attr hex=True ''' - kwargs = salt.utils.clean_kwargs(**kwargs) + kwargs = salt.utils.args.clean_kwargs(**kwargs) hex_ = kwargs.pop('hex', False) if kwargs: - salt.utils.invalid_kwargs(kwargs) + salt.utils.args.invalid_kwargs(kwargs) cmd = ['xattr', '-p'] if hex_: @@ -147,10 +148,10 @@ def write(path, attribute, value, **kwargs): salt '*' xattr.write /path/to/file "com.test.attr" "value" ''' - kwargs = salt.utils.clean_kwargs(**kwargs) + kwargs = salt.utils.args.clean_kwargs(**kwargs) hex_ = kwargs.pop('hex', False) if kwargs: - salt.utils.invalid_kwargs(kwargs) + salt.utils.args.invalid_kwargs(kwargs) cmd = ['xattr', '-w'] if hex_: diff --git a/salt/modules/makeconf.py b/salt/modules/makeconf.py index 0451ef67c6e..8e1b04bf229 100644 --- a/salt/modules/makeconf.py +++ b/salt/modules/makeconf.py @@ -7,7 +7,7 @@ Support for modifying make.conf under Gentoo from __future__ import absolute_import, print_function # Import Salt libs -import salt.utils +import salt.utils.files def __virtual__(): @@ -184,7 +184,7 @@ def get_var(var): ''' makeconf = _get_makeconf() # Open makeconf - with salt.utils.fopen(makeconf) as fn_: + with salt.utils.files.fopen(makeconf) as fn_: conf_file = fn_.readlines() for line in conf_file: if line.startswith(var): diff --git a/salt/modules/marathon.py b/salt/modules/marathon.py index 0d405b32d60..ad53c077472 100644 --- a/salt/modules/marathon.py +++ b/salt/modules/marathon.py @@ -10,8 +10,8 @@ from __future__ import absolute_import import json import logging -import salt.utils import salt.utils.http +import salt.utils.platform from salt.exceptions import get_error_message @@ -21,9 +21,13 @@ log = logging.getLogger(__file__) def __virtual__(): # only valid in proxy minions for now - if salt.utils.is_proxy() and 'proxy' in __opts__: + if salt.utils.platform.is_proxy() and 'proxy' in __opts__: return True - return (False, 'The marathon execution module cannot be loaded: this only works in proxy minions.') + return ( + False, + 'The marathon execution module cannot be loaded: this only works on ' + 'proxy minions.' + ) def _base_url(): diff --git a/salt/modules/match.py b/salt/modules/match.py index 20b50df14b5..4f87f313a03 100644 --- a/salt/modules/match.py +++ b/salt/modules/match.py @@ -11,7 +11,7 @@ import sys # Import salt libs import salt.minion -import salt.utils +import salt.utils.versions from salt.defaults import DEFAULT_TARGET_DELIM from salt.ext.six import string_types @@ -338,7 +338,7 @@ def filter_by(lookup, # remember to remove the expr_form argument from this function when # performing the cleanup on this deprecation. if expr_form is not None: - salt.utils.warn_until( + salt.utils.versions.warn_until( 'Fluorine', 'the target type should be passed using the \'tgt_type\' ' 'argument instead of \'expr_form\'. Support for using ' diff --git a/salt/modules/mdadm.py b/salt/modules/mdadm.py index 0b4b5115fac..0b453a26898 100644 --- a/salt/modules/mdadm.py +++ b/salt/modules/mdadm.py @@ -10,9 +10,12 @@ import logging import re # Import salt libs -import salt.utils +import salt.utils.path from salt.exceptions import CommandExecutionError, SaltInvocationError +# Import 3rd-party libs +from salt.ext import six + # Set up logger log = logging.getLogger(__name__) @@ -32,7 +35,7 @@ def __virtual__(): ''' if __grains__['kernel'] != 'Linux': return (False, 'The mdadm execution module cannot be loaded: only available on Linux.') - if not salt.utils.which('mdadm'): + if not salt.utils.path.which('mdadm'): return (False, 'The mdadm execution module cannot be loaded: the mdadm binary is not in the path.') return __virtualname__ @@ -344,7 +347,7 @@ def assemble(name, opts.append(kwargs[key]) # Devices may have been written with a blob: - if isinstance(devices, str): + if isinstance(devices, six.string_types): devices = devices.split(',') cmd = ['mdadm', '-A', name, '-v'] + opts + devices diff --git a/salt/modules/mdata.py b/salt/modules/mdata.py index a38a28653a0..cc9ff1ece67 100644 --- a/salt/modules/mdata.py +++ b/salt/modules/mdata.py @@ -14,7 +14,8 @@ from __future__ import absolute_import import logging # Import Salt libs -import salt.utils +import salt.utils.path +import salt.utils.platform import salt.utils.decorators as decorators log = logging.getLogger(__name__) @@ -36,7 +37,7 @@ def _check_mdata_list(): ''' looks to see if mdata-list is present on the system ''' - return salt.utils.which('mdata-list') + return salt.utils.path.which('mdata-list') @decorators.memoize @@ -44,7 +45,7 @@ def _check_mdata_get(): ''' looks to see if mdata-get is present on the system ''' - return salt.utils.which('mdata-get') + return salt.utils.path.which('mdata-get') @decorators.memoize @@ -52,7 +53,7 @@ def _check_mdata_put(): ''' looks to see if mdata-put is present on the system ''' - return salt.utils.which('mdata-put') + return salt.utils.path.which('mdata-put') @decorators.memoize @@ -60,14 +61,14 @@ def _check_mdata_delete(): ''' looks to see if mdata-delete is present on the system ''' - return salt.utils.which('mdata-delete') + return salt.utils.path.which('mdata-delete') def __virtual__(): ''' Provides mdata only on SmartOS ''' - if _check_mdata_list() and not salt.utils.is_smartos_globalzone(): + if _check_mdata_list() and not salt.utils.platform.is_smartos_globalzone(): return __virtualname__ return ( False, diff --git a/salt/modules/mine.py b/salt/modules/mine.py index fce8a7120c2..b9ba6271cfc 100644 --- a/salt/modules/mine.py +++ b/salt/modules/mine.py @@ -14,12 +14,14 @@ import traceback import salt.crypt import salt.payload import salt.utils -import salt.utils.network +import salt.utils.args import salt.utils.event +import salt.utils.network +import salt.utils.versions from salt.exceptions import SaltClientError # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six MINE_INTERNAL_KEYWORDS = frozenset([ '__pub_user', @@ -197,12 +199,12 @@ def send(func, *args, **kwargs): salt '*' mine.send network.ip_addrs eth0 salt '*' mine.send eth0_ip_addrs mine_function=network.ip_addrs eth0 ''' - kwargs = salt.utils.clean_kwargs(**kwargs) + kwargs = salt.utils.args.clean_kwargs(**kwargs) mine_func = kwargs.pop('mine_function', func) if mine_func not in __salt__: return False data = {} - arg_data = salt.utils.arg_lookup(__salt__[mine_func]) + arg_data = salt.utils.args.arg_lookup(__salt__[mine_func]) func_data = copy.deepcopy(kwargs) for ind, _ in enumerate(arg_data.get('args', [])): try: @@ -291,7 +293,7 @@ def get(tgt, # remember to remove the expr_form argument from this function when # performing the cleanup on this deprecation. if expr_form is not None: - salt.utils.warn_until( + salt.utils.versions.warn_until( 'Fluorine', 'the target type should be passed using the \'tgt_type\' ' 'argument instead of \'expr_form\'. Support for using ' diff --git a/salt/modules/minion.py b/salt/modules/minion.py index 6b468323921..94bf948b0bc 100644 --- a/salt/modules/minion.py +++ b/salt/modules/minion.py @@ -14,7 +14,7 @@ import salt.utils import salt.key # Import third party libs -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import range # Don't shadow built-ins. diff --git a/salt/modules/mod_random.py b/salt/modules/mod_random.py index 0482d3ba7df..a4c53ee826e 100644 --- a/salt/modules/mod_random.py +++ b/salt/modules/mod_random.py @@ -17,7 +17,7 @@ import salt.utils.pycrypto from salt.exceptions import SaltInvocationError # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six if six.PY2: ALGORITHMS_ATTR_NAME = 'algorithms' diff --git a/salt/modules/modjk.py b/salt/modules/modjk.py index 3e7c7531980..88a11007745 100644 --- a/salt/modules/modjk.py +++ b/salt/modules/modjk.py @@ -34,6 +34,7 @@ from __future__ import absolute_import # Import 3rd-party libs # pylint: disable=import-error,no-name-in-module +from salt.ext import six from salt.ext.six.moves.urllib.parse import urlencode as _urlencode from salt.ext.six.moves.urllib.request import ( HTTPBasicAuthHandler as _HTTPBasicAuthHandler, @@ -330,7 +331,7 @@ def bulk_stop(workers, lbn, profile='default'): ret = {} - if isinstance(workers, str): + if isinstance(workers, six.string_types): workers = workers.split(',') for worker in workers: @@ -359,7 +360,7 @@ def bulk_activate(workers, lbn, profile='default'): ret = {} - if isinstance(workers, str): + if isinstance(workers, six.string_types): workers = workers.split(',') for worker in workers: @@ -388,7 +389,7 @@ def bulk_disable(workers, lbn, profile='default'): ret = {} - if isinstance(workers, str): + if isinstance(workers, six.string_types): workers = workers.split(',') for worker in workers: @@ -417,7 +418,7 @@ def bulk_recover(workers, lbn, profile='default'): ret = {} - if isinstance(workers, str): + if isinstance(workers, six.string_types): workers = workers.split(',') for worker in workers: diff --git a/salt/modules/mongodb.py b/salt/modules/mongodb.py index 7aeef8180d8..83f5430b3be 100644 --- a/salt/modules/mongodb.py +++ b/salt/modules/mongodb.py @@ -26,7 +26,7 @@ from salt.exceptions import get_error_message as _get_error_message # Import third party libs -import salt.ext.six as six +from salt.ext import six try: import pymongo HAS_MONGODB = True diff --git a/salt/modules/monit.py b/salt/modules/monit.py index 4588ee299eb..cfb18cd063b 100644 --- a/salt/modules/monit.py +++ b/salt/modules/monit.py @@ -9,7 +9,7 @@ from __future__ import absolute_import import re # Import salt libs -import salt.utils +import salt.utils.path # Function alias to make sure not to shadow built-in's __func_alias__ = { @@ -19,7 +19,7 @@ __func_alias__ = { def __virtual__(): - if salt.utils.which('monit') is not None: + if salt.utils.path.which('monit') is not None: # The monit binary exists, let the module load return True return (False, 'The monit execution module cannot be loaded: the monit binary is not in the path.') diff --git a/salt/modules/moosefs.py b/salt/modules/moosefs.py index 29be8e86fb0..dcf8114cd72 100644 --- a/salt/modules/moosefs.py +++ b/salt/modules/moosefs.py @@ -5,14 +5,14 @@ Module for gathering and managing information about MooseFS from __future__ import absolute_import # Import salt libs -import salt.utils +import salt.utils.path def __virtual__(): ''' Only load if the mfs commands are installed ''' - if salt.utils.which('mfsgetgoal'): + if salt.utils.path.which('mfsgetgoal'): return 'moosefs' return (False, 'The moosefs execution module cannot be loaded: the mfsgetgoal binary is not in the path.') diff --git a/salt/modules/mount.py b/salt/modules/mount.py index 1418bc76732..9463283a4c8 100644 --- a/salt/modules/mount.py +++ b/salt/modules/mount.py @@ -10,12 +10,15 @@ import re import logging # Import salt libs -import salt.utils -from salt.utils import which as _which +import salt.utils # Can be removed once test_mode is moved +import salt.utils.files +import salt.utils.path +import salt.utils.platform +import salt.utils.mount from salt.exceptions import CommandNotFoundError, CommandExecutionError # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import filter, zip # pylint: disable=import-error,redefined-builtin # Set up logger @@ -30,7 +33,7 @@ def __virtual__(): Only load on POSIX-like systems ''' # Disable on Windows, a specific file module exists: - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): return (False, 'The mount module cannot be loaded: not a POSIX-like system.') return True @@ -58,7 +61,7 @@ def _active_mountinfo(ret): blkid_info = __salt__['disk.blkid']() - with salt.utils.fopen(filename) as ifile: + with salt.utils.files.fopen(filename) as ifile: for line in ifile: comps = line.split() device = comps[2].split(':') @@ -100,7 +103,7 @@ def _active_mounts(ret): msg = 'File not readable {0}' raise CommandExecutionError(msg.format(filename)) - with salt.utils.fopen(filename) as ifile: + with salt.utils.files.fopen(filename) as ifile: for line in ifile: comps = line.split() ret[comps[1]] = {'device': comps[0], @@ -403,7 +406,7 @@ def fstab(config='/etc/fstab'): ret = {} if not os.path.isfile(config): return ret - with salt.utils.fopen(config) as ifile: + with salt.utils.files.fopen(config) as ifile: for line in ifile: try: if __grains__['kernel'] == 'SunOS': @@ -465,7 +468,7 @@ def rm_fstab(name, device, config='/etc/fstab'): lines = [] try: - with salt.utils.fopen(config, 'r') as ifile: + with salt.utils.files.fopen(config, 'r') as ifile: for line in ifile: try: if criteria.match(line): @@ -484,7 +487,7 @@ def rm_fstab(name, device, config='/etc/fstab'): if modified: try: - with salt.utils.fopen(config, 'w+') as ofile: + with salt.utils.files.fopen(config, 'w+') as ofile: ofile.writelines(lines) except (IOError, OSError) as exc: msg = "Couldn't write to {0}: {1}" @@ -594,7 +597,7 @@ def set_fstab( raise CommandExecutionError('Bad config file "{0}"'.format(config)) try: - with salt.utils.fopen(config, 'r') as ifile: + with salt.utils.files.fopen(config, 'r') as ifile: for line in ifile: try: if criteria.match(line): @@ -624,7 +627,7 @@ def set_fstab( if ret != 'present': # ret in ['new', 'change']: if not salt.utils.test_mode(test=test, **kwargs): try: - with salt.utils.fopen(config, 'w+') as ofile: + with salt.utils.files.fopen(config, 'w+') as ofile: # The line was changed, commit it! ofile.writelines(lines) except (IOError, OSError): @@ -722,7 +725,7 @@ def set_vfstab( raise CommandExecutionError('Bad config file "{0}"'.format(config)) try: - with salt.utils.fopen(config, 'r') as ifile: + with salt.utils.files.fopen(config, 'r') as ifile: for line in ifile: try: if criteria.match(line): @@ -752,7 +755,7 @@ def set_vfstab( if ret != 'present': # ret in ['new', 'change']: if not salt.utils.test_mode(test=test, **kwargs): try: - with salt.utils.fopen(config, 'w+') as ofile: + with salt.utils.files.fopen(config, 'w+') as ofile: # The line was changed, commit it! ofile.writelines(lines) except (IOError, OSError): @@ -778,7 +781,7 @@ def rm_automaster(name, device, config='/etc/auto_salt'): # The entry is present, get rid of it lines = [] try: - with salt.utils.fopen(config, 'r') as ifile: + with salt.utils.files.fopen(config, 'r') as ifile: for line in ifile: if line.startswith('#'): # Commented @@ -811,7 +814,7 @@ def rm_automaster(name, device, config='/etc/auto_salt'): raise CommandExecutionError(msg.format(config, str(exc))) try: - with salt.utils.fopen(config, 'w+') as ofile: + with salt.utils.files.fopen(config, 'w+') as ofile: ofile.writelines(lines) except (IOError, OSError) as exc: msg = "Couldn't write to {0}: {1}" @@ -860,7 +863,7 @@ def set_automaster( device_fmt = device_fmt.replace(fstype, "") try: - with salt.utils.fopen(config, 'r') as ifile: + with salt.utils.files.fopen(config, 'r') as ifile: for line in ifile: if line.startswith('#'): # Commented @@ -907,7 +910,7 @@ def set_automaster( if change: if not salt.utils.test_mode(test=test, **kwargs): try: - with salt.utils.fopen(config, 'w+') as ofile: + with salt.utils.files.fopen(config, 'w+') as ofile: # The line was changed, commit it! ofile.writelines(lines) except (IOError, OSError): @@ -929,7 +932,7 @@ def set_automaster( ) lines.append(newline) try: - with salt.utils.fopen(config, 'w+') as ofile: + with salt.utils.files.fopen(config, 'w+') as ofile: # The line was changed, commit it! ofile.writelines(lines) except (IOError, OSError): @@ -954,7 +957,7 @@ def automaster(config='/etc/auto_salt'): ret = {} if not os.path.isfile(config): return ret - with salt.utils.fopen(config) as ifile: + with salt.utils.files.fopen(config) as ifile: for line in ifile: if line.startswith('#'): # Commented @@ -1114,12 +1117,12 @@ def is_fuse_exec(cmd): salt '*' mount.is_fuse_exec sshfs ''' - cmd_path = _which(cmd) + cmd_path = salt.utils.path.which(cmd) # No point in running ldd on a command that doesn't exist if not cmd_path: return False - elif not _which('ldd'): + elif not salt.utils.path.which('ldd'): raise CommandNotFoundError('ldd') out = __salt__['cmd.run']('ldd {0}'.format(cmd_path), python_shell=False) @@ -1149,7 +1152,7 @@ def swaps(): 'used': (int(comps[3]) - int(comps[4])), 'priority': '-'} elif __grains__['os'] != 'OpenBSD': - with salt.utils.fopen('/proc/swaps') as fp_: + with salt.utils.files.fopen('/proc/swaps') as fp_: for line in fp_: if line.startswith('Filename'): continue @@ -1260,3 +1263,91 @@ def is_mounted(name): return True else: return False + + +def read_mount_cache(name): + ''' + .. versionadded:: Oxygen + + Provide information if the path is mounted + + CLI Example: + + .. code-block:: bash + + salt '*' mount.read_mount_cache /mnt/share + ''' + cache = salt.utils.mount.read_cache(__opts__) + if cache: + if 'mounts' in cache and cache['mounts']: + if name in cache['mounts']: + return cache['mounts'][name] + return {} + + +def write_mount_cache(real_name, + device, + mkmnt, + fstype, + mount_opts): + ''' + .. versionadded:: Oxygen + + Provide information if the path is mounted + + :param real_name: The real name of the mount point where the device is mounted. + :param device: The device that is being mounted. + :param mkmnt: Whether or not the mount point should be created. + :param fstype: The file system that is used. + :param mount_opts: Additional options used when mounting the device. + :return: Boolean if message was sent successfully. + + CLI Example: + + .. code-block:: bash + + salt '*' mount.write_mount_cache /mnt/share /dev/sda1 False ext4 defaults,nosuid + ''' + cache = salt.utils.mount.read_cache(__opts__) + + if not cache: + cache = {} + cache['mounts'] = {} + else: + if 'mounts' not in cache: + cache['mounts'] = {} + + cache['mounts'][real_name] = {'device': device, + 'fstype': fstype, + 'mkmnt': mkmnt, + 'opts': mount_opts} + + cache_write = salt.utils.mount.write_cache(cache, __opts__) + if cache_write: + return True + else: + raise CommandExecutionError('Unable to write mount cache.') + + +def delete_mount_cache(real_name): + ''' + .. versionadded:: Oxygen + + Provide information if the path is mounted + + CLI Example: + + .. code-block:: bash + + salt '*' mount.delete_mount_cache /mnt/share + ''' + cache = salt.utils.mount.read_cache(__opts__) + + if cache: + if 'mounts' in cache: + if real_name in cache['mounts']: + del cache['mounts'][real_name] + cache_write = salt.utils.mount.write_cache(cache, __opts__) + if not cache_write: + raise CommandExecutionError('Unable to write mount cache.') + return True diff --git a/salt/modules/mssql.py b/salt/modules/mssql.py index 1c9d8d5929a..f617479789f 100644 --- a/salt/modules/mssql.py +++ b/salt/modules/mssql.py @@ -25,6 +25,9 @@ Module to provide MS SQL Server compatibility to salt. from __future__ import absolute_import from json import JSONEncoder, loads +import salt.ext.six as six + + try: import pymssql HAS_ALL_IMPORTS = True @@ -127,6 +130,37 @@ def db_exists(database_name, **kwargs): return len(tsql_query("SELECT database_id FROM sys.databases WHERE NAME='{0}'".format(database_name), **kwargs)) == 1 +def db_create(database, containment='NONE', new_database_options=None, **kwargs): + ''' + Creates a new database. + Does not update options of existing databases. + new_database_options can only be a list of strings + + CLI Example: + .. code-block:: bash + salt minion mssql.db_create DB_NAME + ''' + if containment not in ['NONE', 'PARTIAL']: + return 'CONTAINMENT can be one of NONE and PARTIAL' + sql = "CREATE DATABASE [{0}] CONTAINMENT = {1} ".format(database, containment) + if new_database_options: + sql += ' WITH ' + ', '.join(new_database_options) + conn = None + try: + conn = _get_connection(**kwargs) + conn.autocommit(True) + # cur = conn.cursor() + # cur.execute(sql) + conn.cursor().execute(sql) + except Exception as e: + return 'Could not create the login: {0}'.format(e) + finally: + if conn: + conn.autocommit(False) + conn.close() + return True + + def db_remove(database_name, **kwargs): ''' Drops a specific database from the MS SQL server. @@ -183,31 +217,41 @@ def role_exists(role, **kwargs): return len(tsql_query(query='sp_helprole "{0}"'.format(role), as_dict=True, **kwargs)) == 1 -def role_create(role, owner=None, **kwargs): +def role_create(role, owner=None, grants=None, **kwargs): ''' Creates a new database role. If no owner is specified, the role will be owned by the user that executes CREATE ROLE, which is the user argument or mssql.user option. + grants is list of strings. CLI Example: .. code-block:: bash - salt minion mssql.role_create role=product01 owner=sysdba + salt minion mssql.role_create role=product01 owner=sysdba grants='["SELECT", "INSERT", "UPDATE", "DELETE", "EXECUTE"]' ''' + if not grants: + grants = [] + + sql = 'CREATE ROLE {0}'.format(role) + if owner: + sql += ' AUTHORIZATION {0}'.format(owner) + conn = None try: conn = _get_connection(**kwargs) conn.autocommit(True) - cur = conn.cursor() - if owner: - cur.execute('CREATE ROLE {0} AUTHORIZATION {1}'.format(role, owner)) - else: - cur.execute('CREATE ROLE {0}'.format(role)) - conn.autocommit(True) - conn.close() - return True + # cur = conn.cursor() + # cur.execute(sql) + conn.cursor().execute(sql) + for grant in grants: + conn.cursor().execute('GRANT {0} TO [{1}]'.format(grant, role)) except Exception as e: return 'Could not create the role: {0}'.format(e) + finally: + if conn: + conn.autocommit(False) + conn.close() + return True def role_remove(role, **kwargs): @@ -232,9 +276,10 @@ def role_remove(role, **kwargs): return 'Could not create the role: {0}'.format(e) -def login_exists(login, **kwargs): +def login_exists(login, domain='', **kwargs): ''' Find if a login exists in the MS SQL server. + domain, if provided, will be prepended to login CLI Example: @@ -242,6 +287,8 @@ def login_exists(login, **kwargs): salt minion mssql.login_exists 'LOGIN' ''' + if domain: + login = '{0}\\{1}'.format(domain, login) try: # We should get one, and only one row return len(tsql_query(query="SELECT name FROM sys.syslogins WHERE name='{0}'".format(login), **kwargs)) == 1 @@ -250,12 +297,87 @@ def login_exists(login, **kwargs): return 'Could not find the login: {0}'.format(e) -def user_exists(username, **kwargs): +def login_create(login, new_login_password=None, new_login_domain='', new_login_roles=None, new_login_options=None, **kwargs): + ''' + Creates a new login. + Does not update password of existing logins. + For Windows authentication, provide new_login_domain. + For SQL Server authentication, prvide new_login_password. + Since hashed passwords are varbinary values, if the + new_login_password is 'int / long', it will be considered + to be HASHED. + new_login_roles can only be a list of SERVER roles + new_login_options can only be a list of strings + + CLI Example: + .. code-block:: bash + salt minion mssql.login_create LOGIN_NAME database=DBNAME [new_login_password=PASSWORD] + ''' + # One and only one of password and domain should be specifies + if bool(new_login_password) == bool(new_login_domain): + return False + if login_exists(login, new_login_domain, **kwargs): + return False + if new_login_domain: + login = '{0}\\{1}'.format(new_login_domain, login) + if not new_login_roles: + new_login_roles = [] + if not new_login_options: + new_login_options = [] + + sql = "CREATE LOGIN [{0}] ".format(login) + if new_login_domain: + sql += " FROM WINDOWS " + elif isinstance(new_login_password, six.integer_types): + new_login_options.insert(0, "PASSWORD=0x{0:x} HASHED".format(new_login_password)) + else: # Plain test password + new_login_options.insert(0, "PASSWORD=N'{0}'".format(new_login_password)) + if new_login_options: + sql += ' WITH ' + ', '.join(new_login_options) + conn = None + try: + conn = _get_connection(**kwargs) + conn.autocommit(True) + # cur = conn.cursor() + # cur.execute(sql) + conn.cursor().execute(sql) + for role in new_login_roles: + conn.cursor().execute('ALTER SERVER ROLE [{0}] ADD MEMBER [{1}]'.format(role, login)) + except Exception as e: + return 'Could not create the login: {0}'.format(e) + finally: + if conn: + conn.autocommit(False) + conn.close() + return True + + +def login_remove(login, **kwargs): + ''' + Removes an login. + + CLI Example: + + .. code-block:: bash + + salt minion mssql.login_remove LOGINNAME + ''' + try: + conn = _get_connection(**kwargs) + conn.autocommit(True) + cur = conn.cursor() + cur.execute("DROP LOGIN [{0}]".format(login)) + conn.autocommit(False) + conn.close() + return True + except Exception as e: + return 'Could not remove the login: {0}'.format(e) + + +def user_exists(username, domain='', database=None, **kwargs): ''' Find if an user exists in a specific database on the MS SQL server. - - Note: - *database* argument is mandatory + domain, if provided, will be prepended to username CLI Example: @@ -263,10 +385,10 @@ def user_exists(username, **kwargs): salt minion mssql.user_exists 'USERNAME' [database='DBNAME'] ''' - # 'database' argument is mandatory - if 'database' not in kwargs: - return False - + if domain: + username = '{0}\\{1}'.format(domain, username) + if database: + kwargs['database'] = database # We should get one, and only one row return len(tsql_query(query="SELECT name FROM sysusers WHERE name='{0}'".format(username), **kwargs)) == 1 @@ -284,42 +406,57 @@ def user_list(**kwargs): return [row[0] for row in tsql_query("SELECT name FROM sysusers where issqluser=1 or isntuser=1", as_dict=False, **kwargs)] -def user_create(username, new_login_password=None, **kwargs): +def user_create(username, login=None, domain='', database=None, roles=None, options=None, **kwargs): ''' Creates a new user. - If new_login_password is not specified, the user will be created without a login. + If login is not specified, the user will be created without a login. + domain, if provided, will be prepended to username. + options can only be a list of strings CLI Example: - .. code-block:: bash - - salt minion mssql.user_create USERNAME database=DBNAME [new_login_password=PASSWORD] + salt minion mssql.user_create USERNAME database=DBNAME ''' - # 'database' argument is mandatory - if 'database' not in kwargs: - return False - if user_exists(username, **kwargs): - return False + if domain and not login: + return 'domain cannot be set without login' + if user_exists(username, domain, **kwargs): + return 'User {0} already exists'.format(username) + if domain: + username = '{0}\\{1}'.format(domain, username) + login = '{0}\\{1}'.format(domain, login) if login else login + if database: + kwargs['database'] = database + if not roles: + roles = [] + if not options: + options = [] + sql = "CREATE USER [{0}] ".format(username) + if login: + # If the login does not exist, user creation will throw + # if not login_exists(name, **kwargs): + # return False + sql += " FOR LOGIN [{0}]".format(login) + else: # Plain test password + sql += " WITHOUT LOGIN" + if options: + sql += ' WITH ' + ', '.join(options) + conn = None try: conn = _get_connection(**kwargs) conn.autocommit(True) - cur = conn.cursor() - - if new_login_password: - if login_exists(username, **kwargs): - conn.close() - return False - cur.execute("CREATE LOGIN {0} WITH PASSWORD='{1}',check_policy = off".format(username, new_login_password)) - cur.execute("CREATE USER {0} FOR LOGIN {1}".format(username, username)) - else: # new_login_password is not specified - cur.execute("CREATE USER {0} WITHOUT LOGIN".format(username)) - - conn.autocommit(False) - conn.close() - return True + # cur = conn.cursor() + # cur.execute(sql) + conn.cursor().execute(sql) + for role in roles: + conn.cursor().execute('ALTER ROLE [{0}] ADD MEMBER [{1}]'.format(role, username)) except Exception as e: return 'Could not create the user: {0}'.format(e) + finally: + if conn: + conn.autocommit(False) + conn.close() + return True def user_remove(username, **kwargs): diff --git a/salt/modules/munin.py b/salt/modules/munin.py index 565dbadb966..0a78ce1a89f 100644 --- a/salt/modules/munin.py +++ b/salt/modules/munin.py @@ -9,7 +9,7 @@ import os import stat # Import salt libs -import salt.utils +import salt.utils.files from salt.ext.six import string_types PLUGINDIR = '/etc/munin/plugins/' @@ -25,7 +25,7 @@ def __virtual__(): def _get_conf(fname='/etc/munin/munin-node.cfg'): - with salt.utils.fopen(fname, 'r') as fp_: + with salt.utils.files.fopen(fname, 'r') as fp_: return fp_.read() diff --git a/salt/modules/mysql.py b/salt/modules/mysql.py index e97525ad087..26e4150ce41 100644 --- a/salt/modules/mysql.py +++ b/salt/modules/mysql.py @@ -44,9 +44,10 @@ import os # Import salt libs import salt.utils +import salt.utils.files # Import third party libs -import salt.ext.six as six +from salt.ext import six # pylint: disable=import-error from salt.ext.six.moves import range, zip # pylint: disable=no-name-in-module,redefined-builtin try: @@ -701,7 +702,7 @@ def file_query(database, file_name, **connection_args): ''' if os.path.exists(file_name): - with salt.utils.fopen(file_name, 'r') as ifile: + with salt.utils.files.fopen(file_name, 'r') as ifile: contents = ifile.read() else: log.error('File "{0}" does not exist'.format(file_name)) diff --git a/salt/modules/nacl.py b/salt/modules/nacl.py index 9a8f78cd866..228d72974d0 100644 --- a/salt/modules/nacl.py +++ b/salt/modules/nacl.py @@ -11,112 +11,160 @@ regardless if they are encrypted or not. When generating keys and encrypting passwords use --local when using salt-call for extra security. Also consider using just the salt runner nacl when encrypting pillar passwords. +:configuration: The following configuration defaults can be + define (pillar or config files) Avoid storing private keys in pillars! Ensure master does not have `pillar_opts=True`: + + .. code-block:: python + + # cat /etc/salt/master.d/nacl.conf + nacl.config: + # NOTE: `key` and `key_file` have been renamed to `sk`, `sk_file` + # also `box_type` default changed from secretbox to sealedbox. + box_type: sealedbox (default) + sk_file: /etc/salt/pki/master/nacl (default) + pk_file: /etc/salt/pki/master/nacl.pub (default) + sk: None + pk: None + + Usage can override the config defaults: + + .. code-block:: bash + + salt-call nacl.enc sk_file=/etc/salt/pki/master/nacl pk_file=/etc/salt/pki/master/nacl.pub + + The nacl lib uses 32byte keys, these keys are base64 encoded to make your life more simple. -To generate your `key` or `keyfile` you can use: +To generate your `sk_file` and `pk_file` use: .. code-block:: bash - salt-call --local nacl.keygen keyfile=/root/.nacl + salt-call --local nacl.keygen sk_file=/etc/salt/pki/master/nacl + # or if you want to work without files. + salt-call --local nacl.keygen + local: + ---------- + pk: + /kfGX7PbWeu099702PBbKWLpG/9p06IQRswkdWHCDk0= + sk: + SVWut5SqNpuPeNzb1b9y6b2eXg2PLIog43GBzp48Sow= -Now with your key, you can encrypt some data: +Now with your keypair, you can encrypt data: + +You have two option, `sealedbox` or `secretbox`. + +SecretBox is data encrypted using private key `pk`. Sealedbox is encrypted using public key `pk`. + +Recommend using Sealedbox because the one way encryption permits developers to encrypt data for source control but not decrypt. +Sealedbox only has one key that is for both encryption and decryption. .. code-block:: bash - salt-call --local nacl.enc mypass keyfile=/root/.nacl - DRB7Q6/X5gGSRCTpZyxS6hXO5LnlJIIJ4ivbmUlbWj0llUA+uaVyvou3vJ4= + salt-call --local nacl.enc asecretpass pk=/kfGX7PbWeu099702PBbKWLpG/9p06IQRswkdWHCDk0= + tqXzeIJnTAM9Xf0mdLcpEdklMbfBGPj2oTKmlgrm3S1DTVVHNnh9h8mU1GKllGq/+cYsk6m5WhGdk58= To decrypt the data: .. code-block:: bash - salt-call --local nacl.dec data='DRB7Q6/X5gGSRCTpZyxS6hXO5LnlJIIJ4ivbmUlbWj0llUA+uaVyvou3vJ4=' keyfile=/root/.nacl - mypass + salt-call --local nacl.dec data='tqXzeIJnTAM9Xf0mdLcpEdklMbfBGPj2oTKmlgrm3S1DTVVHNnh9h8mU1GKllGq/+cYsk6m5WhGdk58=' \ + sk='SVWut5SqNpuPeNzb1b9y6b2eXg2PLIog43GBzp48Sow=' -The following optional configurations can be defined in the -minion or master config. Avoid storing the config in pillars! +When the keys are defined in the master config you can use them from the nacl runner +without extra parameters: -.. code-block:: yaml +.. code-block:: python - cat /etc/salt/master.d/nacl.conf + # cat /etc/salt/master.d/nacl.conf nacl.config: - key: 'cKEzd4kXsbeCE7/nLTIqXwnUiD1ulg4NoeeYcCFpd9k=' - keyfile: /root/.nacl - -When the key is defined in the master config you can use it from the nacl runner: + sk_file: /etc/salt/pki/master/nacl + pk: 'cTIqXwnUiD1ulg4kXsbeCE7/NoeKEzd4nLeYcCFpd9k=' .. code-block:: bash - salt-run nacl.enc 'myotherpass' + salt-run nacl.enc 'asecretpass' + salt-run nacl.dec 'tqXzeIJnTAM9Xf0mdLcpEdklMbfBGPj2oTKmlgrm3S1DTVVHNnh9h8mU1GKllGq/+cYsk6m5WhGdk58=' -Now you can create a pillar with protected data like: +.. code-block:: yam + # a salt developers minion could have pillar data that includes a nacl public key + nacl.config: + pk: '/kfGX7PbWeu099702PBbKWLpG/9p06IQRswkdWHCDk0=' + +The developer can then use a less-secure system to encrypt data. + +.. code-block:: bash + + salt-call --local nacl.enc apassword + + +Pillar files can include protected data that the salt master decrypts: .. code-block:: jinja pillarexample: user: root - password: {{ salt.nacl.dec('DRB7Q6/X5gGSRCTpZyxS6hXO5LnlJIIJ4ivbmUlbWj0llUA+uaVyvou3vJ4=') }} + password1: {{salt.nacl.dec('DRB7Q6/X5gGSRCTpZyxS6hlbWj0llUA+uaVyvou3vJ4=')|json}} + cert_key: {{salt.nacl.dec_file('/srv/salt/certs/example.com/key.nacl')|json}} + cert_key2: {{salt.nacl.dec_file('salt:///certs/example.com/key.nacl')|json}} -Or do something interesting with grains like: - -.. code-block:: jinja - - salt-call nacl.enc minionname:dbrole - AL24Z2C5OlkReer3DuQTFdrNLchLuz3NGIhGjZkLtKRYry/b/CksWM8O9yskLwH2AGVLoEXI5jAa - - salt minionname grains.setval role 'AL24Z2C5OlkReer3DuQTFdrNLchLuz3NGIhGjZkLtKRYry/b/CksWM8O9yskLwH2AGVLoEXI5jAa' - - {%- set r = grains.get('role') %} - {%- set role = None %} - {%- if r and 'nacl.dec' in salt %} - {%- set r = salt['nacl.dec'](r,keyfile='/root/.nacl').split(':') %} - {%- if opts['id'] == r[0] %} - {%- set role = r[1] %} - {%- endif %} - {%- endif %} - base: - {%- if role %} - '{{ opts['id'] }}': - - {{ role }} - {%- endif %} - -Multi-line text items like certificates require a bit of extra work. You have to strip the new lines -and replace them with '/n' characters. Certificates specifically require some leading white space when -calling nacl.enc so that the '--' in the first line (commonly -----BEGIN CERTIFICATE-----) doesn't get -interpreted as an argument to nacl.enc. For instance if you have a certificate file that lives in cert.crt: +Larger files like certificates can be encrypted with: .. code-block:: bash - cert=$(cat cert.crt |awk '{printf "%s\\n",$0} END {print ""}'); salt-run nacl.enc " $cert" + salt-call nacl.enc_file /tmp/cert.crt out=/tmp/cert.nacl + # or more advanced + cert=$(cat /tmp/cert.crt) + salt-call --out=newline_values_only nacl.enc_pub data="$cert" > /tmp/cert.nacl -Pillar data should look the same, even though the secret will be quite long. However, when calling -multiline encrypted secrets from pillar in a state, use the following format to avoid issues with /n -creating extra whitespace at the beginning of each line in the cert file: +In pillars rended with jinja be sure to include `|json` so line breaks are encoded: .. code-block:: jinja - secret.txt: - file.managed: - - template: jinja - - user: user - - group: group - - mode: 700 - - contents: "{{- salt['pillar.get']('secret') }}" + cert: "{{salt.nacl.dec('S2uogToXkgENz9...085KYt')|json}}" + +In states rendered with jinja it is also good pratice to include `|json`: + +.. code-block:: jinja + + {{sls}} private key: + file.managed: + - name: /etc/ssl/private/cert.key + - mode: 700 + - contents: "{{pillar['pillarexample']['cert_key']|json}}" + + +Optional small program to encrypt data without needing salt modules. + +.. code-block:: python + + #!/bin/python3 + import sys, base64, libnacl.sealed + pk = base64.b64decode('YOURPUBKEY') + b = libnacl.sealed.SealedBox(pk) + data = sys.stdin.buffer.read() + print(base64.b64encode(b.encrypt(data)).decode()) + +.. code-block:: bash + + echo 'apassword' | nacl_enc.py -The '{{-' will tell jinja to strip the whitespace from the beginning of each of the new lines. ''' from __future__ import absolute_import import base64 import os -import salt.utils +import salt.utils.files +import salt.utils.platform +import salt.utils.win_functions +import salt.utils.win_dacl import salt.syspaths REQ_ERROR = None try: import libnacl.secret + import libnacl.sealed except (ImportError, OSError) as e: - REQ_ERROR = 'libnacl import error, perhaps missing python libnacl package' + REQ_ERROR = 'libnacl import error, perhaps missing python libnacl package or should update.' __virtualname__ = 'nacl' @@ -130,91 +178,294 @@ def _get_config(**kwargs): Return configuration ''' config = { - 'key': None, - 'keyfile': None, + 'box_type': 'sealedbox', + 'sk': None, + 'sk_file': '/etc/salt/pki/master/nacl', + 'pk': None, + 'pk_file': '/etc/salt/pki/master/nacl.pub', } config_key = '{0}.config'.format(__virtualname__) - config.update(__salt__['config.get'](config_key, {})) - for k in set(config) & set(kwargs): + try: + config.update(__salt__['config.get'](config_key, {})) + except (NameError, KeyError) as e: + # likly using salt-run so fallback to __opts__ + config.update(__opts__.get(config_key, {})) + # pylint: disable=C0201 + for k in set(config.keys()) & set(kwargs.keys()): config[k] = kwargs[k] return config -def _get_key(rstrip_newline=True, **kwargs): +def _get_sk(**kwargs): ''' - Return key + Return sk ''' config = _get_config(**kwargs) - key = config['key'] - keyfile = config['keyfile'] - if not key and keyfile: - if not os.path.isfile(keyfile): - raise Exception('file not found: {0}'.format(keyfile)) - with salt.utils.fopen(keyfile, 'rb') as keyf: - key = keyf.read() + key = config['sk'] + sk_file = config['sk_file'] + if not key and sk_file: + with salt.utils.files.fopen(sk_file, 'rb') as keyf: + key = str(keyf.read()).rstrip('\n') if key is None: - raise Exception('no key found') - key = str(key) - if rstrip_newline: - key = key.rstrip('\n') - return key + raise Exception('no key or sk_file found') + return base64.b64decode(key) -def keygen(keyfile=None): +def _get_pk(**kwargs): ''' - Use libnacl to generate a private key + Return pk + ''' + config = _get_config(**kwargs) + pubkey = config['pk'] + pk_file = config['pk_file'] + if not pubkey and pk_file: + with salt.utils.files.fopen(pk_file, 'rb') as keyf: + pubkey = str(keyf.read()).rstrip('\n') + if pubkey is None: + raise Exception('no pubkey or pk_file found') + pubkey = str(pubkey) + return base64.b64decode(pubkey) + + +def keygen(sk_file=None, pk_file=None): + ''' + Use libnacl to generate a keypair. + + If no `sk_file` is defined return a keypair. + + If only the `sk_file` is defined `pk_file` will use the same name with a postfix `.pub`. + + When the `sk_file` is already existing, but `pk_file` is not. The `pk_file` will be generated + using the `sk_file`. CLI Examples: .. code-block:: bash + salt-call nacl.keygen + salt-call nacl.keygen sk_file=/etc/salt/pki/master/nacl + salt-call nacl.keygen sk_file=/etc/salt/pki/master/nacl pk_file=/etc/salt/pki/master/nacl.pub salt-call --local nacl.keygen - salt-call --local nacl.keygen keyfile=/root/.nacl - salt-call --local --out=newline_values_only nacl.keygen > /root/.nacl ''' - b = libnacl.secret.SecretBox() - key = b.sk - key = base64.b64encode(key) - if keyfile: - if os.path.isfile(keyfile): - raise Exception('file already found: {0}'.format(keyfile)) - with salt.utils.fopen(keyfile, 'w') as keyf: - keyf.write(key) - return 'saved: {0}'.format(keyfile) - return key + if sk_file is None: + kp = libnacl.public.SecretKey() + return {'sk': base64.b64encode(kp.sk), 'pk': base64.b64encode(kp.pk)} + + if pk_file is None: + pk_file = '{0}.pub'.format(sk_file) + + if sk_file and pk_file is None: + if not os.path.isfile(sk_file): + kp = libnacl.public.SecretKey() + with salt.utils.files.fopen(sk_file, 'w') as keyf: + keyf.write(base64.b64encode(kp.sk)) + if salt.utils.platform.is_windows(): + cur_user = salt.utils.win_functions.get_current_user() + salt.utils.win_dacl.set_owner(sk_file, cur_user) + salt.utils.win_dacl.set_permissions(sk_file, cur_user, 'full_control', 'grant', reset_perms=True, protected=True) + else: + # chmod 0600 file + os.chmod(sk_file, 1536) + return 'saved sk_file: {0}'.format(sk_file) + else: + raise Exception('sk_file:{0} already exist.'.format(sk_file)) + + if sk_file is None and pk_file: + raise Exception('sk_file: Must be set inorder to generate a public key.') + + if os.path.isfile(sk_file) and os.path.isfile(pk_file): + raise Exception('sk_file:{0} and pk_file:{1} already exist.'.format(sk_file, pk_file)) + + if os.path.isfile(sk_file) and not os.path.isfile(pk_file): + # generate pk using the sk + with salt.utils.files.fopen(sk_file, 'rb') as keyf: + sk = str(keyf.read()).rstrip('\n') + sk = base64.b64decode(sk) + kp = libnacl.public.SecretKey(sk) + with salt.utils.files.fopen(pk_file, 'w') as keyf: + keyf.write(base64.b64encode(kp.pk)) + return 'saved pk_file: {0}'.format(pk_file) + + kp = libnacl.public.SecretKey() + with salt.utils.files.fopen(sk_file, 'w') as keyf: + keyf.write(base64.b64encode(kp.sk)) + if salt.utils.platform.is_windows(): + cur_user = salt.utils.win_functions.get_current_user() + salt.utils.win_dacl.set_owner(sk_file, cur_user) + salt.utils.win_dacl.set_permissions(sk_file, cur_user, 'full_control', 'grant', reset_perms=True, protected=True) + else: + # chmod 0600 file + os.chmod(sk_file, 1536) + with salt.utils.files.fopen(pk_file, 'w') as keyf: + keyf.write(base64.b64encode(kp.pk)) + return 'saved sk_file:{0} pk_file: {1}'.format(sk_file, pk_file) def enc(data, **kwargs): ''' - Takes a key generated from `nacl.keygen` and encrypt some data. + Alias to `{box_type}_encrypt` + + box_type: secretbox, sealedbox(default) + ''' + box_type = _get_config(**kwargs)['box_type'] + if box_type == 'sealedbox': + return sealedbox_encrypt(data, **kwargs) + if box_type == 'secretbox': + return secretbox_encrypt(data, **kwargs) + return sealedbox_encrypt(data, **kwargs) + + +def enc_file(name, out=None, **kwargs): + ''' + This is a helper function to encrypt a file and return its contents. + + You can provide an optional output file using `out` + + `name` can be a local file or when not using `salt-run` can be a url like `salt://`, `https://` etc. CLI Examples: .. code-block:: bash - salt-call --local nacl.enc datatoenc - salt-call --local nacl.enc datatoenc keyfile=/root/.nacl - salt-call --local nacl.enc datatoenc key='cKEzd4kXsbeCE7/nLTIqXwnUiD1ulg4NoeeYcCFpd9k=' + salt-run nacl.enc_file name=/tmp/id_rsa + salt-call nacl.enc_file name=salt://crt/mycert out=/tmp/cert + salt-run nacl.enc_file name=/tmp/id_rsa box_type=secretbox \ + sk_file=/etc/salt/pki/master/nacl.pub ''' - key = _get_key(**kwargs) - sk = base64.b64decode(key) - b = libnacl.secret.SecretBox(sk) - return base64.b64encode(b.encrypt(data)) + try: + data = __salt__['cp.get_file_str'](name) + except Exception as e: + # likly using salt-run so fallback to local filesystem + with salt.utils.files.fopen(name, 'rb') as f: + data = f.read() + d = enc(data, **kwargs) + if out: + if os.path.isfile(out): + raise Exception('file:{0} already exist.'.format(out)) + with salt.utils.files.fopen(out, 'wb') as f: + f.write(d) + return 'Wrote: {0}'.format(out) + return d def dec(data, **kwargs): ''' - Takes a key generated from `nacl.keygen` and decrypt some data. + Alias to `{box_type}_decrypt` + + box_type: secretbox, sealedbox(default) + ''' + box_type = _get_config(**kwargs)['box_type'] + if box_type == 'sealedbox': + return sealedbox_decrypt(data, **kwargs) + if box_type == 'secretbox': + return secretbox_decrypt(data, **kwargs) + return sealedbox_decrypt(data, **kwargs) + + +def dec_file(name, out=None, **kwargs): + ''' + This is a helper function to decrypt a file and return its contents. + + You can provide an optional output file using `out` + + `name` can be a local file or when not using `salt-run` can be a url like `salt://`, `https://` etc. CLI Examples: .. code-block:: bash - salt-call --local nacl.dec pEXHQM6cuaF7A= - salt-call --local nacl.dec data='pEXHQM6cuaF7A=' keyfile=/root/.nacl - salt-call --local nacl.dec data='pEXHQM6cuaF7A=' key='cKEzd4kXsbeCE7/nLTIqXwnUiD1ulg4NoeeYcCFpd9k=' + salt-run nacl.dec_file name=/tmp/id_rsa.nacl + salt-call nacl.dec_file name=salt://crt/mycert.nacl out=/tmp/id_rsa + salt-run nacl.dec_file name=/tmp/id_rsa.nacl box_type=secretbox \ + sk_file=/etc/salt/pki/master/nacl.pub ''' - key = _get_key(**kwargs) - sk = base64.b64decode(key) - b = libnacl.secret.SecretBox(key=sk) + try: + data = __salt__['cp.get_file_str'](name) + except Exception as e: + # likly using salt-run so fallback to local filesystem + with salt.utils.files.fopen(name, 'rb') as f: + data = f.read() + d = dec(data, **kwargs) + if out: + if os.path.isfile(out): + raise Exception('file:{0} already exist.'.format(out)) + with salt.utils.files.fopen(out, 'wb') as f: + f.write(d) + return 'Wrote: {0}'.format(out) + return d + + +def sealedbox_encrypt(data, **kwargs): + ''' + Encrypt data using a public key generated from `nacl.keygen`. + The encryptd data can be decrypted using `nacl.sealedbox_decrypt` only with the secret key. + + CLI Examples: + + .. code-block:: bash + + salt-run nacl.sealedbox_encrypt datatoenc + salt-call --local nacl.sealedbox_encrypt datatoenc pk_file=/etc/salt/pki/master/nacl.pub + salt-call --local nacl.sealedbox_encrypt datatoenc pk='vrwQF7cNiNAVQVAiS3bvcbJUnF0cN6fU9YTZD9mBfzQ=' + ''' + pk = _get_pk(**kwargs) + b = libnacl.sealed.SealedBox(pk) + return base64.b64encode(b.encrypt(data)) + + +def sealedbox_decrypt(data, **kwargs): + ''' + Decrypt data using a secret key that was encrypted using a public key with `nacl.sealedbox_encrypt`. + + CLI Examples: + + .. code-block:: bash + + salt-call nacl.sealedbox_decrypt pEXHQM6cuaF7A= + salt-call --local nacl.sealedbox_decrypt data='pEXHQM6cuaF7A=' sk_file=/etc/salt/pki/master/nacl + salt-call --local nacl.sealedbox_decrypt data='pEXHQM6cuaF7A=' sk='YmFkcGFzcwo=' + ''' + if data is None: + return None + sk = _get_sk(**kwargs) + keypair = libnacl.public.SecretKey(sk) + b = libnacl.sealed.SealedBox(keypair) + return b.decrypt(base64.b64decode(data)) + + +def secretbox_encrypt(data, **kwargs): + ''' + Encrypt data using a secret key generated from `nacl.keygen`. + The same secret key can be used to decrypt the data using `nacl.secretbox_decrypt`. + + CLI Examples: + + .. code-block:: bash + + salt-run nacl.secretbox_encrypt datatoenc + salt-call --local nacl.secretbox_encrypt datatoenc sk_file=/etc/salt/pki/master/nacl + salt-call --local nacl.secretbox_encrypt datatoenc sk='YmFkcGFzcwo=' + ''' + sk = _get_sk(**kwargs) + b = libnacl.secret.SecretBox(sk) + return base64.b64encode(b.encrypt(data)) + + +def secretbox_decrypt(data, **kwargs): + ''' + Decrypt data that was encrypted using `nacl.secretbox_encrypt` using the secret key + that was generated from `nacl.keygen`. + + CLI Examples: + + .. code-block:: bash + + salt-call nacl.secretbox_decrypt pEXHQM6cuaF7A= + salt-call --local nacl.secretbox_decrypt data='pEXHQM6cuaF7A=' sk_file=/etc/salt/pki/master/nacl + salt-call --local nacl.secretbox_decrypt data='pEXHQM6cuaF7A=' sk='YmFkcGFzcwo=' + ''' + if data is None: + return None + key = _get_sk(**kwargs) + b = libnacl.secret.SecretBox(key=key) return b.decrypt(base64.b64decode(data)) diff --git a/salt/modules/nagios.py b/salt/modules/nagios.py index f7e77d47700..0f4aa008230 100644 --- a/salt/modules/nagios.py +++ b/salt/modules/nagios.py @@ -10,7 +10,7 @@ import stat import logging # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six log = logging.getLogger(__name__) diff --git a/salt/modules/namecheap_domains.py b/salt/modules/namecheap_domains.py index 4f89ccdba0e..55707179ec5 100644 --- a/salt/modules/namecheap_domains.py +++ b/salt/modules/namecheap_domains.py @@ -49,7 +49,7 @@ except ImportError: CAN_USE_NAMECHEAP = False # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six def __virtual__(): diff --git a/salt/modules/namecheap_ssl.py b/salt/modules/namecheap_ssl.py index 66bee981aac..a0171cedf42 100644 --- a/salt/modules/namecheap_ssl.py +++ b/salt/modules/namecheap_ssl.py @@ -44,7 +44,7 @@ from __future__ import absolute_import # Import Salt libs -import salt.utils +import salt.utils.files try: import salt.utils.namecheap CAN_USE_NAMECHEAP = True @@ -52,7 +52,7 @@ except ImportError: CAN_USE_NAMECHEAP = False # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six def __virtual__(): @@ -224,7 +224,7 @@ def __get_certificates(command, opts = salt.utils.namecheap.get_opts(command) - with salt.utils.fopen(csr_file, 'rb') as csr_handle: + with salt.utils.files.fopen(csr_file, 'rb') as csr_handle: opts['csr'] = csr_handle.read() opts['CertificateID'] = certificate_id @@ -597,7 +597,7 @@ def parse_csr(csr_file, certificate_type, http_dc_validation=False): opts = salt.utils.namecheap.get_opts('namecheap.ssl.parseCSR') - with salt.utils.fopen(csr_file, 'rb') as csr_handle: + with salt.utils.files.fopen(csr_file, 'rb') as csr_handle: opts['csr'] = csr_handle.read() opts['CertificateType'] = certificate_type diff --git a/salt/modules/napalm_network.py b/salt/modules/napalm_network.py index 5042eee0259..4a317246638 100644 --- a/salt/modules/napalm_network.py +++ b/salt/modules/napalm_network.py @@ -21,15 +21,17 @@ Dependencies from __future__ import absolute_import -# Import python lib +# Import Python lib import logging log = logging.getLogger(__name__) -# salt libs -from salt.ext import six -import salt.utils.templates +# Import Salt libs +import salt.utils.files import salt.utils.napalm -from salt.utils.napalm import proxy_napalm_wrap +import salt.utils.templates + +# Import 3rd-party libs +from salt.ext import six # ---------------------------------------------------------------------------------------------------------------------- # module properties @@ -220,7 +222,7 @@ def _config_logic(napalm_device, # ---------------------------------------------------------------------------------------------------------------------- -@proxy_napalm_wrap +@salt.utils.napalm.proxy_napalm_wrap def connected(**kwarvs): # pylint: disable=unused-argument ''' Specifies if the connection to the device succeeded. @@ -237,7 +239,7 @@ def connected(**kwarvs): # pylint: disable=unused-argument } -@proxy_napalm_wrap +@salt.utils.napalm.proxy_napalm_wrap def facts(**kwargs): # pylint: disable=unused-argument ''' Returns characteristics of the network device. @@ -292,7 +294,7 @@ def facts(**kwargs): # pylint: disable=unused-argument ) -@proxy_napalm_wrap +@salt.utils.napalm.proxy_napalm_wrap def environment(**kwargs): # pylint: disable=unused-argument ''' Returns the environment of the device. @@ -359,14 +361,129 @@ def environment(**kwargs): # pylint: disable=unused-argument ) -@proxy_napalm_wrap +@salt.utils.napalm.proxy_napalm_wrap def cli(*commands, **kwargs): # pylint: disable=unused-argument - ''' Returns a dictionary with the raw output of all commands passed as arguments. - :param commands: list of commands to be executed on the device - :return: a dictionary with the mapping between each command and its raw output + commands + List of commands to be executed on the device. + + textfsm_parse: ``False`` + Try parsing the outputs using the TextFSM templates. + + .. versionadded:: Oxygen + + .. note:: + This option can be also specified in the minion configuration + file or pillar as ``napalm_cli_textfsm_parse``. + + textfsm_path + The path where the TextFSM templates can be found. This option implies + the usage of the TextFSM index file. + ``textfsm_path`` can be either absolute path on the server, + either specified using the following URL mschemes: ``file://``, + ``salt://``, ``http://``, ``https://``, ``ftp://``, + ``s3://``, ``swift://``. + + .. versionadded:: Oxygen + + .. note:: + This needs to be a directory with a flat structure, having an + index file (whose name can be specified using the ``index_file`` option) + and a number of TextFSM templates. + + .. note:: + This option can be also specified in the minion configuration + file or pillar as ``textfsm_path``. + + textfsm_template + The path to a certain the TextFSM template. + This can be specified using the absolute path + to the file, or using one of the following URL schemes: + + - ``salt://``, to fetch the template from the Salt fileserver. + - ``http://`` or ``https://`` + - ``ftp://`` + - ``s3://`` + - ``swift://`` + + .. versionadded:: Oxygen + + textfsm_template_dict + A dictionary with the mapping between a command + and the corresponding TextFSM path to use to extract the data. + The TextFSM paths can be specified as in ``textfsm_template``. + + .. versionadded:: Oxygen + + .. note:: + This option can be also specified in the minion configuration + file or pillar as ``napalm_cli_textfsm_template_dict``. + + platform_grain_name: ``os`` + The name of the grain used to identify the platform name + in the TextFSM index file. Default: ``os``. + + .. versionadded:: Oxygen + + .. note:: + This option can be also specified in the minion configuration + file or pillar as ``textfsm_platform_grain``. + + platform_column_name: ``Platform`` + The column name used to identify the platform, + exactly as specified in the TextFSM index file. + Default: ``Platform``. + + .. versionadded:: Oxygen + + .. note:: + This is field is case sensitive, make sure + to assign the correct value to this option, + exactly as defined in the index file. + + .. note:: + This option can be also specified in the minion configuration + file or pillar as ``textfsm_platform_column_name``. + + index_file: ``index`` + The name of the TextFSM index file, under the ``textfsm_path``. Default: ``index``. + + .. versionadded:: Oxygen + + .. note:: + This option can be also specified in the minion configuration + file or pillar as ``textfsm_index_file``. + + saltenv: ``base`` + Salt fileserver envrionment from which to retrieve the file. + Ignored if ``textfsm_path`` is not a ``salt://`` URL. + + .. versionadded:: Oxygen + + include_empty: ``False`` + Include empty files under the ``textfsm_path``. + + .. versionadded:: Oxygen + + include_pat + Glob or regex to narrow down the files cached from the given path. + If matching with a regex, the regex must be prefixed with ``E@``, + otherwise the expression will be interpreted as a glob. + + .. versionadded:: Oxygen + + exclude_pat + Glob or regex to exclude certain files from being cached from the given path. + If matching with a regex, the regex must be prefixed with ``E@``, + otherwise the expression will be interpreted as a glob. + + .. versionadded:: Oxygen + + .. note:: + If used with ``include_pat``, files matching this pattern will be + excluded from the subset of files defined by ``include_pat``. CLI Example: @@ -374,6 +491,12 @@ def cli(*commands, **kwargs): # pylint: disable=unused-argument salt '*' net.cli "show version" "show chassis fan" + CLI Example with TextFSM template: + + .. code-block:: + + salt '*' net.cli textfsm_parse=True textfsm_path=salt://textfsm/ + Example output: .. code-block:: python @@ -395,9 +518,31 @@ def cli(*commands, **kwargs): # pylint: disable=unused-argument Bottom Front Fan OK 3840 Spinning at intermediate-speed ' } - ''' - return salt.utils.napalm.call( + Example output with TextFSM parsing: + + .. code-block:: json + + { + "comment": "", + "result": true, + "out": { + "sh ver": [ + { + "kernel": "9.1S3.5", + "documentation": "9.1S3.5", + "boot": "9.1S3.5", + "crypto": "9.1S3.5", + "chassis": "", + "routing": "9.1S3.5", + "base": "9.1S3.5", + "model": "mx960" + } + ] + } + } + ''' + raw_cli_outputs = salt.utils.napalm.call( napalm_device, # pylint: disable=undefined-variable 'cli', **{ @@ -406,9 +551,110 @@ def cli(*commands, **kwargs): # pylint: disable=unused-argument ) # thus we can display the output as is # in case of errors, they'll be catched in the proxy + if not raw_cli_outputs['result']: + # Error -> dispaly the output as-is. + return raw_cli_outputs + textfsm_parse = kwargs.get('textfsm_parse') or __opts__.get('napalm_cli_textfsm_parse') or\ + __pillar__.get('napalm_cli_textfsm_parse', False) + if not textfsm_parse: + # No TextFSM parsing required, return raw commands. + log.debug('No TextFSM parsing requested.') + return raw_cli_outputs + if 'textfsm.extract' not in __salt__ or 'textfsm.index' not in __salt__: + raw_cli_outputs['comment'] += 'Unable to process: is TextFSM installed?' + log.error(raw_cli_outputs['comment']) + return raw_cli_outputs + textfsm_template = kwargs.get('textfsm_template') + log.debug('textfsm_template: {}'.format(textfsm_template)) + textfsm_path = kwargs.get('textfsm_path') or __opts__.get('textfsm_path') or\ + __pillar__.get('textfsm_path') + log.debug('textfsm_path: {}'.format(textfsm_path)) + textfsm_template_dict = kwargs.get('textfsm_template_dict') or __opts__.get('napalm_cli_textfsm_template_dict') or\ + __pillar__.get('napalm_cli_textfsm_template_dict', {}) + log.debug('TextFSM command-template mapping: {}'.format(textfsm_template_dict)) + index_file = kwargs.get('index_file') or __opts__.get('textfsm_index_file') or\ + __pillar__.get('textfsm_index_file') + log.debug('index_file: {}'.format(index_file)) + platform_grain_name = kwargs.get('platform_grain_name') or __opts__.get('textfsm_platform_grain') or\ + __pillar__.get('textfsm_platform_grain', 'os') + log.debug('platform_grain_name: {}'.format(platform_grain_name)) + platform_column_name = kwargs.get('platform_column_name') or __opts__.get('textfsm_platform_column_name') or\ + __pillar__.get('textfsm_platform_column_name', 'Platform') + log.debug('platform_column_name: {}'.format(platform_column_name)) + saltenv = kwargs.get('saltenv', 'base') + include_empty = kwargs.get('include_empty', False) + include_pat = kwargs.get('include_pat') + exclude_pat = kwargs.get('exclude_pat') + processed_cli_outputs = { + 'comment': raw_cli_outputs.get('comment', ''), + 'result': raw_cli_outputs['result'], + 'out': {} + } + log.debug('Starting to analyse the raw outputs') + for command in list(commands): + command_output = raw_cli_outputs['out'][command] + log.debug('Output from command: {}'.format(command)) + log.debug(command_output) + processed_command_output = None + if textfsm_path: + log.debug('Using the templates under {}'.format(textfsm_path)) + processed_cli_output = __salt__['textfsm.index'](command, + platform_grain_name=platform_grain_name, + platform_column_name=platform_column_name, + output=command_output.strip(), + textfsm_path=textfsm_path, + saltenv=saltenv, + include_empty=include_empty, + include_pat=include_pat, + exclude_pat=exclude_pat) + log.debug('Processed CLI output:') + log.debug(processed_cli_output) + if not processed_cli_output['result']: + log.debug('Apparently this didnt work, returnin the raw output') + processed_command_output = command_output + processed_cli_outputs['comment'] += '\nUnable to process the output from {0}: {1}.'.format(command, + processed_cli_output['comment']) + log.error(processed_cli_outputs['comment']) + elif processed_cli_output['out']: + log.debug('All good, {} has a nice output!'.format(command)) + processed_command_output = processed_cli_output['out'] + else: + comment = '''\nProcessing "{}" didn't fail, but didn't return anything either. Dumping raw.'''.format( + command) + processed_cli_outputs['comment'] += comment + log.error(comment) + processed_command_output = command_output + elif textfsm_template or command in textfsm_template_dict: + if command in textfsm_template_dict: + textfsm_template = textfsm_template_dict[command] + log.debug('Using {0} to process the command: {1}'.format(textfsm_template, command)) + processed_cli_output = __salt__['textfsm.extract'](textfsm_template, + raw_text=command_output, + saltenv=saltenv) + log.debug('Processed CLI output:') + log.debug(processed_cli_output) + if not processed_cli_output['result']: + log.debug('Apparently this didnt work, returnin the raw output') + processed_command_output = command_output + processed_cli_outputs['comment'] += '\nUnable to process the output from {0}: {1}'.format(command, + processed_cli_output['comment']) + log.error(processed_cli_outputs['comment']) + elif processed_cli_output['out']: + log.debug('All good, {} has a nice output!'.format(command)) + processed_command_output = processed_cli_output['out'] + else: + log.debug('Processing {} didnt fail, but didnt return anything either. Dumping raw.'.format(command)) + processed_command_output = command_output + else: + log.error('No TextFSM template specified, or no TextFSM path defined') + processed_command_output = command_output + processed_cli_outputs['comment'] += '\nUnable to process the output from {}.'.format(command) + processed_cli_outputs['out'][command] = processed_command_output + processed_cli_outputs['comment'] = processed_cli_outputs['comment'].strip() + return processed_cli_outputs -@proxy_napalm_wrap +@salt.utils.napalm.proxy_napalm_wrap def traceroute(destination, source=None, ttl=None, timeout=None, vrf=None, **kwargs): # pylint: disable=unused-argument ''' @@ -453,7 +699,7 @@ def traceroute(destination, source=None, ttl=None, timeout=None, vrf=None, **kwa ) -@proxy_napalm_wrap +@salt.utils.napalm.proxy_napalm_wrap def ping(destination, source=None, ttl=None, timeout=None, size=None, count=None, vrf=None, **kwargs): # pylint: disable=unused-argument ''' @@ -506,7 +752,7 @@ def ping(destination, source=None, ttl=None, timeout=None, size=None, count=None ) -@proxy_napalm_wrap +@salt.utils.napalm.proxy_napalm_wrap def arp(interface='', ipaddr='', macaddr='', **kwargs): # pylint: disable=unused-argument ''' @@ -572,7 +818,7 @@ def arp(interface='', ipaddr='', macaddr='', **kwargs): # pylint: disable=unuse return proxy_output -@proxy_napalm_wrap +@salt.utils.napalm.proxy_napalm_wrap def ipaddrs(**kwargs): # pylint: disable=unused-argument ''' @@ -632,7 +878,7 @@ def ipaddrs(**kwargs): # pylint: disable=unused-argument ) -@proxy_napalm_wrap +@salt.utils.napalm.proxy_napalm_wrap def interfaces(**kwargs): # pylint: disable=unused-argument ''' @@ -679,7 +925,7 @@ def interfaces(**kwargs): # pylint: disable=unused-argument ) -@proxy_napalm_wrap +@salt.utils.napalm.proxy_napalm_wrap def lldp(interface='', **kwargs): # pylint: disable=unused-argument ''' @@ -741,7 +987,7 @@ def lldp(interface='', **kwargs): # pylint: disable=unused-argument return proxy_output -@proxy_napalm_wrap +@salt.utils.napalm.proxy_napalm_wrap def mac(address='', interface='', vlan=0, **kwargs): # pylint: disable=unused-argument ''' @@ -814,7 +1060,7 @@ def mac(address='', interface='', vlan=0, **kwargs): # pylint: disable=unused-a return proxy_output -@proxy_napalm_wrap +@salt.utils.napalm.proxy_napalm_wrap def config(source=None, **kwargs): # pylint: disable=unused-argument ''' .. versionadded:: 2017.7.0 @@ -867,7 +1113,7 @@ def config(source=None, **kwargs): # pylint: disable=unused-argument ) -@proxy_napalm_wrap +@salt.utils.napalm.proxy_napalm_wrap def optics(**kwargs): # pylint: disable=unused-argument ''' .. versionadded:: 2017.7.0 @@ -917,7 +1163,7 @@ def optics(**kwargs): # pylint: disable=unused-argument # ----- Configuration specific functions ------------------------------------------------------------------------------> -@proxy_napalm_wrap +@salt.utils.napalm.proxy_napalm_wrap def load_config(filename=None, text=None, test=False, @@ -1016,7 +1262,7 @@ def load_config(filename=None, loaded_config = None if debug: if filename: - with salt.utils.fopen(filename) as rfh: + with salt.utils.files.fopen(filename) as rfh: loaded_config = rfh.read() else: loaded_config = text @@ -1027,7 +1273,7 @@ def load_config(filename=None, loaded_config=loaded_config) -@proxy_napalm_wrap +@salt.utils.napalm.proxy_napalm_wrap def load_template(template_name, template_source=None, template_path=None, @@ -1036,6 +1282,7 @@ def load_template(template_name, template_user='root', template_group='root', template_mode='755', + template_attrs='--------------e----', saltenv=None, template_engine='jinja', skip_verify=False, @@ -1122,11 +1369,16 @@ def load_template(template_name, .. versionadded:: 2016.11.2 - template_user: 755 + template_mode: 755 Permissions of file. .. versionadded:: 2016.11.2 + template_attrs: "--------------e----" + attributes of file. (see `man lsattr`) + + .. versionadded:: oxygen + saltenv: base Specifies the template environment. This will influence the relative imports inside the templates. @@ -1340,6 +1592,7 @@ def load_template(template_name, user=template_user, group=template_group, mode=template_mode, + attrs=template_attrs, template=template_engine, context=template_vars, defaults=defaults, @@ -1361,7 +1614,7 @@ def load_template(template_name, _loaded['result'] = False _loaded['comment'] = 'Error while rendering the template.' return _loaded - with salt.utils.fopen(_temp_tpl_file) as rfh: + with salt.utils.files.fopen(_temp_tpl_file) as rfh: _rendered = rfh.read() __salt__['file.remove'](_temp_tpl_file) else: @@ -1426,7 +1679,7 @@ def load_template(template_name, loaded_config=loaded_config) -@proxy_napalm_wrap +@salt.utils.napalm.proxy_napalm_wrap def commit(inherit_napalm_device=None, **kwargs): # pylint: disable=unused-argument ''' @@ -1446,7 +1699,7 @@ def commit(inherit_napalm_device=None, **kwargs): # pylint: disable=unused-argu ) -@proxy_napalm_wrap +@salt.utils.napalm.proxy_napalm_wrap def discard_config(inherit_napalm_device=None, **kwargs): # pylint: disable=unused-argument """ @@ -1466,7 +1719,7 @@ def discard_config(inherit_napalm_device=None, **kwargs): # pylint: disable=unu ) -@proxy_napalm_wrap +@salt.utils.napalm.proxy_napalm_wrap def compare_config(inherit_napalm_device=None, **kwargs): # pylint: disable=unused-argument ''' @@ -1486,7 +1739,7 @@ def compare_config(inherit_napalm_device=None, **kwargs): # pylint: disable=unu ) -@proxy_napalm_wrap +@salt.utils.napalm.proxy_napalm_wrap def rollback(inherit_napalm_device=None, **kwargs): # pylint: disable=unused-argument ''' @@ -1506,7 +1759,7 @@ def rollback(inherit_napalm_device=None, **kwargs): # pylint: disable=unused-ar ) -@proxy_napalm_wrap +@salt.utils.napalm.proxy_napalm_wrap def config_changed(inherit_napalm_device=None, **kwargs): # pylint: disable=unused-argument ''' @@ -1537,7 +1790,7 @@ def config_changed(inherit_napalm_device=None, **kwargs): # pylint: disable=unu return is_config_changed, reason -@proxy_napalm_wrap +@salt.utils.napalm.proxy_napalm_wrap def config_control(inherit_napalm_device=None, **kwargs): # pylint: disable=unused-argument ''' diff --git a/salt/modules/netbsd_sysctl.py b/salt/modules/netbsd_sysctl.py index 487ef77631c..b6995580aae 100644 --- a/salt/modules/netbsd_sysctl.py +++ b/salt/modules/netbsd_sysctl.py @@ -7,7 +7,7 @@ import os import re # Import salt libs -import salt.utils +import salt.utils.files from salt.exceptions import CommandExecutionError # Define the module's virtual name @@ -117,13 +117,13 @@ def persist(name, value, config='/etc/sysctl.conf'): # create /etc/sysctl.conf if not present if not os.path.isfile(config): try: - with salt.utils.fopen(config, 'w+'): + with salt.utils.files.fopen(config, 'w+'): pass except (IOError, OSError): msg = 'Could not create {0}' raise CommandExecutionError(msg.format(config)) - with salt.utils.fopen(config, 'r') as ifile: + with salt.utils.files.fopen(config, 'r') as ifile: for line in ifile: m = re.match(r'{0}(\??=)'.format(name), line) if not m: @@ -148,7 +148,7 @@ def persist(name, value, config='/etc/sysctl.conf'): newline = '{0}={1}'.format(name, value) nlines.append("{0}\n".format(newline)) - with salt.utils.fopen(config, 'w+') as ofile: + with salt.utils.files.fopen(config, 'w+') as ofile: ofile.writelines(nlines) assign(name, value) diff --git a/salt/modules/netscaler.py b/salt/modules/netscaler.py index 409ef19ab75..ab91db51260 100644 --- a/salt/modules/netscaler.py +++ b/salt/modules/netscaler.py @@ -43,9 +43,12 @@ Module to provide Citrix Netscaler compatibility to Salt (compatible with netsca salt-call netscaler.server_up server_name3 netscaler_host=1.2.3.6 netscaler_useSSL=False ''' +# Import Python libs from __future__ import absolute_import import logging -import salt.utils + +# Import Salt libs +import salt.utils.platform try: from nsnitro.nsnitro import NSNitro @@ -68,11 +71,19 @@ def __virtual__(): ''' Only load this module if the nsnitro library is installed ''' - if salt.utils.is_windows(): - return (False, 'The netscaler execution module failed to load: not available on Windows.') + if salt.utils.platform.is_windows(): + return ( + False, + 'The netscaler execution module failed to load: not available ' + 'on Windows.' + ) if HAS_NSNITRO: return 'netscaler' - return (False, 'The netscaler execution module failed to load: the nsnitro python library is not available.') + return ( + False, + 'The netscaler execution module failed to load: the nsnitro python ' + 'library is not available.' + ) def _connect(**kwargs): diff --git a/salt/modules/network.py b/salt/modules/network.py index 05e67ec2c69..a023d6ff7be 100644 --- a/salt/modules/network.py +++ b/salt/modules/network.py @@ -13,14 +13,18 @@ import os import socket # Import salt libs -import salt.utils -import salt.utils.decorators as decorators +import salt.utils # Can be removed when alias_function mac_str_to_bytes are moved +import salt.utils.decorators.path +import salt.utils.files import salt.utils.network +import salt.utils.path +import salt.utils.platform +import salt.utils.stringutils import salt.utils.validate.net from salt.exceptions import CommandExecutionError # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import range # pylint: disable=import-error,no-name-in-module,redefined-builtin if six.PY3: import ipaddress @@ -36,7 +40,7 @@ def __virtual__(): Only work on POSIX-like systems ''' # Disable on Windows, a specific file module exists: - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): return (False, 'The network execution module cannot be loaded on Windows: use win_network instead.') return True @@ -139,6 +143,63 @@ def _netstat_linux(): return ret +def _ss_linux(): + ''' + Return ss information for Linux distros + (netstat is deprecated and may not be available) + ''' + ret = [] + cmd = 'ss -tulpnea' + out = __salt__['cmd.run'](cmd) + for line in out.splitlines(): + comps = line.split() + ss_user = 0 + ss_inode = 0 + ss_program = '' + length = len(comps) + if line.startswith('tcp') or line.startswith('udp'): + i = 6 + while i < (length - 1): + fields = comps[i].split(":") + if fields[0] == "users": + users = fields[1].split(",") + ss_program = users[0].split("\"")[1] + + if fields[0] == "uid": + ss_user = fields[1] + + if fields[0] == "ino": + ss_inode = fields[1] + + i += 1 + + if line.startswith('tcp'): + ss_state = comps[1] + if ss_state == "ESTAB": + ss_state = "ESTABLISHED" + ret.append({ + 'proto': comps[0], + 'recv-q': comps[2], + 'send-q': comps[3], + 'local-address': comps[4], + 'remote-address': comps[5], + 'state': ss_state, + 'user': ss_user, + 'inode': ss_inode, + 'program': ss_program}) + if line.startswith('udp'): + ret.append({ + 'proto': comps[0], + 'recv-q': comps[2], + 'send-q': comps[3], + 'local-address': comps[4], + 'remote-address': comps[5], + 'user': ss_user, + 'inode': ss_inode, + 'program': ss_program}) + return ret + + def _netinfo_openbsd(): ''' Get process information for network connections using fstat @@ -409,7 +470,7 @@ def _netstat_route_linux(): 'destination': comps[0], 'gateway': comps[1], 'netmask': '', - 'flags': comps[3], + 'flags': comps[2], 'interface': comps[5]}) elif len(comps) == 7: ret.append({ @@ -417,13 +478,109 @@ def _netstat_route_linux(): 'destination': comps[0], 'gateway': comps[1], 'netmask': '', - 'flags': comps[3], + 'flags': comps[2], 'interface': comps[6]}) else: continue return ret +def _ip_route_linux(): + ''' + Return ip routing information for Linux distros + (netstat is deprecated and may not be available) + ''' + # table main closest to old netstat inet output + ret = [] + cmd = 'ip -4 route show table main' + out = __salt__['cmd.run'](cmd, python_shell=True) + for line in out.splitlines(): + comps = line.split() + + # need to fake similar output to that provided by netstat + # to maintain output format + if comps[0] == "unreachable": + continue + + if comps[0] == "default": + ip_interface = '' + if comps[3] == "dev": + ip_interface = comps[4] + + ret.append({ + 'addr_family': 'inet', + 'destination': '0.0.0.0', + 'gateway': comps[2], + 'netmask': '0.0.0.0', + 'flags': 'UG', + 'interface': ip_interface}) + else: + address_mask = convert_cidr(comps[0]) + ip_interface = '' + if comps[1] == "dev": + ip_interface = comps[2] + + ret.append({ + 'addr_family': 'inet', + 'destination': address_mask['network'], + 'gateway': '0.0.0.0', + 'netmask': address_mask['netmask'], + 'flags': 'U', + 'interface': ip_interface}) + + # table all closest to old netstat inet6 output + cmd = 'ip -6 route show table all' + out = __salt__['cmd.run'](cmd, python_shell=True) + for line in out.splitlines(): + comps = line.split() + + # need to fake similar output to that provided by netstat + # to maintain output format + if comps[0] == "unreachable": + continue + + if comps[0] == "default": + ip_interface = '' + if comps[3] == "dev": + ip_interface = comps[4] + + ret.append({ + 'addr_family': 'inet6', + 'destination': '::', + 'gateway': comps[2], + 'netmask': '', + 'flags': 'UG', + 'interface': ip_interface}) + + elif comps[0] == "local": + ip_interface = '' + if comps[2] == "dev": + ip_interface = comps[3] + + local_address = comps[1] + "/128" + ret.append({ + 'addr_family': 'inet6', + 'destination': local_address, + 'gateway': '::', + 'netmask': '', + 'flags': 'U', + 'interface': ip_interface}) + else: + address_mask = convert_cidr(comps[0]) + ip_interface = '' + if comps[1] == "dev": + ip_interface = comps[2] + + ret.append({ + 'addr_family': 'inet6', + 'destination': comps[0], + 'gateway': '::', + 'netmask': '', + 'flags': 'U', + 'interface': ip_interface}) + return ret + + def _netstat_route_freebsd(): ''' Return netstat routing information for FreeBSD and macOS @@ -607,7 +764,10 @@ def netstat(): salt '*' network.netstat ''' if __grains__['kernel'] == 'Linux': - return _netstat_linux() + if not salt.utils.path.which('netstat'): + return _ss_linux() + else: + return _netstat_linux() elif __grains__['kernel'] in ('OpenBSD', 'FreeBSD', 'NetBSD'): return _netstat_bsd() elif __grains__['kernel'] == 'SunOS': @@ -684,7 +844,7 @@ def traceroute(host): salt '*' network.traceroute archlinux.org ''' ret = [] - if not salt.utils.which('traceroute'): + if not salt.utils.path.which('traceroute'): log.info('This minion does not have traceroute installed') return ret @@ -693,7 +853,7 @@ def traceroute(host): out = __salt__['cmd.run'](cmd) # Parse version of traceroute - if salt.utils.is_sunos() or salt.utils.is_aix(): + if salt.utils.platform.is_sunos() or salt.utils.platform.is_aix(): traceroute_version = [0, 0, 0] else: cmd2 = 'traceroute --version' @@ -726,7 +886,7 @@ def traceroute(host): if line.startswith('traceroute'): continue - if salt.utils.is_aix(): + if salt.utils.platform.is_aix(): if line.startswith('trying to get source for'): continue @@ -799,7 +959,7 @@ def traceroute(host): return ret -@decorators.which('dig') +@salt.utils.decorators.path.which('dig') def dig(host): ''' Performs a DNS lookup with dig @@ -814,7 +974,7 @@ def dig(host): return __salt__['cmd.run'](cmd) -@decorators.which('arp') +@salt.utils.decorators.path.which('arp') def arp(): ''' Return the arp table from the minion @@ -1111,10 +1271,10 @@ def mod_hostname(hostname): if hostname is None: return False - hostname_cmd = salt.utils.which('hostnamectl') or salt.utils.which('hostname') - if salt.utils.is_sunos(): - uname_cmd = '/usr/bin/uname' if salt.utils.is_smartos() else salt.utils.which('uname') - check_hostname_cmd = salt.utils.which('check-hostname') + hostname_cmd = salt.utils.path.which('hostnamectl') or salt.utils.path.which('hostname') + if salt.utils.platform.is_sunos(): + uname_cmd = '/usr/bin/uname' if salt.utils.platform.is_smartos() else salt.utils.path.which('uname') + check_hostname_cmd = salt.utils.path.which('check-hostname') # Grab the old hostname so we know which hostname to change and then # change the hostname using the hostname command @@ -1124,7 +1284,7 @@ def mod_hostname(hostname): line = line.split(':') if 'Static hostname' in line[0]: o_hostname = line[1].strip() - elif not salt.utils.is_sunos(): + elif not salt.utils.platform.is_sunos(): # don't run hostname -f because -f is not supported on all platforms o_hostname = socket.getfqdn() else: @@ -1133,23 +1293,23 @@ def mod_hostname(hostname): if hostname_cmd.endswith('hostnamectl'): __salt__['cmd.run']('{0} set-hostname {1}'.format(hostname_cmd, hostname)) - elif not salt.utils.is_sunos(): + elif not salt.utils.platform.is_sunos(): __salt__['cmd.run']('{0} {1}'.format(hostname_cmd, hostname)) else: __salt__['cmd.run']('{0} -S {1}'.format(uname_cmd, hostname.split('.')[0])) # Modify the /etc/hosts file to replace the old hostname with the # new hostname - with salt.utils.fopen('/etc/hosts', 'r') as fp_: + with salt.utils.files.fopen('/etc/hosts', 'r') as fp_: host_c = fp_.readlines() - with salt.utils.fopen('/etc/hosts', 'w') as fh_: + with salt.utils.files.fopen('/etc/hosts', 'w') as fh_: for host in host_c: host = host.split() try: host[host.index(o_hostname)] = hostname - if salt.utils.is_sunos(): + if salt.utils.platform.is_sunos(): # also set a copy of the hostname host[host.index(o_hostname.split('.')[0])] = hostname.split('.')[0] except ValueError: @@ -1160,30 +1320,30 @@ def mod_hostname(hostname): # Modify the /etc/sysconfig/network configuration file to set the # new hostname if __grains__['os_family'] == 'RedHat': - with salt.utils.fopen('/etc/sysconfig/network', 'r') as fp_: + with salt.utils.files.fopen('/etc/sysconfig/network', 'r') as fp_: network_c = fp_.readlines() - with salt.utils.fopen('/etc/sysconfig/network', 'w') as fh_: + with salt.utils.files.fopen('/etc/sysconfig/network', 'w') as fh_: for net in network_c: if net.startswith('HOSTNAME'): old_hostname = net.split('=', 1)[1].rstrip() - quote_type = salt.utils.is_quoted(old_hostname) + quote_type = salt.utils.stringutils.is_quoted(old_hostname) fh_.write('HOSTNAME={1}{0}{1}\n'.format( - salt.utils.dequote(hostname), quote_type)) + salt.utils.stringutils.dequote(hostname), quote_type)) else: fh_.write(net) elif __grains__['os_family'] in ('Debian', 'NILinuxRT'): - with salt.utils.fopen('/etc/hostname', 'w') as fh_: + with salt.utils.files.fopen('/etc/hostname', 'w') as fh_: fh_.write(hostname + '\n') elif __grains__['os_family'] == 'OpenBSD': - with salt.utils.fopen('/etc/myname', 'w') as fh_: + with salt.utils.files.fopen('/etc/myname', 'w') as fh_: fh_.write(hostname + '\n') # Update /etc/nodename and /etc/defaultdomain on SunOS - if salt.utils.is_sunos(): - with salt.utils.fopen('/etc/nodename', 'w') as fh_: + if salt.utils.platform.is_sunos(): + with salt.utils.files.fopen('/etc/nodename', 'w') as fh_: fh_.write(hostname.split('.')[0] + '\n') - with salt.utils.fopen('/etc/defaultdomain', 'w') as fh_: + with salt.utils.files.fopen('/etc/defaultdomain', 'w') as fh_: fh_.write(".".join(hostname.split('.')[1:]) + '\n') return True @@ -1445,7 +1605,10 @@ def routes(family=None): raise CommandExecutionError('Invalid address family {0}'.format(family)) if __grains__['kernel'] == 'Linux': - routes_ = _netstat_route_linux() + if not salt.utils.path.which('netstat'): + routes_ = _ip_route_linux() + else: + routes_ = _netstat_route_linux() elif __grains__['kernel'] == 'SunOS': routes_ = _netstat_route_sunos() elif __grains__['os'] in ['FreeBSD', 'MacOS', 'Darwin']: diff --git a/salt/modules/nfs3.py b/salt/modules/nfs3.py index f4f7d227ee7..0d692022108 100644 --- a/salt/modules/nfs3.py +++ b/salt/modules/nfs3.py @@ -8,7 +8,8 @@ from __future__ import absolute_import import logging # Import salt libs -import salt.utils +import salt.utils.files +import salt.utils.path log = logging.getLogger(__name__) @@ -17,7 +18,7 @@ def __virtual__(): ''' Only work on POSIX-like systems ''' - if not salt.utils.which('showmount'): + if not salt.utils.path.which('showmount'): return (False, 'The nfs3 execution module failed to load: the showmount binary is not in the path.') return True @@ -33,14 +34,18 @@ def list_exports(exports='/etc/exports'): salt '*' nfs.list_exports ''' ret = {} - with salt.utils.fopen(exports, 'r') as efl: + with salt.utils.files.fopen(exports, 'r') as efl: for line in efl.read().splitlines(): if not line: continue if line.startswith('#'): continue comps = line.split() - ret[comps[0]] = [] + + # Handle the case where the same path is given twice + if not comps[0] in ret: + ret[comps[0]] = [] + newshares = [] for perm in comps[1:]: if perm.startswith('/'): @@ -48,7 +53,10 @@ def list_exports(exports='/etc/exports'): continue permcomps = perm.split('(') permcomps[1] = permcomps[1].replace(')', '') - hosts = permcomps[0].split(',') + hosts = permcomps[0] + if type(hosts) is not str: + # Lists, etc would silently mangle /etc/exports + raise TypeError('hosts argument must be a string') options = permcomps[1].split(',') ret[comps[0]].append({'hosts': hosts, 'options': options}) for share in newshares: @@ -72,6 +80,31 @@ def del_export(exports='/etc/exports', path=None): return edict +def add_export(exports='/etc/exports', path=None, hosts=None, options=None): + ''' + Add an export + + CLI Example: + + .. code-block:: bash + + salt '*' nfs3.add_export path='/srv/test' hosts='127.0.0.1' options=['rw'] + ''' + if options is None: + options = [] + if type(hosts) is not str: + # Lists, etc would silently mangle /etc/exports + raise TypeError('hosts argument must be a string') + edict = list_exports(exports) + if path not in edict: + edict[path] = [] + new = {'hosts': hosts, 'options': options} + edict[path].append(new) + _write_exports(exports, edict) + + return new + + def _write_exports(exports, edict): ''' Write an exports file to disk @@ -85,11 +118,35 @@ def _write_exports(exports, edict): /media/storage *(ro,sync,no_subtree_check) /media/data *(ro,sync,no_subtree_check) ''' - with salt.utils.fopen(exports, 'w') as efh: + with salt.utils.files.fopen(exports, 'w') as efh: for export in edict: line = export for perms in edict[export]: - hosts = ','.join(perms['hosts']) + hosts = perms['hosts'] options = ','.join(perms['options']) line += ' {0}({1})'.format(hosts, options) efh.write('{0}\n'.format(line)) + + +def reload_exports(): + ''' + Trigger a reload of the exports file to apply changes + + CLI Example: + + .. code-block:: bash + + salt '*' nfs3.reload_exports + ''' + ret = {} + + command = 'exportfs -r' + + output = __salt__['cmd.run_all'](command) + ret['stdout'] = output['stdout'] + ret['stderr'] = output['stderr'] + # exportfs always returns 0, so retcode is useless + # We will consider it an error if stderr is nonempty + ret['result'] = output['stderr'] == '' + + return ret diff --git a/salt/modules/nftables.py b/salt/modules/nftables.py index 90d4f7d3c63..57ab63ca3fb 100644 --- a/salt/modules/nftables.py +++ b/salt/modules/nftables.py @@ -9,7 +9,8 @@ import logging import re # Import salt libs -import salt.utils +import salt.utils.files +import salt.utils.path from salt.state import STATE_INTERNAL_KEYWORDS as _STATE_INTERNAL_KEYWORDS from salt.exceptions import ( CommandExecutionError @@ -36,7 +37,7 @@ def __virtual__(): ''' Only load the module if nftables is installed ''' - if salt.utils.which('nft'): + if salt.utils.path.which('nft'): return 'nftables' return (False, 'The nftables execution module failed to load: nftables is not installed.') @@ -260,7 +261,7 @@ def get_saved_rules(conf_file=None, family='ipv4'): if _conf() and not conf_file: conf_file = _conf() - with salt.utils.fopen(conf_file) as fp_: + with salt.utils.files.fopen(conf_file) as fp_: lines = fp_.readlines() rules = [] for line in lines: @@ -327,7 +328,7 @@ def save(filename=None, family='ipv4'): rules = rules + '\n' try: - with salt.utils.fopen(filename, 'w+') as _fh: + with salt.utils.files.fopen(filename, 'w+') as _fh: # Write out any changes _fh.writelines(rules) except (IOError, OSError) as exc: diff --git a/salt/modules/nginx.py b/salt/modules/nginx.py index d2841b122de..dbaea65e465 100644 --- a/salt/modules/nginx.py +++ b/salt/modules/nginx.py @@ -8,7 +8,7 @@ from __future__ import absolute_import from salt.ext.six.moves.urllib.request import urlopen as _urlopen # pylint: disable=no-name-in-module,import-error # Import salt libs -import salt.utils +import salt.utils.path import salt.utils.decorators as decorators import re @@ -19,7 +19,7 @@ import re # for nginx over and over and over for each function herein @decorators.memoize def __detect_os(): - return salt.utils.which('nginx') + return salt.utils.path.which('nginx') def __virtual__(): diff --git a/salt/modules/nilrt_ip.py b/salt/modules/nilrt_ip.py index a215fb26f30..a2450b067d1 100644 --- a/salt/modules/nilrt_ip.py +++ b/salt/modules/nilrt_ip.py @@ -15,7 +15,7 @@ import salt.utils.validate.net import salt.exceptions # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six try: import pyconnman HAS_PYCONNMAN = True @@ -183,10 +183,10 @@ def _get_service_info(service): except Exception as exc: log.warning('Unable to get IPv6 {0} for service {1}\n'.format(info, service)) - domains = [] - for x in service_info.get_property('Domains'): - domains.append(str(x)) - data['ipv4']['dns'] = domains + nameservers = [] + for x in service_info.get_property('Nameservers'): + nameservers.append(str(x)) + data['ipv4']['dns'] = nameservers else: data['up'] = False @@ -351,13 +351,13 @@ def set_dhcp_linklocal_all(interface): ipv4['Gateway'] = dbus.String('', variant_level=1) try: service.set_property('IPv4.Configuration', ipv4) - service.set_property('Domains.Configuration', ['']) # reset domains list + service.set_property('Nameservers.Configuration', ['']) # reset nameservers list except Exception as exc: raise salt.exceptions.CommandExecutionError('Couldn\'t set dhcp linklocal for service: {0}\nError: {1}\n'.format(service, exc)) return True -def set_static_all(interface, address, netmask, gateway, domains): +def set_static_all(interface, address, netmask, gateway, nameservers): ''' Configure specified adapter to use ipv4 manual settings @@ -365,7 +365,7 @@ def set_static_all(interface, address, netmask, gateway, domains): :param str address: ipv4 address :param str netmask: ipv4 netmask :param str gateway: ipv4 gateway - :param str domains: list of domains servers separated by spaces + :param str nameservers: list of nameservers servers separated by spaces :return: True if the settings were applied, otherwise an exception will be thrown. :rtype: bool @@ -373,7 +373,7 @@ def set_static_all(interface, address, netmask, gateway, domains): .. code-block:: bash - salt '*' ip.set_static_all interface-label address netmask gateway domains + salt '*' ip.set_static_all interface-label address netmask gateway nameservers ''' service = _interface_to_service(interface) if not service: @@ -381,9 +381,15 @@ def set_static_all(interface, address, netmask, gateway, domains): validate, msg = _validate_ipv4([address, netmask, gateway]) if not validate: raise salt.exceptions.CommandExecutionError(msg) - validate, msg = _space_delimited_list(domains) - if not validate: - raise salt.exceptions.CommandExecutionError(msg) + if nameservers: + validate, msg = _space_delimited_list(nameservers) + if not validate: + raise salt.exceptions.CommandExecutionError(msg) + if not isinstance(nameservers, list): + nameservers = nameservers.split(' ') + service = _interface_to_service(interface) + if not service: + raise salt.exceptions.CommandExecutionError('Invalid interface name: {0}'.format(interface)) service = pyconnman.ConnService(_add_path(service)) ipv4 = service.get_property('IPv4.Configuration') ipv4['Method'] = dbus.String('manual', variant_level=1) @@ -392,10 +398,8 @@ def set_static_all(interface, address, netmask, gateway, domains): ipv4['Gateway'] = dbus.String('{0}'.format(gateway), variant_level=1) try: service.set_property('IPv4.Configuration', ipv4) - if not isinstance(domains, list): - dns = domains.split(' ') - domains = dns - service.set_property('Domains.Configuration', [dbus.String('{0}'.format(d)) for d in domains]) + if nameservers: + service.set_property('Nameservers.Configuration', [dbus.String('{0}'.format(d)) for d in nameservers]) except Exception as exc: raise salt.exceptions.CommandExecutionError('Couldn\'t set manual settings for service: {0}\nError: {1}\n'.format(service, exc)) return True diff --git a/salt/modules/nix.py b/salt/modules/nix.py index 862bd9f3c0a..af2884751f4 100644 --- a/salt/modules/nix.py +++ b/salt/modules/nix.py @@ -23,7 +23,8 @@ import logging import itertools import os -import salt.utils +import salt.utils.itertools +import salt.utils.path from salt.ext.six.moves import zip @@ -35,7 +36,7 @@ def __virtual__(): This only works if we have access to nix-env ''' nixhome = os.path.join(os.path.expanduser('~{0}'.format(__opts__['user'])), '.nix-profile/bin/') - if salt.utils.which(os.path.join(nixhome, 'nix-env')) and salt.utils.which(os.path.join(nixhome, 'nix-collect-garbage')): + if salt.utils.path.which(os.path.join(nixhome, 'nix-env')) and salt.utils.path.which(os.path.join(nixhome, 'nix-collect-garbage')): return True else: return (False, "The `nix` binaries required cannot be found or are not installed. (`nix-store` and `nix-env`)") @@ -222,8 +223,7 @@ def list_pkgs(installed=True, out = _run(cmd) - # TODO: This could be a tad long when using `installed=False`, maybe use salt.utils.itertools.split? - return [s.split() for s in out['stdout'].splitlines()] + return [s.split() for s in salt.utils.itertools.split(out['stdout'], '\n')] def uninstall(*pkgs): diff --git a/salt/modules/npm.py b/salt/modules/npm.py index 969aa2ae906..4a0bbf9c91d 100644 --- a/salt/modules/npm.py +++ b/salt/modules/npm.py @@ -14,6 +14,7 @@ import logging # Import salt libs import salt.utils +import salt.utils.path import salt.modules.cmdmod from salt.exceptions import CommandExecutionError from salt.utils.versions import LooseVersion as _LooseVersion @@ -32,7 +33,7 @@ def __virtual__(): Only work when npm is installed. ''' try: - if salt.utils.which('npm') is not None: + if salt.utils.path.which('npm') is not None: _check_valid_version() return True else: @@ -188,7 +189,7 @@ def _extract_json(npm_output): # macOS with fsevents includes the following line in the return # when a new module is installed which is invalid JSON: # [fsevents] Success: "..." - while lines and lines[0].startswith('[fsevents]'): + while lines and (lines[0].startswith('[fsevents]') or lines[0].startswith('Pass ')): lines = lines[1:] try: return json.loads(''.join(lines)) diff --git a/salt/modules/nspawn.py b/salt/modules/nspawn.py index 9fcca78d420..eb3d94bc1b7 100644 --- a/salt/modules/nspawn.py +++ b/salt/modules/nspawn.py @@ -35,6 +35,8 @@ import tempfile # Import Salt libs import salt.defaults.exitcodes import salt.utils +import salt.utils.args +import salt.utils.path import salt.utils.systemd from salt.exceptions import CommandExecutionError, SaltInvocationError from salt.ext import six @@ -84,7 +86,7 @@ def _ensure_exists(wrapped): raise CommandExecutionError( 'Container \'{0}\' does not exist'.format(name) ) - return wrapped(name, *args, **salt.utils.clean_kwargs(**kwargs)) + return wrapped(name, *args, **salt.utils.args.clean_kwargs(**kwargs)) return check_exists @@ -146,7 +148,7 @@ def _bootstrap_arch(name, **kwargs): ''' Bootstrap an Arch Linux container ''' - if not salt.utils.which('pacstrap'): + if not salt.utils.path.which('pacstrap'): raise CommandExecutionError( 'pacstrap not found, is the arch-install-scripts package ' 'installed?' @@ -774,7 +776,7 @@ def bootstrap_salt(name, pub_key=pub_key, priv_key=priv_key) if needs_install or force_install or unconditional_install: if install: - rstr = __salt__['test.rand_str']() + rstr = __salt__['test.random_hash']() configdir = '/tmp/.c_{0}'.format(rstr) run(name, 'install -m 0700 -d {0}'.format(configdir), @@ -961,10 +963,10 @@ def info(name, **kwargs): salt myminion nspawn.info arch1 salt myminion nspawn.info arch1 force_start=False ''' - kwargs = salt.utils.clean_kwargs(**kwargs) + kwargs = salt.utils.args.clean_kwargs(**kwargs) start_ = kwargs.pop('start', False) if kwargs: - salt.utils.invalid_kwargs(kwargs) + salt.utils.args.invalid_kwargs(kwargs) if not start_: _ensure_running(name) @@ -1341,14 +1343,14 @@ def _pull_image(pull_type, image, name, **kwargs): 'Unsupported image type \'{0}\''.format(pull_type) ) - kwargs = salt.utils.clean_kwargs(**kwargs) + kwargs = salt.utils.args.clean_kwargs(**kwargs) bad_kwargs = dict( - [(x, y) for x, y in six.iteritems(salt.utils.clean_kwargs(**kwargs)) + [(x, y) for x, y in six.iteritems(salt.utils.args.clean_kwargs(**kwargs)) if x not in valid_kwargs] ) if bad_kwargs: - salt.utils.invalid_kwargs(bad_kwargs) + salt.utils.args.invalid_kwargs(bad_kwargs) pull_opts = [] diff --git a/salt/modules/nxos.py b/salt/modules/nxos.py index 45b39bc09ba..5b1d8f14404 100644 --- a/salt/modules/nxos.py +++ b/salt/modules/nxos.py @@ -7,16 +7,18 @@ Execution module for Cisco NX OS Switches Proxy minions For documentation on setting up the nxos proxy minion look in the documentation for :mod:`salt.proxy.nxos `. ''' +# Import Python libs from __future__ import absolute_import -import salt.utils +# Import Salt libs +import salt.utils.platform __proxyenabled__ = ['nxos'] __virtualname__ = 'nxos' def __virtual__(): - if salt.utils.is_proxy(): + if salt.utils.platform.is_proxy(): return __virtualname__ return (False, 'The nxos execution module failed to load: ' 'only available on proxy minions.') diff --git a/salt/modules/openbsd_sysctl.py b/salt/modules/openbsd_sysctl.py index 2aed213e688..4ea7d19e130 100644 --- a/salt/modules/openbsd_sysctl.py +++ b/salt/modules/openbsd_sysctl.py @@ -6,7 +6,7 @@ from __future__ import absolute_import import os # Import salt libs -import salt.utils +import salt.utils.files from salt.exceptions import CommandExecutionError # Define the module's virtual name @@ -98,13 +98,13 @@ def persist(name, value, config='/etc/sysctl.conf'): # create /etc/sysctl.conf if not present if not os.path.isfile(config): try: - with salt.utils.fopen(config, 'w+'): + with salt.utils.files.fopen(config, 'w+'): pass except (IOError, OSError): msg = 'Could not create {0}' raise CommandExecutionError(msg.format(config)) - with salt.utils.fopen(config, 'r') as ifile: + with salt.utils.files.fopen(config, 'r') as ifile: for line in ifile: if not line.startswith('{0}='.format(name)): nlines.append(line) @@ -125,7 +125,7 @@ def persist(name, value, config='/etc/sysctl.conf'): edited = True if not edited: nlines.append('{0}={1}\n'.format(name, value)) - with salt.utils.fopen(config, 'w+') as ofile: + with salt.utils.files.fopen(config, 'w+') as ofile: ofile.writelines(nlines) assign(name, value) diff --git a/salt/modules/openbsdpkg.py b/salt/modules/openbsdpkg.py index f500a343559..238d879ba24 100644 --- a/salt/modules/openbsdpkg.py +++ b/salt/modules/openbsdpkg.py @@ -30,6 +30,7 @@ import logging # Import Salt libs import salt.utils +import salt.utils.versions from salt.exceptions import CommandExecutionError, MinionError log = logging.getLogger(__name__) @@ -123,7 +124,7 @@ def latest_version(*names, **kwargs): continue pkgname += '--{0}'.format(flavor) if flavor else '' cur = pkgs.get(pkgname, '') - if not cur or salt.utils.compare_versions(ver1=cur, + if not cur or salt.utils.versions.compare(ver1=cur, oper='<', ver2=pkgver): ret[pkgname] = pkgver diff --git a/salt/modules/openbsdrcctl.py b/salt/modules/openbsdrcctl.py index 54545ceb823..16c31bdfc7d 100644 --- a/salt/modules/openbsdrcctl.py +++ b/salt/modules/openbsdrcctl.py @@ -8,7 +8,7 @@ from __future__ import absolute_import import os # Import salt libs -import salt.utils +import salt.utils.path import salt.utils.decorators as decorators from salt.exceptions import CommandNotFoundError @@ -35,7 +35,7 @@ def _cmd(): ''' Return the full path to the rcctl(8) command. ''' - rcctl = salt.utils.which('rcctl') + rcctl = salt.utils.path.which('rcctl') if not rcctl: raise CommandNotFoundError return rcctl diff --git a/salt/modules/openbsdservice.py b/salt/modules/openbsdservice.py index 4e09785de4d..27332ee24c9 100644 --- a/salt/modules/openbsdservice.py +++ b/salt/modules/openbsdservice.py @@ -12,16 +12,15 @@ The service module for OpenBSD # Import python libs from __future__ import absolute_import import os -import re import fnmatch import logging # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import map # pylint: disable=import-error,redefined-builtin # Import Salt libs -import salt.utils +import salt.utils.files log = logging.getLogger(__name__) @@ -169,7 +168,7 @@ def _get_rc(): try: # now read the system startup script /etc/rc # to know what are the system enabled daemons - with salt.utils.fopen('/etc/rc', 'r') as handle: + with salt.utils.files.fopen('/etc/rc', 'r') as handle: lines = handle.readlines() except IOError: log.error('Unable to read /etc/rc') diff --git a/salt/modules/openscap.py b/salt/modules/openscap.py index 20615500123..0dfb911f4a5 100644 --- a/salt/modules/openscap.py +++ b/salt/modules/openscap.py @@ -26,7 +26,7 @@ _XCCDF_MAP = { 'cmd_pattern': ( "oscap xccdf eval " "--oval-results --results results.xml --report report.html " - "--profile {0} {1}" + "--profile {0} {1} {2}" ) } } @@ -73,6 +73,7 @@ def xccdf(params): ''' params = shlex.split(params) policy = params[-1] + del params[-1] success = True error = None @@ -89,7 +90,7 @@ def xccdf(params): error = str(err) if success: - cmd = _XCCDF_MAP[action]['cmd_pattern'].format(args.profile, policy) + cmd = _XCCDF_MAP[action]['cmd_pattern'].format(args.profile, " ".join(argv), policy) tempdir = tempfile.mkdtemp() proc = Popen( shlex.split(cmd), stdout=PIPE, stderr=PIPE, cwd=tempdir) diff --git a/salt/modules/openstack_config.py b/salt/modules/openstack_config.py index 97dbc5ce91f..1033f4a68da 100644 --- a/salt/modules/openstack_config.py +++ b/salt/modules/openstack_config.py @@ -13,7 +13,7 @@ from __future__ import absolute_import import salt.utils import salt.exceptions -from salt.utils.decorators import which as _which +import salt.utils.decorators.path import shlex try: @@ -45,7 +45,7 @@ def _fallback(*args, **kw): return 'The "openstack-config" command needs to be installed for this function to work. Typically this is included in the "openstack-utils" package.' -@_which('openstack-config') +@salt.utils.decorators.path.which('openstack-config') def set_(filename, section, parameter, value): ''' Set a value in an OpenStack configuration file. @@ -87,7 +87,7 @@ def set_(filename, section, parameter, value): raise salt.exceptions.CommandExecutionError(result['stderr']) -@_which('openstack-config') +@salt.utils.decorators.path.which('openstack-config') def get(filename, section, parameter): ''' Get a value from an OpenStack configuration file. @@ -126,7 +126,7 @@ def get(filename, section, parameter): raise salt.exceptions.CommandExecutionError(result['stderr']) -@_which('openstack-config') +@salt.utils.decorators.path.which('openstack-config') def delete(filename, section, parameter): ''' Delete a value from an OpenStack configuration file. diff --git a/salt/modules/openstack_mng.py b/salt/modules/openstack_mng.py index 2c046546bee..05e35cf2b60 100644 --- a/salt/modules/openstack_mng.py +++ b/salt/modules/openstack_mng.py @@ -13,7 +13,7 @@ import logging import os.path # Import salt libs -import salt.utils +import salt.utils.files log = logging.getLogger(__name__) @@ -80,7 +80,7 @@ def restart_service(service_name, minimum_running_time=None): services = __salt__['cmd.run'](['/usr/bin/openstack-service', 'list', service_name]).split('\n') for service in services: service_info = __salt__['service.show'](service) - with salt.utils.fopen('/proc/uptime') as rfh: + with salt.utils.files.fopen('/proc/uptime') as rfh: boot_time = float(rfh.read().split(' ')[0]) expr_time = int(service_info.get('ExecMainStartTimestampMonotonic', 0)) / 1000000 < boot_time - minimum_running_time diff --git a/salt/modules/openvswitch.py b/salt/modules/openvswitch.py index 0113859a68c..bd287b89837 100644 --- a/salt/modules/openvswitch.py +++ b/salt/modules/openvswitch.py @@ -12,7 +12,7 @@ from __future__ import absolute_import import logging # Import salt libs -import salt.utils +import salt.utils.path log = logging.getLogger(__name__) @@ -21,7 +21,7 @@ def __virtual__(): ''' Only load the module if Open vSwitch is installed ''' - if salt.utils.which('ovs-vsctl'): + if salt.utils.path.which('ovs-vsctl'): return 'openvswitch' return False diff --git a/salt/modules/opkg.py b/salt/modules/opkg.py index 03f847e38b4..36284dfab8b 100644 --- a/salt/modules/opkg.py +++ b/salt/modules/opkg.py @@ -24,15 +24,18 @@ import re import logging # Import salt libs -import salt.utils -import salt.utils.pkg +import salt.utils # Can be removed when is_true, compare_dicts are moved +import salt.utils.args +import salt.utils.files import salt.utils.itertools -from salt.utils.versions import LooseVersion as _LooseVersion +import salt.utils.path +import salt.utils.pkg +import salt.utils.versions from salt.exceptions import ( CommandExecutionError, MinionError, SaltInvocationError ) # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import shlex_quote as _cmd_quote # pylint: disable=import-error REPO_REGEXP = r'^#?\s*(src|src/gz)\s+([^\s<>]+|"[^<>]+")\s+[^\s<>]+' @@ -324,13 +327,13 @@ def install(name=None, else: pkgstr = '{0}={1}'.format(pkgname, version_num) cver = old.get(pkgname, '') - if reinstall and cver and salt.utils.compare_versions( + if reinstall and cver and salt.utils.versions.compare( ver1=version_num, oper='==', ver2=cver, cmp_func=version_cmp): to_reinstall.append(pkgstr) - elif not cver or salt.utils.compare_versions( + elif not cver or salt.utils.versions.compare( ver1=version_num, oper='>=', ver2=cver, @@ -928,7 +931,7 @@ def info_installed(*names, **kwargs): attr = kwargs.pop('attr', None) if attr is None: filter_attrs = None - elif isinstance(attr, str): + elif isinstance(attr, six.string_types): filter_attrs = set(attr.split(',')) else: filter_attrs = set(attr) @@ -1011,9 +1014,10 @@ def version_cmp(pkg1, pkg2, ignore_epoch=False): output_loglevel='trace', python_shell=False) opkg_version = output.split(' ')[2].strip() - if _LooseVersion(opkg_version) >= _LooseVersion('0.3.4'): + if salt.utils.versions.LooseVersion(opkg_version) >= \ + salt.utils.versions.LooseVersion('0.3.4'): cmd_compare = ['opkg', 'compare-versions'] - elif salt.utils.which('opkg-compare-versions'): + elif salt.utils.path.which('opkg-compare-versions'): cmd_compare = ['opkg-compare-versions'] else: log.warning('Unable to find a compare-versions utility installed. Either upgrade opkg to ' @@ -1048,7 +1052,7 @@ def list_repos(): regex = re.compile(REPO_REGEXP) for filename in os.listdir(OPKG_CONFDIR): if filename.endswith(".conf"): - with salt.utils.fopen(os.path.join(OPKG_CONFDIR, filename)) as conf_file: + with salt.utils.files.fopen(os.path.join(OPKG_CONFDIR, filename)) as conf_file: for line in conf_file: if regex.search(line): repo = {} @@ -1057,7 +1061,7 @@ def list_repos(): line = line[1:] else: repo['enabled'] = True - cols = salt.utils.shlex_split(line.strip()) + cols = salt.utils.args.shlex_split(line.strip()) if cols[0] in 'src': repo['compressed'] = False else: @@ -1095,17 +1099,17 @@ def _del_repo_from_file(alias, filepath): ''' Remove a repo from filepath ''' - with salt.utils.fopen(filepath) as fhandle: + with salt.utils.files.fopen(filepath) as fhandle: output = [] regex = re.compile(REPO_REGEXP) for line in fhandle: if regex.search(line): if line.startswith('#'): line = line[1:] - cols = salt.utils.shlex_split(line.strip()) + cols = salt.utils.args.shlex_split(line.strip()) if alias != cols[1]: output.append(line) - with salt.utils.fopen(filepath, 'w') as fhandle: + with salt.utils.files.fopen(filepath, 'w') as fhandle: fhandle.writelines(output) @@ -1122,7 +1126,7 @@ def _add_new_repo(alias, uri, compressed, enabled=True): repostr += uri + '\n' conffile = os.path.join(OPKG_CONFDIR, alias + '.conf') - with salt.utils.fopen(conffile, 'a') as fhandle: + with salt.utils.files.fopen(conffile, 'a') as fhandle: fhandle.write(repostr) @@ -1130,15 +1134,15 @@ def _mod_repo_in_file(alias, repostr, filepath): ''' Replace a repo entry in filepath with repostr ''' - with salt.utils.fopen(filepath) as fhandle: + with salt.utils.files.fopen(filepath) as fhandle: output = [] for line in fhandle: - cols = salt.utils.shlex_split(line.strip()) + cols = salt.utils.args.shlex_split(line.strip()) if alias not in cols: output.append(line) else: output.append(repostr + '\n') - with salt.utils.fopen(filepath, 'w') as fhandle: + with salt.utils.files.fopen(filepath, 'w') as fhandle: fhandle.writelines(output) diff --git a/salt/modules/oracle.py b/salt/modules/oracle.py index 1069061c9b6..8e527fc1767 100644 --- a/salt/modules/oracle.py +++ b/salt/modules/oracle.py @@ -32,7 +32,7 @@ from __future__ import absolute_import import os import logging from salt.utils.decorators import depends -import salt.ext.six as six +from salt.ext import six log = logging.getLogger(__name__) diff --git a/salt/modules/osquery.py b/salt/modules/osquery.py index 137125f070f..403d70bb6ff 100644 --- a/salt/modules/osquery.py +++ b/salt/modules/osquery.py @@ -10,7 +10,8 @@ from __future__ import absolute_import import json # Import Salt libs -import salt.utils +import salt.utils.path +import salt.utils.platform import logging log = logging.getLogger(__name__) @@ -26,7 +27,7 @@ __virtualname__ = 'osquery' def __virtual__(): - if salt.utils.which('osqueryi'): + if salt.utils.path.which('osqueryi'): return __virtualname__ return (False, 'The osquery execution module cannot be loaded: ' 'osqueryi binary is not in the path.') @@ -36,7 +37,7 @@ def _table_attrs(table): ''' Helper function to find valid table attributes ''' - cmd = 'osqueryi --json "pragma table_info({0})"'.format(table) + cmd = ['osqueryi'] + ['--json'] + ['pragma table_info{0}'.format(table)] res = __salt__['cmd.run_all'](cmd) if res['retcode'] == 0: attrs = [] @@ -55,13 +56,13 @@ def _osquery(sql, format='json'): 'result': True, } - cmd = 'osqueryi --json "{0}"'.format(sql) + cmd = ['osqueryi'] + ['--json'] + [sql] res = __salt__['cmd.run_all'](cmd) - if res['retcode'] == 0: - ret['data'] = json.loads(res['stdout']) - else: + if res['stderr']: ret['result'] = False ret['error'] = res['stderr'] + else: + ret['data'] = json.loads(res['stdout']) return ret @@ -657,7 +658,7 @@ def alf(attrs=None, where=None): salt '*' osquery.alf ''' - if salt.utils.is_darwin(): + if salt.utils.platform.is_darwin(): return _osquery_cmd(table='alf', attrs=attrs, where=where) return {'result': False, 'comment': 'Only available on macOS systems.'} @@ -672,7 +673,7 @@ def alf_exceptions(attrs=None, where=None): salt '*' osquery.alf_exceptions ''' - if salt.utils.is_darwin(): + if salt.utils.platform.is_darwin(): return _osquery_cmd(table='alf_exceptions', attrs=attrs, where=where) return {'result': False, 'comment': 'Only available on macOS systems.'} @@ -687,7 +688,7 @@ def alf_explicit_auths(attrs=None, where=None): salt '*' osquery.alf_explicit_auths ''' - if salt.utils.is_darwin(): + if salt.utils.platform.is_darwin(): return _osquery_cmd(table='alf_explicit_auths', attrs=attrs, where=where) return {'result': False, 'comment': 'Only available on macOS systems.'} @@ -702,7 +703,7 @@ def alf_services(attrs=None, where=None): salt '*' osquery.alf_services ''' - if salt.utils.is_darwin(): + if salt.utils.platform.is_darwin(): return _osquery_cmd(table='alf_services', attrs=attrs, where=where) return {'result': False, 'comment': 'Only available on macOS systems.'} @@ -717,7 +718,7 @@ def apps(attrs=None, where=None): salt '*' osquery.apps ''' - if salt.utils.is_darwin(): + if salt.utils.platform.is_darwin(): return _osquery_cmd(table='apps', attrs=attrs, where=where) return {'result': False, 'comment': 'Only available on macOS systems.'} @@ -732,7 +733,7 @@ def certificates(attrs=None, where=None): salt '*' osquery.certificates ''' - if salt.utils.is_darwin(): + if salt.utils.platform.is_darwin(): return _osquery_cmd(table='certificates', attrs=attrs, where=where) return {'result': False, 'comment': 'Only available on macOS systems.'} @@ -747,7 +748,7 @@ def chrome_extensions(attrs=None, where=None): salt '*' osquery.chrome_extensions ''' - if salt.utils.is_darwin(): + if salt.utils.platform.is_darwin(): return _osquery_cmd(table='chrome_extensions', attrs=attrs, where=where) return {'result': False, 'comment': 'Only available on macOS systems.'} @@ -762,7 +763,7 @@ def firefox_addons(attrs=None, where=None): salt '*' osquery.firefox_addons ''' - if salt.utils.is_darwin(): + if salt.utils.platform.is_darwin(): return _osquery_cmd(table='firefox_addons', attrs=attrs, where=where) return {'result': False, 'comment': 'Only available on macOS systems.'} @@ -777,7 +778,7 @@ def homebrew_packages(attrs=None, where=None): salt '*' osquery.homebrew_packages ''' - if salt.utils.is_darwin(): + if salt.utils.platform.is_darwin(): return _osquery_cmd(table='homebrew_packages', attrs=attrs, where=where) return {'result': False, 'comment': 'Only available on macOS systems.'} @@ -792,7 +793,7 @@ def iokit_devicetree(attrs=None, where=None): salt '*' osquery.iokit_devicetree ''' - if salt.utils.is_darwin(): + if salt.utils.platform.is_darwin(): return _osquery_cmd(table='iokit_devicetree', attrs=attrs, where=where) return {'result': False, 'comment': 'Only available on macOS systems.'} @@ -807,7 +808,7 @@ def iokit_registry(attrs=None, where=None): salt '*' osquery.iokit_registry ''' - if salt.utils.is_darwin(): + if salt.utils.platform.is_darwin(): return _osquery_cmd(table='iokit_registry', attrs=attrs, where=where) return {'result': False, 'comment': 'Only available on macOS systems.'} @@ -822,7 +823,7 @@ def kernel_extensions(attrs=None, where=None): salt '*' osquery.kernel_extensions ''' - if salt.utils.is_darwin(): + if salt.utils.platform.is_darwin(): return _osquery_cmd(table='kernel_extensions', attrs=attrs, where=where) return {'result': False, 'comment': 'Only available on macOS systems.'} @@ -837,7 +838,7 @@ def keychain_items(attrs=None, where=None): salt '*' osquery.keychain_items ''' - if salt.utils.is_darwin(): + if salt.utils.platform.is_darwin(): return _osquery_cmd(table='keychain_items', attrs=attrs, where=where) return {'result': False, 'comment': 'Only available on macOS systems.'} @@ -852,7 +853,7 @@ def launchd(attrs=None, where=None): salt '*' osquery.launchd ''' - if salt.utils.is_darwin(): + if salt.utils.platform.is_darwin(): return _osquery_cmd(table='launchd', attrs=attrs, where=where) return {'result': False, 'comment': 'Only available on macOS systems.'} @@ -867,7 +868,7 @@ def nfs_shares(attrs=None, where=None): salt '*' osquery.nfs_shares ''' - if salt.utils.is_darwin(): + if salt.utils.platform.is_darwin(): return _osquery_cmd(table='nfs_shares', attrs=attrs, where=where) return {'result': False, 'comment': 'Only available on macOS systems.'} @@ -882,7 +883,7 @@ def nvram(attrs=None, where=None): salt '*' osquery.nvram ''' - if salt.utils.is_darwin(): + if salt.utils.platform.is_darwin(): return _osquery_cmd(table='nvram', attrs=attrs, where=where) return {'result': False, 'comment': 'Only available on macOS systems.'} @@ -897,7 +898,7 @@ def preferences(attrs=None, where=None): salt '*' osquery.preferences ''' - if salt.utils.is_darwin(): + if salt.utils.platform.is_darwin(): return _osquery_cmd(table='preferences', attrs=attrs, where=where) return {'result': False, 'comment': 'Only available on macOS systems.'} @@ -912,7 +913,7 @@ def quarantine(attrs=None, where=None): salt '*' osquery.quarantine ''' - if salt.utils.is_darwin(): + if salt.utils.platform.is_darwin(): return _osquery_cmd(table='quarantine', attrs=attrs, where=where) return {'result': False, 'comment': 'Only available on macOS systems.'} @@ -927,7 +928,7 @@ def safari_extensions(attrs=None, where=None): salt '*' osquery.safari_extensions ''' - if salt.utils.is_darwin(): + if salt.utils.platform.is_darwin(): return _osquery_cmd(table='safari_extensions', attrs=attrs, where=where) return {'result': False, 'comment': 'Only available on macOS systems.'} @@ -942,7 +943,7 @@ def startup_items(attrs=None, where=None): salt '*' osquery.startup_items ''' - if salt.utils.is_darwin(): + if salt.utils.platform.is_darwin(): return _osquery_cmd(table='startup_items', attrs=attrs, where=where) return {'result': False, 'comment': 'Only available on macOS systems.'} @@ -957,7 +958,7 @@ def xattr_where_from(attrs=None, where=None): salt '*' osquery.xattr_where_from ''' - if salt.utils.is_darwin(): + if salt.utils.platform.is_darwin(): return _osquery_cmd(table='xattr_where_from', attrs=attrs, where=where) return {'result': False, 'comment': 'Only available on macOS systems.'} @@ -972,7 +973,7 @@ def xprotect_entries(attrs=None, where=None): salt '*' osquery.xprotect_entries ''' - if salt.utils.is_darwin(): + if salt.utils.platform.is_darwin(): return _osquery_cmd(table='xprotect_entries', attrs=attrs, where=where) return {'result': False, 'comment': 'Only available on macOS systems.'} @@ -987,7 +988,7 @@ def xprotect_reports(attrs=None, where=None): salt '*' osquery.xprotect_reports ''' - if salt.utils.is_darwin(): + if salt.utils.platform.is_darwin(): return _osquery_cmd(table='xprotect_reports', attrs=attrs, where=where) return {'result': False, 'comment': 'Only available on macOS systems.'} diff --git a/salt/modules/out.py b/salt/modules/out.py new file mode 100644 index 00000000000..332ed47b309 --- /dev/null +++ b/salt/modules/out.py @@ -0,0 +1,115 @@ +# -*- coding: utf-8 -*- +''' +Output Module +============= + +.. versionadded:: Oxygen + +Execution module that processes JSON serializable data +and returns string having the format as processed by the outputters. + +Although this does not bring much value on the CLI, it turns very handy +in applications that require human readable data rather than Python objects. + +For example, inside a Jinja template: + +.. code-block:: jinja + + {{ salt.out.string_format(complex_object, out='highstate') }} +''' +from __future__ import absolute_import + +# Import python libs +import logging +log = logging.getLogger(__name__) + +# Import salt modules +import salt.output + +__virtualname__ = 'out' +__proxyenabled__ = ['*'] + + +def __virtual__(): + return __virtualname__ + + +def out_format(data, out='nested', opts=None, **kwargs): + ''' + Return the formatted outputter string for the Python object. + + data + The JSON serializable object. + + out: ``nested`` + The name of the output to use to transform the data. Default: ``nested``. + + opts + Dictionary of configuration options. Default: ``__opts__``. + + **kwargs + Arguments to sent to the outputter module. + + CLI Example: + + ..code-block:: bash + + salt '*' out.out_format "{'key': 'value'}" + ''' + if not opts: + opts = __opts__ + return salt.output.out_format(data, out, opts=opts, **kwargs) + + +def string_format(data, out='nested', opts=None, **kwargs): + ''' + Return the outputter formatted string, removing the ANSI escape sequences. + + data + The JSON serializable object. + + out: ``nested`` + The name of the output to use to transform the data. Default: ``nested``. + + opts + Dictionary of configuration options. Default: ``__opts__``. + + **kwargs + Arguments to sent to the outputter module. + + CLI Example: + + ..code-block:: bash + + salt '*' out.string_format "{'key': 'value'}" out=table + ''' + if not opts: + opts = __opts__ + return salt.output.string_format(data, out, opts=opts, **kwargs) + + +def html_format(data, out='nested', opts=None, **kwargs): + ''' + Return the formatted string as HTML. + + data + The JSON serializable object. + + out: ``nested`` + The name of the output to use to transform the data. Default: ``nested``. + + opts + Dictionary of configuration options. Default: ``__opts__``. + + **kwargs + Arguments to sent to the outputter module. + + CLI Example: + + ..code-block:: bash + + salt '*' out.html_format "{'key': 'value'}" out=yaml + ''' + if not opts: + opts = __opts__ + return salt.output.html_format(data, out, opts=opts, **kwargs) diff --git a/salt/modules/pacman.py b/salt/modules/pacman.py index 7984f808c31..ab35b9d85d2 100644 --- a/salt/modules/pacman.py +++ b/salt/modules/pacman.py @@ -19,6 +19,7 @@ import os.path # Import salt libs import salt.utils +import salt.utils.args import salt.utils.pkg import salt.utils.itertools import salt.utils.systemd @@ -26,7 +27,7 @@ from salt.exceptions import CommandExecutionError, MinionError from salt.utils.versions import LooseVersion as _LooseVersion # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six log = logging.getLogger(__name__) @@ -538,6 +539,7 @@ def install(name=None, cmd.append('pacman') errors = [] + targets = [] if pkg_type == 'file': cmd.extend(['-U', '--noprogressbar', '--noconfirm']) cmd.extend(pkg_params) @@ -548,7 +550,6 @@ def install(name=None, if sysupgrade: cmd.append('-u') cmd.extend(['--noprogressbar', '--noconfirm', '--needed']) - targets = [] wildcards = [] for param, version_num in six.iteritems(pkg_params): if version_num is None: @@ -569,33 +570,33 @@ def install(name=None, continue targets.append('{0}{1}{2}'.format(param, prefix, verstr)) - if wildcards: - # Resolve wildcard matches - _available = list_repo_pkgs(*[x[0] for x in wildcards], refresh=refresh) - for pkgname, verstr in wildcards: - candidates = _available.get(pkgname, []) - match = salt.utils.fnmatch_multiple(candidates, verstr) - if match is not None: - targets.append('='.join((pkgname, match))) - else: - errors.append( - 'No version matching \'{0}\' found for package \'{1}\' ' - '(available: {2})'.format( - verstr, - pkgname, - ', '.join(candidates) if candidates else 'none' + if wildcards: + # Resolve wildcard matches + _available = list_repo_pkgs(*[x[0] for x in wildcards], refresh=refresh) + for pkgname, verstr in wildcards: + candidates = _available.get(pkgname, []) + match = salt.utils.fnmatch_multiple(candidates, verstr) + if match is not None: + targets.append('='.join((pkgname, match))) + else: + errors.append( + 'No version matching \'{0}\' found for package \'{1}\' ' + '(available: {2})'.format( + verstr, + pkgname, + ', '.join(candidates) if candidates else 'none' + ) ) - ) - if refresh: - try: - # Prevent a second refresh when we run the install command - cmd.remove('-y') - except ValueError: - # Shouldn't happen since we only add -y when refresh is True, - # but just in case that code above is inadvertently changed, - # don't let this result in a traceback. - pass + if refresh: + try: + # Prevent a second refresh when we run the install command + cmd.remove('-y') + except ValueError: + # Shouldn't happen since we only add -y when refresh is True, + # but just in case that code above is inadvertently changed, + # don't let this result in a traceback. + pass if not errors: cmd.extend(targets) @@ -991,12 +992,12 @@ def list_repo_pkgs(*args, **kwargs): salt '*' pkg.list_repo_pkgs 'samba4*' fromrepo=base,updates salt '*' pkg.list_repo_pkgs 'python2-*' byrepo=True ''' - kwargs = salt.utils.clean_kwargs(**kwargs) + kwargs = salt.utils.args.clean_kwargs(**kwargs) fromrepo = kwargs.pop('fromrepo', '') or '' byrepo = kwargs.pop('byrepo', False) refresh = kwargs.pop('refresh', False) if kwargs: - salt.utils.invalid_kwargs(kwargs) + salt.utils.args.invalid_kwargs(kwargs) if fromrepo: try: diff --git a/salt/modules/pam.py b/salt/modules/pam.py index bcdda816d68..94355d1c45f 100644 --- a/salt/modules/pam.py +++ b/salt/modules/pam.py @@ -9,7 +9,7 @@ import os import logging # Import salt libs -import salt.utils +import salt.utils.files log = logging.getLogger(__name__) @@ -31,7 +31,7 @@ def _parse(contents=None, file_name=None): if contents: pass elif file_name and os.path.exists(file_name): - with salt.utils.fopen(file_name, 'r') as ifile: + with salt.utils.files.fopen(file_name, 'r') as ifile: contents = ifile.read() else: log.error('File "{0}" does not exist'.format(file_name)) diff --git a/salt/modules/panos.py b/salt/modules/panos.py new file mode 100644 index 00000000000..aecf93fffed --- /dev/null +++ b/salt/modules/panos.py @@ -0,0 +1,1985 @@ +# -*- coding: utf-8 -*- +''' +Module to provide Palo Alto compatibility to Salt. + +:codeauthor: :email:`Spencer Ervin ` +:maturity: new +:depends: none +:platform: unix + + +Configuration +============= +This module accepts connection configuration details either as +parameters, or as configuration settings in pillar as a Salt proxy. +Options passed into opts will be ignored if options are passed into pillar. + +.. seealso:: + :prox:`Palo Alto Proxy Module ` + +About +===== +This execution module was designed to handle connections to a Palo Alto based +firewall. This module adds support to send connections directly to the device +through the XML API or through a brokered connection to Panorama. + +''' + +# Import Python Libs +from __future__ import absolute_import +import logging +import time + +# Import Salt Libs +import salt.utils.platform +import salt.proxy.panos + +log = logging.getLogger(__name__) + +__virtualname__ = 'panos' + + +def __virtual__(): + ''' + Will load for the panos proxy minions. + ''' + try: + if salt.utils.platform.is_proxy() and \ + __opts__['proxy']['proxytype'] == 'panos': + return __virtualname__ + except KeyError: + pass + + return False, 'The panos execution module can only be loaded for panos proxy minions.' + + +def _get_job_results(query=None): + ''' + Executes a query that requires a job for completion. This funciton will wait for the job to complete + and return the results. + ''' + if not query: + raise salt.exception.CommandExecutionError("Query parameters cannot be empty.") + + response = __proxy__['panos.call'](query) + + # If the response contains a job, we will wait for the results + if 'job' in response: + jid = response['job'] + + while get_job(jid)['job']['status'] != 'FIN': + time.sleep(5) + + return get_job(jid) + else: + return response + + +def add_config_lock(): + ''' + Prevent other users from changing configuration until the lock is released. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.add_config_lock + + ''' + query = {'type': 'op', 'cmd': ''} + + return __proxy__['panos.call'](query) + + +def check_antivirus(): + ''' + Get anti-virus information from PaloAlto Networks server + + CLI Example: + + .. code-block:: bash + + salt '*' panos.check_antivirus + + ''' + query = {'type': 'op', 'cmd': ''} + + return __proxy__['panos.call'](query) + + +def check_software(): + ''' + Get software information from PaloAlto Networks server. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.check_software + + ''' + query = {'type': 'op', 'cmd': ''} + + return __proxy__['panos.call'](query) + + +def clear_commit_tasks(): + ''' + Clear all commit tasks. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.clear_commit_tasks + + ''' + query = {'type': 'op', 'cmd': ''} + + return __proxy__['panos.call'](query) + + +def commit(): + ''' + Commits the candidate configuration to the running configuration. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.commit + + ''' + query = {'type': 'commit', 'cmd': ''} + + return _get_job_results(query) + + +def deactivate_license(key_name=None): + ''' + Deactivates an installed license. + Required version 7.0.0 or greater. + + key_name(str): The file name of the license key installed. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.deactivate_license key_name=License_File_Name.key + + ''' + + _required_version = '7.0.0' + if not __proxy__['panos.is_required_version'](_required_version): + return False, 'The panos device requires version {0} or greater for this command.'.format(_required_version) + + if not key_name: + return False, 'You must specify a key_name.' + else: + query = {'type': 'op', 'cmd': '{0}' + ''.format(key_name)} + + return __proxy__['panos.call'](query) + + +def delete_license(key_name=None): + ''' + Remove license keys on disk. + + key_name(str): The file name of the license key to be deleted. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.delete_license key_name=License_File_Name.key + + ''' + + if not key_name: + return False, 'You must specify a key_name.' + else: + query = {'type': 'op', 'cmd': '{0}'.format(key_name)} + + return __proxy__['panos.call'](query) + + +def download_antivirus(): + ''' + Download the most recent anti-virus package. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.download_antivirus + + ''' + query = {'type': 'op', + 'cmd': '' + ''} + + return _get_job_results(query) + + +def download_software_file(filename=None, synch=False): + ''' + Download software packages by filename. + + Args: + filename(str): The filename of the PANOS file to download. + + synch (bool): If true then the file will synch to the peer unit. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.download_software_file PanOS_5000-8.0.0 + salt '*' panos.download_software_file PanOS_5000-8.0.0 True + + ''' + if not filename: + raise salt.exception.CommandExecutionError("Filename option must not be none.") + + if not isinstance(synch, bool): + raise salt.exception.CommandExecutionError("Synch option must be boolean..") + + if synch is True: + query = {'type': 'op', + 'cmd': '' + '{0}'.format(filename)} + else: + query = {'type': 'op', + 'cmd': 'yes' + '{0}'.format(filename)} + + return _get_job_results(query) + + +def download_software_version(version=None, synch=False): + ''' + Download software packages by version number. + + Args: + version(str): The version of the PANOS file to download. + + synch (bool): If true then the file will synch to the peer unit. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.download_software_version 8.0.0 + salt '*' panos.download_software_version 8.0.0 True + + ''' + if not version: + raise salt.exception.CommandExecutionError("Version option must not be none.") + + if not isinstance(synch, bool): + raise salt.exception.CommandExecutionError("Synch option must be boolean..") + + if synch is True: + query = {'type': 'op', + 'cmd': '' + '{0}'.format(version)} + else: + query = {'type': 'op', + 'cmd': 'yes' + '{0}'.format(version)} + + return _get_job_results(query) + + +def fetch_license(auth_code=None): + ''' + Get new license(s) using from the Palo Alto Network Server. + + auth_code + The license authorization code. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.fetch_license + salt '*' panos.fetch_license auth_code=foobar + + ''' + if not auth_code: + query = {'type': 'op', 'cmd': ''} + else: + query = {'type': 'op', 'cmd': '{0}' + ''.format(auth_code)} + + return __proxy__['panos.call'](query) + + +def get_admins_active(): + ''' + Show active administrators. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.get_admins_active + + ''' + query = {'type': 'op', 'cmd': ''} + + return __proxy__['panos.call'](query) + + +def get_admins_all(): + ''' + Show all administrators. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.get_admins_all + + ''' + query = {'type': 'op', 'cmd': ''} + + return __proxy__['panos.call'](query) + + +def get_antivirus_info(): + ''' + Show information about available anti-virus packages. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.get_antivirus_info + + ''' + query = {'type': 'op', 'cmd': ''} + + return __proxy__['panos.call'](query) + + +def get_arp(): + ''' + Show ARP information. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.get_arp + + ''' + query = {'type': 'op', 'cmd': ''} + + return __proxy__['panos.call'](query) + + +def get_cli_idle_timeout(): + ''' + Show timeout information for this administrative session. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.get_cli_idle_timeout + + ''' + query = {'type': 'op', 'cmd': ''} + + return __proxy__['panos.call'](query) + + +def get_cli_permissions(): + ''' + Show cli administrative permissions. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.get_cli_permissions + + ''' + query = {'type': 'op', 'cmd': ''} + + return __proxy__['panos.call'](query) + + +def get_disk_usage(): + ''' + Report filesystem disk space usage. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.get_disk_usage + + ''' + query = {'type': 'op', 'cmd': ''} + + return __proxy__['panos.call'](query) + + +def get_dns_server_config(): + ''' + Get the DNS server configuration from the candidate configuration. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.get_dns_server_config + + ''' + query = {'type': 'config', + 'action': 'get', + 'xpath': '/config/devices/entry[@name=\'localhost.localdomain\']/deviceconfig/system/dns-setting/servers'} + + return __proxy__['panos.call'](query) + + +def get_domain_config(): + ''' + Get the domain name configuration from the candidate configuration. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.get_domain_config + + ''' + query = {'type': 'config', + 'action': 'get', + 'xpath': '/config/devices/entry[@name=\'localhost.localdomain\']/deviceconfig/system/domain'} + + return __proxy__['panos.call'](query) + + +def get_fqdn_cache(): + ''' + Print FQDNs used in rules and their IPs. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.get_fqdn_cache + + ''' + query = {'type': 'op', 'cmd': ''} + + return __proxy__['panos.call'](query) + + +def get_ha_config(): + ''' + Get the high availability configuration. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.get_ha_config + + ''' + query = {'type': 'config', + 'action': 'get', + 'xpath': '/config/devices/entry[@name=\'localhost.localdomain\']/deviceconfig/high-availability'} + + return __proxy__['panos.call'](query) + + +def get_ha_link(): + ''' + Show high-availability link-monitoring state. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.get_ha_link + + ''' + query = {'type': 'op', + 'cmd': ''} + + return __proxy__['panos.call'](query) + + +def get_ha_path(): + ''' + Show high-availability path-monitoring state. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.get_ha_path + + ''' + query = {'type': 'op', + 'cmd': ''} + + return __proxy__['panos.call'](query) + + +def get_ha_state(): + ''' + Show high-availability state information. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.get_ha_state + + ''' + + query = {'type': 'op', + 'cmd': ''} + + return __proxy__['panos.call'](query) + + +def get_ha_transitions(): + ''' + Show high-availability transition statistic information. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.get_ha_transitions + + ''' + + query = {'type': 'op', + 'cmd': ''} + + return __proxy__['panos.call'](query) + + +def get_hostname(): + ''' + Get the hostname of the device. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.get_hostname + + ''' + query = {'type': 'config', + 'action': 'get', + 'xpath': '/config/devices/entry[@name=\'localhost.localdomain\']/deviceconfig/system/hostname'} + + return __proxy__['panos.call'](query)['hostname'] + + +def get_interface_counters(name='all'): + ''' + Get the counter statistics for interfaces. + + Args: + name (str): The name of the interface to view. By default, all interface statistics are viewed. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.get_interface_counters + salt '*' panos.get_interface_counters ethernet1/1 + + ''' + query = {'type': 'op', + 'cmd': '{0}'.format(name)} + + return __proxy__['panos.call'](query) + + +def get_interfaces(name='all'): + ''' + Show interface information. + + Args: + name (str): The name of the interface to view. By default, all interface statistics are viewed. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.get_interfaces + salt '*' panos.get_interfaces ethernet1/1 + + ''' + query = {'type': 'op', + 'cmd': '{0}'.format(name)} + + return __proxy__['panos.call'](query) + + +def get_job(jid=None): + ''' + List all a single job by ID. + + jid + The ID of the job to retrieve. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.get_job jid=15 + + ''' + if not jid: + raise salt.exception.CommandExecutionError("ID option must not be none.") + + query = {'type': 'op', 'cmd': '{0}'.format(jid)} + + return __proxy__['panos.call'](query) + + +def get_jobs(state='all'): + ''' + List all jobs on the device. + + state + The state of the jobs to display. Valid options are all, pending, or processed. Pending jobs are jobs + that are currently in a running or waiting state. Processed jobs are jobs that have completed + execution. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.get_jobs + salt '*' panos.get_jobs state=pending + + ''' + if state.lower() == 'all': + query = {'type': 'op', 'cmd': ''} + elif state.lower() == 'pending': + query = {'type': 'op', 'cmd': ''} + elif state.lower() == 'processed': + query = {'type': 'op', 'cmd': ''} + else: + raise salt.exception.CommandExecutionError("The state parameter must be all, pending, or processed.") + + return __proxy__['panos.call'](query) + + +def get_lacp(): + ''' + Show LACP state. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.get_lacp + + ''' + query = {'type': 'op', 'cmd': 'all'} + + return __proxy__['panos.call'](query) + + +def get_license_info(): + ''' + Show information about owned license(s). + + CLI Example: + + .. code-block:: bash + + salt '*' panos.get_license_info + + ''' + query = {'type': 'op', 'cmd': ''} + + return __proxy__['panos.call'](query) + + +def get_license_tokens(): + ''' + Show license token files for manual license deactivation. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.get_license_tokens + + ''' + query = {'type': 'op', 'cmd': ''} + + return __proxy__['panos.call'](query) + + +def get_lldp_config(): + ''' + Show lldp config for interfaces. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.get_lldp_config + + ''' + query = {'type': 'op', 'cmd': 'all'} + + return __proxy__['panos.call'](query) + + +def get_lldp_counters(): + ''' + Show lldp counters for interfaces. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.get_lldp_counters + + ''' + query = {'type': 'op', 'cmd': 'all'} + + return __proxy__['panos.call'](query) + + +def get_lldp_local(): + ''' + Show lldp local info for interfaces. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.get_lldp_local + + ''' + query = {'type': 'op', 'cmd': 'all'} + + return __proxy__['panos.call'](query) + + +def get_lldp_neighbors(): + ''' + Show lldp neighbors info for interfaces. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.get_lldp_neighbors + + ''' + query = {'type': 'op', 'cmd': 'all'} + + return __proxy__['panos.call'](query) + + +def get_logdb_quota(): + ''' + Report the logdb quotas. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.get_logdb_quota + + ''' + query = {'type': 'op', 'cmd': ''} + + return __proxy__['panos.call'](query) + + +def get_master_key(): + ''' + Get the master key properties. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.get_master_key + + ''' + query = {'type': 'op', 'cmd': ''} + + return __proxy__['panos.call'](query) + + +def get_ntp_config(): + ''' + Get the NTP configuration from the candidate configuration. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.get_ntp_config + + ''' + query = {'type': 'config', + 'action': 'get', + 'xpath': '/config/devices/entry[@name=\'localhost.localdomain\']/deviceconfig/system/ntp-servers'} + + return __proxy__['panos.call'](query) + + +def get_ntp_servers(): + ''' + Get list of configured NTP servers. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.get_ntp_servers + + ''' + query = {'type': 'op', 'cmd': ''} + + return __proxy__['panos.call'](query) + + +def get_operational_mode(): + ''' + Show device operational mode setting. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.get_operational_mode + + ''' + query = {'type': 'op', 'cmd': ''} + + return __proxy__['panos.call'](query) + + +def get_panorama_status(): + ''' + Show panorama connection status. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.get_panorama_status + + ''' + query = {'type': 'op', 'cmd': ''} + + return __proxy__['panos.call'](query) + + +def get_permitted_ips(): + ''' + Get the IP addresses that are permitted to establish management connections to the device. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.get_permitted_ips + + ''' + query = {'type': 'config', + 'action': 'get', + 'xpath': '/config/devices/entry[@name=\'localhost.localdomain\']/deviceconfig/system/permitted-ip'} + + return __proxy__['panos.call'](query) + + +def get_platform(): + ''' + Get the platform model information and limitations. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.get_platform + + ''' + query = {'type': 'config', + 'action': 'get', + 'xpath': '/config/devices/entry[@name=\'localhost.localdomain\']/platform'} + + return __proxy__['panos.call'](query) + + +def get_security_rule(rulename=None, vsys='1'): + ''' + Get the candidate configuration for the specified rule. + + rulename(str): The name of the security rule. + + vsys(str): The string representation of the VSYS ID. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.get_security_rule rule01 + salt '*' panos.get_security_rule rule01 3 + + ''' + query = {'type': 'config', + 'action': 'get', + 'xpath': '/config/devices/entry[@name=\'localhost.localdomain\']/vsys/entry[@name=\'vsys{0}\']/' + 'rulebase/security/rules/entry[@name=\'{1}\']'.format(vsys, rulename)} + + return __proxy__['panos.call'](query) + + +def get_session_info(): + ''' + Show device session statistics. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.get_session_info + + ''' + query = {'type': 'op', + 'cmd': ''} + + return __proxy__['panos.call'](query) + + +def get_snmp_config(): + ''' + Get the SNMP configuration from the device. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.get_snmp_config + + ''' + query = {'type': 'config', + 'action': 'get', + 'xpath': '/config/devices/entry[@name=\'localhost.localdomain\']/deviceconfig/system/snmp-setting'} + + return __proxy__['panos.call'](query) + + +def get_software_info(): + ''' + Show information about available software packages. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.get_software_info + + ''' + query = {'type': 'op', 'cmd': ''} + + return __proxy__['panos.call'](query) + + +def get_system_date_time(): + ''' + Get the system date/time. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.get_system_date_time + + ''' + query = {'type': 'op', 'cmd': ''} + + return __proxy__['panos.call'](query) + + +def get_system_files(): + ''' + List important files in the system. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.get_system_files + + ''' + query = {'type': 'op', 'cmd': ''} + + return __proxy__['panos.call'](query) + + +def get_system_info(): + ''' + Get the system information. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.get_system_info + + ''' + query = {'type': 'op', 'cmd': ''} + + return __proxy__['panos.call'](query) + + +def get_system_services(): + ''' + Show system services. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.get_system_services + + ''' + query = {'type': 'op', 'cmd': ''} + + return __proxy__['panos.call'](query) + + +def get_system_state(filter=None): + ''' + Show the system state variables. + + filter + Filters by a subtree or a wildcard. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.get_system_state + salt '*' panos.get_system_state filter=cfg.ha.config.enabled + salt '*' panos.get_system_state filter=cfg.ha.* + + ''' + if filter: + query = {'type': 'op', + 'cmd': '{0}'.format(filter)} + else: + query = {'type': 'op', 'cmd': ''} + + return __proxy__['panos.call'](query) + + +def get_uncommitted_changes(): + ''' + Retrieve a list of all uncommitted changes on the device. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.get_uncommitted_changes + + ''' + query = {'type': 'op', + 'cmd': ''} + + return __proxy__['panos.call'](query) + + +def get_users_config(): + ''' + Get the local administrative user account configuration. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.get_users_config + + ''' + query = {'type': 'config', + 'action': 'get', + 'xpath': '/config/mgt-config/users'} + + return __proxy__['panos.call'](query) + + +def get_vlans(): + ''' + Show all VLAN information. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.get_vlans + + ''' + query = {'type': 'op', 'cmd': 'all'} + + return __proxy__['panos.call'](query) + + +def install_antivirus(version=None, latest=False, synch=False, skip_commit=False,): + ''' + Install anti-virus packages. + + Args: + version(str): The version of the PANOS file to install. + + latest(bool): If true, the latest anti-virus file will be installed. + The specified version option will be ignored. + + synch(bool): If true, the anti-virus will synch to the peer unit. + + skip_commit(bool): If true, the install will skip committing to the device. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.install_antivirus 8.0.0 + + ''' + if not version and latest is False: + raise salt.exception.CommandExecutionError("Version option must not be none.") + + if synch is True: + s = "yes" + else: + s = "no" + + if skip_commit is True: + c = "yes" + else: + c = "no" + + if latest is True: + query = {'type': 'op', + 'cmd': '' + '{0}{1}' + 'latest'.format(c, s)} + else: + query = {'type': 'op', + 'cmd': '' + '{0}{1}' + '{2}'.format(c, s, version)} + + return _get_job_results(query) + + +def install_license(): + ''' + Install the license key(s). + + CLI Example: + + .. code-block:: bash + + salt '*' panos.install_license + + ''' + query = {'type': 'op', 'cmd': ''} + + return __proxy__['panos.call'](query) + + +def install_software(version=None): + ''' + Upgrade to a software package by version. + + Args: + version(str): The version of the PANOS file to install. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.install_license 8.0.0 + + ''' + if not version: + raise salt.exception.CommandExecutionError("Version option must not be none.") + + query = {'type': 'op', + 'cmd': '' + '{0}'.format(version)} + + return _get_job_results(query) + + +def reboot(): + ''' + Reboot a running system. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.reboot + + ''' + query = {'type': 'op', 'cmd': ''} + + return __proxy__['panos.call'](query) + + +def refresh_fqdn_cache(force=False): + ''' + Force refreshes all FQDNs used in rules. + + force + Forces all fqdn refresh + + CLI Example: + + .. code-block:: bash + + salt '*' panos.refresh_fqdn_cache + salt '*' panos.refresh_fqdn_cache force=True + + ''' + if not isinstance(force, bool): + raise salt.exception.CommandExecutionError("Force option must be boolean.") + + if force: + query = {'type': 'op', + 'cmd': 'yes'} + else: + query = {'type': 'op', 'cmd': ''} + + return __proxy__['panos.call'](query) + + +def remove_config_lock(): + ''' + Release config lock previously held. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.remove_config_lock + + ''' + query = {'type': 'op', 'cmd': ''} + + return __proxy__['panos.call'](query) + + +def resolve_address(address=None, vsys=None): + ''' + Resolve address to ip address. + Required version 7.0.0 or greater. + + address + Address name you want to resolve. + + vsys + The vsys name. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.resolve_address foo.bar.com + salt '*' panos.resolve_address foo.bar.com vsys=2 + + ''' + _required_version = '7.0.0' + if not __proxy__['panos.is_required_version'](_required_version): + return False, 'The panos device requires version {0} or greater for this command.'.format(_required_version) + + if not address: + raise salt.exception.CommandExecutionError("FQDN to resolve must be provided as address.") + + if not vsys: + query = {'type': 'op', + 'cmd': '
{0}
'.format(address)} + else: + query = {'type': 'op', + 'cmd': '{0}
{1}
' + ''.format(vsys, address)} + + return __proxy__['panos.call'](query) + + +def save_device_config(filename=None): + ''' + Save device configuration to a named file. + + filename + The filename to save the configuration to. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.save_device_config foo.xml + + ''' + if not filename: + raise salt.exception.CommandExecutionError("Filename must not be empty.") + + query = {'type': 'op', 'cmd': '{0}'.format(filename)} + + return __proxy__['panos.call'](query) + + +def save_device_state(): + ''' + Save files needed to restore device to local disk. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.save_device_state + + ''' + query = {'type': 'op', 'cmd': ''} + + return __proxy__['panos.call'](query) + + +def set_authentication_profile(profile=None, deploy=False): + ''' + Set the authentication profile of the Palo Alto proxy minion. A commit will be required before this is processed. + + CLI Example: + + Args: + profile (str): The name of the authentication profile to set. + + deploy (bool): If true then commit the full candidate configuration, if false only set pending change. + + .. code-block:: bash + + salt '*' panos.set_authentication_profile foo + salt '*' panos.set_authentication_profile foo deploy=True + + ''' + + if not profile: + salt.exception.CommandExecutionError("Profile name option must not be none.") + + ret = {} + + query = {'type': 'config', + 'action': 'set', + 'xpath': '/config/devices/entry[@name=\'localhost.localdomain\']/deviceconfig/system/' + 'authentication-profile', + 'element': '{0}'.format(profile)} + + ret.update(__proxy__['panos.call'](query)) + + if deploy is True: + ret.update(commit()) + + return ret + + +def set_hostname(hostname=None, deploy=False): + ''' + Set the hostname of the Palo Alto proxy minion. A commit will be required before this is processed. + + CLI Example: + + Args: + hostname (str): The hostname to set + + deploy (bool): If true then commit the full candidate configuration, if false only set pending change. + + .. code-block:: bash + + salt '*' panos.set_hostname newhostname + salt '*' panos.set_hostname newhostname deploy=True + + ''' + + if not hostname: + salt.exception.CommandExecutionError("Hostname option must not be none.") + + ret = {} + + query = {'type': 'config', + 'action': 'set', + 'xpath': '/config/devices/entry[@name=\'localhost.localdomain\']/deviceconfig/system', + 'element': '{0}'.format(hostname)} + + ret.update(__proxy__['panos.call'](query)) + + if deploy is True: + ret.update(commit()) + + return ret + + +def set_management_icmp(enabled=True, deploy=False): + ''' + Enables or disables the ICMP management service on the device. + + CLI Example: + + Args: + enabled (bool): If true the service will be enabled. If false the service will be disabled. + + deploy (bool): If true then commit the full candidate configuration, if false only set pending change. + + .. code-block:: bash + + salt '*' panos.set_management_icmp + salt '*' panos.set_management_icmp enabled=False deploy=True + + ''' + + if enabled is True: + value = "no" + elif enabled is False: + value = "yes" + else: + salt.exception.CommandExecutionError("Invalid option provided for service enabled option.") + + ret = {} + + query = {'type': 'config', + 'action': 'set', + 'xpath': '/config/devices/entry[@name=\'localhost.localdomain\']/deviceconfig/system/service', + 'element': '{0}'.format(value)} + + ret.update(__proxy__['panos.call'](query)) + + if deploy is True: + ret.update(commit()) + + return ret + + +def set_management_http(enabled=True, deploy=False): + ''' + Enables or disables the HTTP management service on the device. + + CLI Example: + + Args: + enabled (bool): If true the service will be enabled. If false the service will be disabled. + + deploy (bool): If true then commit the full candidate configuration, if false only set pending change. + + .. code-block:: bash + + salt '*' panos.set_management_http + salt '*' panos.set_management_http enabled=False deploy=True + + ''' + + if enabled is True: + value = "no" + elif enabled is False: + value = "yes" + else: + salt.exception.CommandExecutionError("Invalid option provided for service enabled option.") + + ret = {} + + query = {'type': 'config', + 'action': 'set', + 'xpath': '/config/devices/entry[@name=\'localhost.localdomain\']/deviceconfig/system/service', + 'element': '{0}'.format(value)} + + ret.update(__proxy__['panos.call'](query)) + + if deploy is True: + ret.update(commit()) + + return ret + + +def set_management_https(enabled=True, deploy=False): + ''' + Enables or disables the HTTPS management service on the device. + + CLI Example: + + Args: + enabled (bool): If true the service will be enabled. If false the service will be disabled. + + deploy (bool): If true then commit the full candidate configuration, if false only set pending change. + + .. code-block:: bash + + salt '*' panos.set_management_https + salt '*' panos.set_management_https enabled=False deploy=True + + ''' + + if enabled is True: + value = "no" + elif enabled is False: + value = "yes" + else: + salt.exception.CommandExecutionError("Invalid option provided for service enabled option.") + + ret = {} + + query = {'type': 'config', + 'action': 'set', + 'xpath': '/config/devices/entry[@name=\'localhost.localdomain\']/deviceconfig/system/service', + 'element': '{0}'.format(value)} + + ret.update(__proxy__['panos.call'](query)) + + if deploy is True: + ret.update(commit()) + + return ret + + +def set_management_ocsp(enabled=True, deploy=False): + ''' + Enables or disables the HTTP OCSP management service on the device. + + CLI Example: + + Args: + enabled (bool): If true the service will be enabled. If false the service will be disabled. + + deploy (bool): If true then commit the full candidate configuration, if false only set pending change. + + .. code-block:: bash + + salt '*' panos.set_management_ocsp + salt '*' panos.set_management_ocsp enabled=False deploy=True + + ''' + + if enabled is True: + value = "no" + elif enabled is False: + value = "yes" + else: + salt.exception.CommandExecutionError("Invalid option provided for service enabled option.") + + ret = {} + + query = {'type': 'config', + 'action': 'set', + 'xpath': '/config/devices/entry[@name=\'localhost.localdomain\']/deviceconfig/system/service', + 'element': '{0}'.format(value)} + + ret.update(__proxy__['panos.call'](query)) + + if deploy is True: + ret.update(commit()) + + return ret + + +def set_management_snmp(enabled=True, deploy=False): + ''' + Enables or disables the SNMP management service on the device. + + CLI Example: + + Args: + enabled (bool): If true the service will be enabled. If false the service will be disabled. + + deploy (bool): If true then commit the full candidate configuration, if false only set pending change. + + .. code-block:: bash + + salt '*' panos.set_management_snmp + salt '*' panos.set_management_snmp enabled=False deploy=True + + ''' + + if enabled is True: + value = "no" + elif enabled is False: + value = "yes" + else: + salt.exception.CommandExecutionError("Invalid option provided for service enabled option.") + + ret = {} + + query = {'type': 'config', + 'action': 'set', + 'xpath': '/config/devices/entry[@name=\'localhost.localdomain\']/deviceconfig/system/service', + 'element': '{0}'.format(value)} + + ret.update(__proxy__['panos.call'](query)) + + if deploy is True: + ret.update(commit()) + + return ret + + +def set_management_ssh(enabled=True, deploy=False): + ''' + Enables or disables the SSH management service on the device. + + CLI Example: + + Args: + enabled (bool): If true the service will be enabled. If false the service will be disabled. + + deploy (bool): If true then commit the full candidate configuration, if false only set pending change. + + .. code-block:: bash + + salt '*' panos.set_management_ssh + salt '*' panos.set_management_ssh enabled=False deploy=True + + ''' + + if enabled is True: + value = "no" + elif enabled is False: + value = "yes" + else: + salt.exception.CommandExecutionError("Invalid option provided for service enabled option.") + + ret = {} + + query = {'type': 'config', + 'action': 'set', + 'xpath': '/config/devices/entry[@name=\'localhost.localdomain\']/deviceconfig/system/service', + 'element': '{0}'.format(value)} + + ret.update(__proxy__['panos.call'](query)) + + if deploy is True: + ret.update(commit()) + + return ret + + +def set_management_telnet(enabled=True, deploy=False): + ''' + Enables or disables the Telnet management service on the device. + + CLI Example: + + Args: + enabled (bool): If true the service will be enabled. If false the service will be disabled. + + deploy (bool): If true then commit the full candidate configuration, if false only set pending change. + + .. code-block:: bash + + salt '*' panos.set_management_telnet + salt '*' panos.set_management_telnet enabled=False deploy=True + + ''' + + if enabled is True: + value = "no" + elif enabled is False: + value = "yes" + else: + salt.exception.CommandExecutionError("Invalid option provided for service enabled option.") + + ret = {} + + query = {'type': 'config', + 'action': 'set', + 'xpath': '/config/devices/entry[@name=\'localhost.localdomain\']/deviceconfig/system/service', + 'element': '{0}'.format(value)} + + ret.update(__proxy__['panos.call'](query)) + + if deploy is True: + ret.update(commit()) + + return ret + + +def set_ntp_authentication(target=None, + authentication_type=None, + key_id=None, + authentication_key=None, + algorithm=None, + deploy=False): + ''' + Set the NTP authentication of the Palo Alto proxy minion. A commit will be required before this is processed. + + CLI Example: + + Args: + target(str): Determines the target of the authentication. Valid options are primary, secondary, or both. + + authentication_type(str): The authentication type to be used. Valid options are symmetric, autokey, and none. + + key_id(int): The NTP authentication key ID. + + authentication_key(str): The authentication key. + + algorithm(str): The algorithm type to be used for a symmetric key. Valid options are md5 and sha1. + + deploy (bool): If true then commit the full candidate configuration, if false only set pending change. + + .. code-block:: bash + + salt '*' ntp.set_authentication target=both authentication_type=autokey + salt '*' ntp.set_authentication target=primary authentication_type=none + salt '*' ntp.set_authentication target=both authentication_type=symmetric key_id=15 authentication_key=mykey algorithm=md5 + salt '*' ntp.set_authentication target=both authentication_type=symmetric key_id=15 authentication_key=mykey algorithm=md5 deploy=True + + ''' + ret = {} + + if target not in ['primary', 'secondary', 'both']: + raise salt.exceptions.CommandExecutionError("Target option must be primary, secondary, or both.") + + if authentication_type not in ['symmetric', 'autokey', 'none']: + raise salt.exceptions.CommandExecutionError("Type option must be symmetric, autokey, or both.") + + if authentication_type == "symmetric" and not authentication_key: + raise salt.exceptions.CommandExecutionError("When using symmetric authentication, authentication_key must be " + "provided.") + + if authentication_type == "symmetric" and not key_id: + raise salt.exceptions.CommandExecutionError("When using symmetric authentication, key_id must be provided.") + + if authentication_type == "symmetric" and algorithm not in ['md5', 'sha1']: + raise salt.exceptions.CommandExecutionError("When using symmetric authentication, algorithm must be md5 or " + "sha1.") + + if authentication_type == 'symmetric': + if target == 'primary' or target == 'both': + query = {'type': 'config', + 'action': 'set', + 'xpath': '/config/devices/entry[@name=\'localhost.localdomain\']/deviceconfig/system/ntp-servers/' + 'primary-ntp-server/authentication-type', + 'element': '<{0}>{1}' + '{2}'.format(algorithm, + authentication_key, + key_id)} + ret.update({'primary_server': __proxy__['panos.call'](query)}) + + if target == 'secondary' or target == 'both': + query = {'type': 'config', + 'action': 'set', + 'xpath': '/config/devices/entry[@name=\'localhost.localdomain\']/deviceconfig/system/ntp-servers/' + 'secondary-ntp-server/authentication-type', + 'element': '<{0}>{1}' + '{2}'.format(algorithm, + authentication_key, + key_id)} + ret.update({'secondary_server': __proxy__['panos.call'](query)}) + elif authentication_type == 'autokey': + if target == 'primary' or target == 'both': + query = {'type': 'config', + 'action': 'set', + 'xpath': '/config/devices/entry[@name=\'localhost.localdomain\']/deviceconfig/system/ntp-servers/' + 'primary-ntp-server/authentication-type', + 'element': ''} + ret.update({'primary_server': __proxy__['panos.call'](query)}) + + if target == 'secondary' or target == 'both': + query = {'type': 'config', + 'action': 'set', + 'xpath': '/config/devices/entry[@name=\'localhost.localdomain\']/deviceconfig/system/ntp-servers/' + 'secondary-ntp-server/authentication-type', + 'element': ''} + ret.update({'secondary_server': __proxy__['panos.call'](query)}) + elif authentication_type == 'none': + if target == 'primary' or target == 'both': + query = {'type': 'config', + 'action': 'set', + 'xpath': '/config/devices/entry[@name=\'localhost.localdomain\']/deviceconfig/system/ntp-servers/' + 'primary-ntp-server/authentication-type', + 'element': ''} + ret.update({'primary_server': __proxy__['panos.call'](query)}) + + if target == 'secondary' or target == 'both': + query = {'type': 'config', + 'action': 'set', + 'xpath': '/config/devices/entry[@name=\'localhost.localdomain\']/deviceconfig/system/ntp-servers/' + 'secondary-ntp-server/authentication-type', + 'element': ''} + ret.update({'secondary_server': __proxy__['panos.call'](query)}) + + if deploy is True: + ret.update(commit()) + + return ret + + +def set_ntp_servers(primary_server=None, secondary_server=None, deploy=False): + ''' + Set the NTP servers of the Palo Alto proxy minion. A commit will be required before this is processed. + + CLI Example: + + Args: + primary_server(str): The primary NTP server IP address or FQDN. + + secondary_server(str): The secondary NTP server IP address or FQDN. + + deploy (bool): If true then commit the full candidate configuration, if false only set pending change. + + .. code-block:: bash + + salt '*' ntp.set_servers 0.pool.ntp.org 1.pool.ntp.org + salt '*' ntp.set_servers primary_server=0.pool.ntp.org secondary_server=1.pool.ntp.org + salt '*' ntp.ser_servers 0.pool.ntp.org 1.pool.ntp.org deploy=True + + ''' + ret = {} + + if primary_server: + query = {'type': 'config', + 'action': 'set', + 'xpath': '/config/devices/entry[@name=\'localhost.localdomain\']/deviceconfig/system/ntp-servers/' + 'primary-ntp-server', + 'element': '{0}'.format(primary_server)} + ret.update({'primary_server': __proxy__['panos.call'](query)}) + + if secondary_server: + query = {'type': 'config', + 'action': 'set', + 'xpath': '/config/devices/entry[@name=\'localhost.localdomain\']/deviceconfig/system/ntp-servers/' + 'secondary-ntp-server', + 'element': '{0}'.format(secondary_server)} + ret.update({'secondary_server': __proxy__['panos.call'](query)}) + + if deploy is True: + ret.update(commit()) + + return ret + + +def set_permitted_ip(address=None, deploy=False): + ''' + Add an IPv4 address or network to the permitted IP list. + + CLI Example: + + Args: + address (str): The IPv4 address or network to allow access to add to the Palo Alto device. + + deploy (bool): If true then commit the full candidate configuration, if false only set pending change. + + .. code-block:: bash + + salt '*' panos.set_permitted_ip 10.0.0.1 + salt '*' panos.set_permitted_ip 10.0.0.0/24 + salt '*' panos.set_permitted_ip 10.0.0.1 deploy=True + + ''' + + if not address: + salt.exception.CommandExecutionError("Address option must not be empty.") + + ret = {} + + query = {'type': 'config', + 'action': 'set', + 'xpath': '/config/devices/entry[@name=\'localhost.localdomain\']/deviceconfig/system/permitted-ip', + 'element': ''.format(address)} + + ret.update(__proxy__['panos.call'](query)) + + if deploy is True: + ret.update(commit()) + + return ret + + +def set_timezone(tz=None, deploy=False): + ''' + Set the timezone of the Palo Alto proxy minion. A commit will be required before this is processed. + + CLI Example: + + Args: + tz (str): The name of the timezone to set. + + deploy (bool): If true then commit the full candidate configuration, if false only set pending change. + + .. code-block:: bash + + salt '*' panos.set_timezone UTC + salt '*' panos.set_timezone UTC deploy=True + + ''' + + if not tz: + salt.exception.CommandExecutionError("Timezone name option must not be none.") + + ret = {} + + query = {'type': 'config', + 'action': 'set', + 'xpath': '/config/devices/entry[@name=\'localhost.localdomain\']/deviceconfig/system/timezone', + 'element': '{0}'.format(tz)} + + ret.update(__proxy__['panos.call'](query)) + + if deploy is True: + ret.update(commit()) + + return ret + + +def shutdown(): + ''' + Shutdown a running system. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.shutdown + + ''' + query = {'type': 'op', 'cmd': ''} + + return __proxy__['panos.call'](query) + + +def unlock_admin(username=None): + ''' + Unlocks a locked administrator account. + + username + Username of the administrator. + + CLI Example: + + .. code-block:: bash + + salt '*' panos.unlock_admin username=bob + + ''' + if not username: + raise salt.exception.CommandExecutionError("Username option must not be none.") + + query = {'type': 'op', + 'cmd': '{0}' + ''.format(username)} + + return __proxy__['panos.call'](query) diff --git a/salt/modules/parallels.py b/salt/modules/parallels.py index c72b61ceefe..d361350cc47 100644 --- a/salt/modules/parallels.py +++ b/salt/modules/parallels.py @@ -27,12 +27,12 @@ import shlex import yaml # Import salt libs -import salt.utils -from salt.utils.locales import sdecode as _sdecode +import salt.utils.locales +import salt.utils.path from salt.exceptions import SaltInvocationError # Import 3rd party libs -import salt.ext.six as six +from salt.ext import six __virtualname__ = 'parallels' __func_alias__ = { @@ -47,9 +47,9 @@ def __virtual__(): ''' Load this module if prlctl is available ''' - if not salt.utils.which('prlctl'): + if not salt.utils.path.which('prlctl'): return (False, 'prlctl utility not available') - if not salt.utils.which('prlsrvctl'): + if not salt.utils.path.which('prlsrvctl'): return (False, 'prlsrvctl utility not available') return __virtualname__ @@ -237,7 +237,7 @@ def clone(name, new_name, linked=False, template=False, runas=None): salt '*' parallels.clone macvm macvm_new runas=macdev salt '*' parallels.clone macvm macvm_templ template=True runas=macdev ''' - args = [_sdecode(name), '--name', _sdecode(new_name)] + args = [salt.utils.locales.sdecode(name), '--name', salt.utils.locales.sdecode(new_name)] if linked: args.append('--linked') if template: @@ -263,7 +263,7 @@ def delete(name, runas=None): salt '*' parallels.exec macvm 'find /etc/paths.d' runas=macdev ''' - return prlctl('delete', _sdecode(name), runas=runas) + return prlctl('delete', salt.utils.locales.sdecode(name), runas=runas) def exists(name, runas=None): @@ -307,7 +307,7 @@ def start(name, runas=None): salt '*' parallels.start macvm runas=macdev ''' - return prlctl('start', _sdecode(name), runas=runas) + return prlctl('start', salt.utils.locales.sdecode(name), runas=runas) def stop(name, kill=False, runas=None): @@ -331,7 +331,7 @@ def stop(name, kill=False, runas=None): salt '*' parallels.stop macvm kill=True runas=macdev ''' # Construct argument list - args = [_sdecode(name)] + args = [salt.utils.locales.sdecode(name)] if kill: args.append('--kill') @@ -356,7 +356,7 @@ def restart(name, runas=None): salt '*' parallels.restart macvm runas=macdev ''' - return prlctl('restart', _sdecode(name), runas=runas) + return prlctl('restart', salt.utils.locales.sdecode(name), runas=runas) def reset(name, runas=None): @@ -375,7 +375,7 @@ def reset(name, runas=None): salt '*' parallels.reset macvm runas=macdev ''' - return prlctl('reset', _sdecode(name), runas=runas) + return prlctl('reset', salt.utils.locales.sdecode(name), runas=runas) def status(name, runas=None): @@ -394,7 +394,7 @@ def status(name, runas=None): salt '*' parallels.status macvm runas=macdev ''' - return prlctl('status', _sdecode(name), runas=runas) + return prlctl('status', salt.utils.locales.sdecode(name), runas=runas) def exec_(name, command, runas=None): @@ -417,7 +417,7 @@ def exec_(name, command, runas=None): salt '*' parallels.exec macvm 'find /etc/paths.d' runas=macdev ''' # Construct argument list - args = [_sdecode(name)] + args = [salt.utils.locales.sdecode(name)] args.extend(_normalize_args(command)) # Execute command and return output @@ -459,10 +459,10 @@ def snapshot_id_to_name(name, snap_id, strict=False, runas=None): salt '*' parallels.snapshot_id_to_name macvm a5b8999f-5d95-4aff-82de-e515b0101b66 runas=macdev ''' # Validate VM name and snapshot ID - name = _sdecode(name) + name = salt.utils.locales.sdecode(name) if not re.match(GUID_REGEX, snap_id): raise SaltInvocationError( - u'Snapshot ID "{0}" is not a GUID'.format(_sdecode(snap_id)) + u'Snapshot ID "{0}" is not a GUID'.format(salt.utils.locales.sdecode(snap_id)) ) # Get the snapshot information of the snapshot having the requested ID @@ -503,7 +503,7 @@ def snapshot_id_to_name(name, snap_id, strict=False, runas=None): u'Could not find a snapshot name for snapshot ID "{0}" of VM ' u'"{1}"'.format(snap_id, name) ) - return _sdecode(snap_name) + return salt.utils.locales.sdecode(snap_name) def snapshot_name_to_id(name, snap_name, strict=False, runas=None): @@ -531,8 +531,8 @@ def snapshot_name_to_id(name, snap_name, strict=False, runas=None): salt '*' parallels.snapshot_id_to_name macvm original runas=macdev ''' # Validate VM and snapshot names - name = _sdecode(name) - snap_name = _sdecode(snap_name) + name = salt.utils.locales.sdecode(name) + snap_name = salt.utils.locales.sdecode(snap_name) # Get a multiline string containing all the snapshot GUIDs info = prlctl('snapshot-list', name, runas=runas) @@ -580,7 +580,7 @@ def _validate_snap_name(name, snap_name, strict=True, runas=None): :param str runas: The user that the prlctl command will be run as ''' - snap_name = _sdecode(snap_name) + snap_name = salt.utils.locales.sdecode(snap_name) # Try to convert snapshot name to an ID without {} if re.match(GUID_REGEX, snap_name): @@ -620,7 +620,7 @@ def list_snapshots(name, snap_name=None, tree=False, names=False, runas=None): salt '*' parallels.list_snapshots macvm names=True runas=macdev ''' # Validate VM and snapshot names - name = _sdecode(name) + name = salt.utils.locales.sdecode(name) if snap_name: snap_name = _validate_snap_name(name, snap_name, runas=runas) @@ -643,7 +643,7 @@ def list_snapshots(name, snap_name=None, tree=False, names=False, runas=None): ret = '{0:<38} {1}\n'.format('Snapshot ID', 'Snapshot Name') for snap_id in snap_ids: snap_name = snapshot_id_to_name(name, snap_id, runas=runas) - ret += (u'{{{0}}} {1}\n'.format(snap_id, _sdecode(snap_name))) + ret += (u'{{{0}}} {1}\n'.format(snap_id, salt.utils.locales.sdecode(snap_name))) return ret # Return information directly from parallels desktop @@ -675,9 +675,9 @@ def snapshot(name, snap_name=None, desc=None, runas=None): salt '*' parallels.create_snapshot macvm snap_name=macvm-updates desc='clean install with updates' runas=macdev ''' # Validate VM and snapshot names - name = _sdecode(name) + name = salt.utils.locales.sdecode(name) if snap_name: - snap_name = _sdecode(snap_name) + snap_name = salt.utils.locales.sdecode(snap_name) # Construct argument list args = [name] @@ -724,7 +724,7 @@ def delete_snapshot(name, snap_name, runas=None, all=False): strict = not all # Validate VM and snapshot names - name = _sdecode(name) + name = salt.utils.locales.sdecode(name) snap_ids = _validate_snap_name(name, snap_name, strict=strict, runas=runas) if isinstance(snap_ids, six.string_types): snap_ids = [snap_ids] @@ -767,7 +767,7 @@ def revert_snapshot(name, snap_name, runas=None): salt '*' parallels.revert_snapshot macvm base-with-updates runas=macdev ''' # Validate VM and snapshot names - name = _sdecode(name) + name = salt.utils.locales.sdecode(name) snap_name = _validate_snap_name(name, snap_name, runas=runas) # Construct argument list diff --git a/salt/modules/parted.py b/salt/modules/parted.py index 04a7d37d47d..72a6d3ce654 100644 --- a/salt/modules/parted.py +++ b/salt/modules/parted.py @@ -23,8 +23,9 @@ import stat import string import logging -# Import salt libs -import salt.utils +# Import Salt libs +import salt.utils.path +import salt.utils.platform from salt.exceptions import CommandExecutionError log = logging.getLogger(__name__) @@ -48,16 +49,16 @@ def __virtual__(): Only work on POSIX-like systems, which have parted and lsblk installed. These are usually provided by the ``parted`` and ``util-linux`` packages. ''' - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): return (False, 'The parted execution module failed to load ' 'Windows systems are not supported.') - if not salt.utils.which('parted'): + if not salt.utils.path.which('parted'): return (False, 'The parted execution module failed to load ' 'parted binary is not in the path.') - if not salt.utils.which('lsblk'): + if not salt.utils.path.which('lsblk'): return (False, 'The parted execution module failed to load ' 'lsblk binary is not in the path.') - if not salt.utils.which('partprobe'): + if not salt.utils.path.which('partprobe'): return (False, 'The parted execution module failed to load ' 'partprobe binary is not in the path.') return __virtualname__ @@ -216,7 +217,7 @@ def align_check(device, part_type, partition): 'Invalid partition passed to partition.align_check' ) - cmd = 'parted -m -s {0} align-check {1} {2}'.format( + cmd = 'parted -m {0} align-check {1} {2}'.format( device, part_type, partition ) out = __salt__['cmd.run'](cmd).splitlines() @@ -388,7 +389,7 @@ def mkfs(device, fs_type): else: mkfs_cmd = 'mkfs.{0}'.format(fs_type) - if not salt.utils.which(mkfs_cmd): + if not salt.utils.path.which(mkfs_cmd): return 'Error: {0} is unavailable.'.format(mkfs_cmd) cmd = '{0} {1}'.format(mkfs_cmd, device) out = __salt__['cmd.run'](cmd).splitlines() diff --git a/salt/modules/pcs.py b/salt/modules/pcs.py index 02153c12e07..d99bc29ea5f 100644 --- a/salt/modules/pcs.py +++ b/salt/modules/pcs.py @@ -14,14 +14,15 @@ from __future__ import absolute_import # Import salt libs import salt.utils -import salt.ext.six as six +import salt.utils.path +from salt.ext import six def __virtual__(): ''' Only load if pcs package is installed ''' - if salt.utils.which('pcs'): + if salt.utils.path.which('pcs'): return 'pcs' return False diff --git a/salt/modules/pdbedit.py b/salt/modules/pdbedit.py index d8cb4a75a63..a226398719b 100644 --- a/salt/modules/pdbedit.py +++ b/salt/modules/pdbedit.py @@ -21,6 +21,7 @@ except ImportError: # Import Salt libs import salt.utils +import salt.utils.path log = logging.getLogger(__name__) @@ -38,7 +39,7 @@ def __virtual__(): ''' Provides pdbedit if available ''' - if salt.utils.which('pdbedit'): + if salt.utils.path.which('pdbedit'): return __virtualname__ return ( False, diff --git a/salt/modules/pecl.py b/salt/modules/pecl.py index edca3bcd56c..284ce74e493 100644 --- a/salt/modules/pecl.py +++ b/salt/modules/pecl.py @@ -15,9 +15,10 @@ except ImportError: # Import salt libs import salt.utils +import salt.utils.path # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six __func_alias__ = { 'list_': 'list' @@ -30,7 +31,7 @@ __virtualname__ = 'pecl' def __virtual__(): - if salt.utils.which('pecl'): + if salt.utils.path.which('pecl'): return __virtualname__ return (False, 'The pecl execution module not loaded: ' 'pecl binary is not in the path.') diff --git a/salt/modules/pillar.py b/salt/modules/pillar.py index e9351aed2a2..a81892c0692 100644 --- a/salt/modules/pillar.py +++ b/salt/modules/pillar.py @@ -13,7 +13,7 @@ import os import copy import logging import yaml -import salt.ext.six as six +from salt.ext import six # Import salt libs import salt.pillar @@ -257,7 +257,7 @@ def items(*args, **kwargs): __opts__, __grains__, __opts__['id'], - pillar=pillar_override, + pillar_override=pillar_override, pillarenv=pillarenv) return pillar.compile_pillar() @@ -465,7 +465,7 @@ def ext(external, pillar=None): __opts__['id'], __opts__['environment'], ext=external, - pillar=pillar) + pillar_override=pillar) ret = pillar_obj.compile_pillar() diff --git a/salt/modules/pip.py b/salt/modules/pip.py index c9252db85ca..31a59458ee3 100644 --- a/salt/modules/pip.py +++ b/salt/modules/pip.py @@ -82,12 +82,15 @@ import re import shutil import logging import sys - -# Import salt libs -import salt.utils import tempfile + +# Import Salt libs +import salt.utils # Can be removed once compare_dicts is moved +import salt.utils.files import salt.utils.locales +import salt.utils.platform import salt.utils.url +import salt.utils.versions from salt.ext import six from salt.exceptions import CommandExecutionError, CommandNotFoundError @@ -124,7 +127,7 @@ def _get_pip_bin(bin_env): 'pip{0}'.format(sys.version_info[0]), 'pip', 'pip-python'] ) - if salt.utils.is_windows() and six.PY2: + if salt.utils.platform.is_windows() and six.PY2: which_result.encode('string-escape') if which_result is None: raise CommandNotFoundError('Could not find a `pip` binary') @@ -132,7 +135,7 @@ def _get_pip_bin(bin_env): # try to get pip bin from virtualenv, bin_env if os.path.isdir(bin_env): - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): if six.PY2: pip_bin = os.path.join( bin_env, 'Scripts', 'pip.exe').encode('string-escape') @@ -191,7 +194,7 @@ def _get_env_activate(bin_env): raise CommandNotFoundError('Could not find a `activate` binary') if os.path.isdir(bin_env): - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): activate_bin = os.path.join(bin_env, 'Scripts', 'activate.bat') else: activate_bin = os.path.join(bin_env, 'bin', 'activate') @@ -204,7 +207,7 @@ def _find_req(link): logger.info('_find_req -- link = %s', str(link)) - with salt.utils.fopen(link) as fh_link: + with salt.utils.files.fopen(link) as fh_link: child_links = rex_pip_chain_read.findall(fh_link.read()) base_path = os.path.dirname(link) @@ -575,7 +578,7 @@ def install(pkgs=None, # pylint: disable=R0912,R0913,R0914 if use_wheel: min_version = '1.4' cur_version = __salt__['pip.version'](bin_env) - if not salt.utils.compare_versions(ver1=cur_version, oper='>=', + if not salt.utils.versions.compare(ver1=cur_version, oper='>=', ver2=min_version): logger.error( ('The --use-wheel option is only supported in pip {0} and ' @@ -588,7 +591,7 @@ def install(pkgs=None, # pylint: disable=R0912,R0913,R0914 if no_use_wheel: min_version = '1.4' cur_version = __salt__['pip.version'](bin_env) - if not salt.utils.compare_versions(ver1=cur_version, oper='>=', + if not salt.utils.versions.compare(ver1=cur_version, oper='>=', ver2=min_version): logger.error( ('The --no-use-wheel option is only supported in pip {0} and ' @@ -661,7 +664,7 @@ def install(pkgs=None, # pylint: disable=R0912,R0913,R0914 if mirrors: # https://github.com/pypa/pip/pull/2641/files#diff-3ef137fb9ffdd400f117a565cd94c188L216 pip_version = version(pip_bin) - if salt.utils.compare_versions(ver1=pip_version, oper='>=', ver2='7.0.0'): + if salt.utils.versions.compare(ver1=pip_version, oper='>=', ver2='7.0.0'): raise CommandExecutionError( 'pip >= 7.0.0 does not support mirror argument:' ' use index_url and/or extra_index_url instead' @@ -688,7 +691,7 @@ def install(pkgs=None, # pylint: disable=R0912,R0913,R0914 cmd.extend(['--download', download]) if download_cache or cache_dir: - cmd.extend(['--cache-dir' if salt.utils.compare_versions( + cmd.extend(['--cache-dir' if salt.utils.versions.compare( ver1=version(bin_env), oper='>=', ver2='6.0' ) else '--download-cache', download_cache or cache_dir]) @@ -729,7 +732,7 @@ def install(pkgs=None, # pylint: disable=R0912,R0913,R0914 pip_version = version(pip_bin) # From pip v1.4 the --pre flag is available - if salt.utils.compare_versions(ver1=pip_version, oper='>=', ver2='1.4'): + if salt.utils.versions.compare(ver1=pip_version, oper='>=', ver2='1.4'): cmd.append('--pre') if cert: @@ -936,7 +939,7 @@ def uninstall(pkgs=None, pkgs = [p.strip() for p in pkgs.split(',')] if requirements: for requirement in requirements: - with salt.utils.fopen(requirement) as rq_: + with salt.utils.files.fopen(requirement) as rq_: for req in rq_: try: req_pkg, _ = req.split('==') @@ -1005,7 +1008,7 @@ def freeze(bin_env=None, # Include pip, setuptools, distribute, wheel min_version = '8.0.3' cur_version = version(bin_env) - if not salt.utils.compare_versions(ver1=cur_version, oper='>=', + if not salt.utils.versions.compare(ver1=cur_version, oper='>=', ver2=min_version): logger.warning( ('The version of pip installed is {0}, which is older than {1}. ' diff --git a/salt/modules/pkg_resource.py b/salt/modules/pkg_resource.py index 928dccae7d0..70a2737af95 100644 --- a/salt/modules/pkg_resource.py +++ b/salt/modules/pkg_resource.py @@ -5,6 +5,7 @@ Resources needed by pkg providers # Import python libs from __future__ import absolute_import +import copy import fnmatch import logging import os @@ -12,10 +13,11 @@ import pprint # Import third party libs import yaml -import salt.ext.six as six +from salt.ext import six # Import salt libs import salt.utils +import salt.utils.versions from salt.exceptions import SaltInvocationError log = logging.getLogger(__name__) @@ -104,12 +106,7 @@ def parse_targets(name=None, salt '*' pkg_resource.parse_targets ''' if '__env__' in kwargs: - salt.utils.warn_until( - 'Oxygen', - 'Parameter \'__env__\' has been detected in the argument list. This ' - 'parameter is no longer used and has been replaced by \'saltenv\' ' - 'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.' - ) + # "env" is not supported; Use "saltenv". kwargs.pop('__env__') if __grains__['os'] == 'MacOS' and sources: @@ -306,3 +303,44 @@ def check_extra_requirements(pkgname, pkgver): return __salt__['pkg.check_extra_requirements'](pkgname, pkgver) return True + + +def format_pkg_list(packages, versions_as_list, attr): + ''' + Formats packages according to parameters for list_pkgs. + ''' + ret = copy.deepcopy(packages) + if attr: + requested_attr = set(['epoch', 'version', 'release', 'arch', + 'install_date', 'install_date_time_t']) + + if attr != 'all': + requested_attr &= set(attr + ['version']) + + for name in ret: + versions = [] + for all_attr in ret[name]: + filtered_attr = {} + for key in requested_attr: + if all_attr[key]: + filtered_attr[key] = all_attr[key] + versions.append(filtered_attr) + ret[name] = versions + return ret + + for name in ret: + ret[name] = [format_version(d['epoch'], d['version'], d['release']) + for d in ret[name]] + if not versions_as_list: + stringify(ret) + return ret + + +def format_version(epoch, version, release): + ''' + Formats a version string for list_pkgs. + ''' + full_version = '{0}:{1}'.format(epoch, version) if epoch else version + if release: + full_version += '-{0}'.format(release) + return full_version diff --git a/salt/modules/pkgin.py b/salt/modules/pkgin.py index fa702834e74..a5f253cc162 100644 --- a/salt/modules/pkgin.py +++ b/salt/modules/pkgin.py @@ -18,12 +18,13 @@ import re # Import salt libs import salt.utils +import salt.utils.path import salt.utils.pkg import salt.utils.decorators as decorators from salt.exceptions import CommandExecutionError, MinionError # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six VERSION_MATCH = re.compile(r'pkgin(?:[\s]+)([\d.]+)(?:[\s]+)(?:.*)') log = logging.getLogger(__name__) @@ -37,7 +38,7 @@ def _check_pkgin(): ''' Looks to see if pkgin is present on the system, return full path ''' - ppath = salt.utils.which('pkgin') + ppath = salt.utils.path.which('pkgin') if ppath is None: # pkgin was not found in $PATH, try to find it via LOCALBASE try: diff --git a/salt/modules/pkgng.py b/salt/modules/pkgng.py index d3c6cf643be..dc20541f947 100644 --- a/salt/modules/pkgng.py +++ b/salt/modules/pkgng.py @@ -45,9 +45,12 @@ import os # Import salt libs import salt.utils +import salt.utils.files +import salt.utils.itertools import salt.utils.pkg +import salt.utils.versions from salt.exceptions import CommandExecutionError, MinionError -import salt.ext.six as six +from salt.ext import six log = logging.getLogger(__name__) @@ -151,7 +154,7 @@ def parse_config(file_name='/usr/local/etc/pkg.conf'): if not os.path.isfile(file_name): return 'Unable to find {0} on file system'.format(file_name) - with salt.utils.fopen(file_name) as ifile: + with salt.utils.files.fopen(file_name) as ifile: for line in ifile: if line.startswith('#') or line.startswith('\n'): pass @@ -295,7 +298,7 @@ def latest_version(*names, **kwargs): root = kwargs.get('root') pkgs = list_pkgs(versions_as_list=True, jail=jail, chroot=chroot, root=root) - if salt.utils.compare_versions(_get_pkgng_version(jail, chroot, root), '>=', '1.6.0'): + if salt.utils.versions.compare(_get_pkgng_version(jail, chroot, root), '>=', '1.6.0'): quiet = True else: quiet = False @@ -327,7 +330,7 @@ def latest_version(*names, **kwargs): ret[name] = pkgver else: if not any( - (salt.utils.compare_versions(ver1=x, + (salt.utils.versions.compare(ver1=x, oper='>=', ver2=pkgver) for x in installed) diff --git a/salt/modules/pkgutil.py b/salt/modules/pkgutil.py index 9dd294b7e03..9b16ea56fae 100644 --- a/salt/modules/pkgutil.py +++ b/salt/modules/pkgutil.py @@ -16,8 +16,9 @@ import copy # Import salt libs import salt.utils import salt.utils.pkg +import salt.utils.versions from salt.exceptions import CommandExecutionError, MinionError -import salt.ext.six as six +from salt.ext import six # Define the module's virtual name __virtualname__ = 'pkgutil' @@ -27,7 +28,7 @@ def __virtual__(): ''' Set the virtual pkg module if the os is Solaris ''' - if __grains__['os'] == 'Solaris': + if __grains__['os_family'] == 'Solaris': return __virtualname__ return (False, 'The pkgutil execution module cannot be loaded: ' 'only available on Solaris systems.') @@ -227,7 +228,7 @@ def latest_version(*names, **kwargs): if name in names: cver = pkgs.get(name, '') nver = version_rev.split(',')[0] - if not cver or salt.utils.compare_versions(ver1=cver, + if not cver or salt.utils.versions.compare(ver1=cver, oper='<', ver2=nver): # Remove revision for version comparison diff --git a/salt/modules/portage_config.py b/salt/modules/portage_config.py index 5ab26999df4..8441956bc3b 100644 --- a/salt/modules/portage_config.py +++ b/salt/modules/portage_config.py @@ -10,10 +10,10 @@ import os import shutil # Import salt libs -import salt.utils +import salt.utils.files # Import third party libs -import salt.ext.six as six +from salt.ext import six # pylint: disable=import-error try: import portage @@ -75,6 +75,8 @@ def _get_config_file(conf, atom): if parts.cp == '*/*': # parts.repo will be empty if there is no repo part relative_path = parts.repo or "gentoo" + elif str(parts.cp).endswith('/*'): + relative_path = str(parts.cp).split("/")[0] + "_" else: relative_path = os.path.join(*[x for x in os.path.split(parts.cp) if x != '*']) else: @@ -92,9 +94,20 @@ def _p_to_cp(p): Convert a package name or a DEPEND atom to category/package format. Raises an exception if program name is ambiguous. ''' - ret = _porttree().dbapi.xmatch("match-all", p) - if ret: - return portage.cpv_getkey(ret[0]) + try: + ret = portage.dep_getkey(p) + if ret: + return ret + except portage.exception.InvalidAtom: + pass + + try: + ret = _porttree().dbapi.xmatch('bestmatch-visible', p) + if ret: + return portage.dep_getkey(ret) + except portage.exception.InvalidAtom: + pass + return None @@ -160,7 +173,7 @@ def _unify_keywords(): for triplet in os.walk(old_path): for file_name in triplet[2]: file_path = '{0}/{1}'.format(triplet[0], file_name) - with salt.utils.fopen(file_path) as fh_: + with salt.utils.files.fopen(file_path) as fh_: for line in fh_: line = line.strip() if line and not line.startswith('#'): @@ -168,7 +181,7 @@ def _unify_keywords(): 'accept_keywords', string=line) shutil.rmtree(old_path) else: - with salt.utils.fopen(old_path) as fh_: + with salt.utils.files.fopen(old_path) as fh_: for line in fh_: line = line.strip() if line and not line.startswith('#'): @@ -188,12 +201,7 @@ def _package_conf_file_to_dir(file_name): else: os.rename(path, path + '.tmpbak') os.mkdir(path, 0o755) - with salt.utils.fopen(path + '.tmpbak') as fh_: - for line in fh_: - line = line.strip() - if line and not line.startswith('#'): - append_to_package_conf(file_name, string=line) - os.remove(path + '.tmpbak') + os.rename(path + '.tmpbak', os.path.join(path, 'tmp')) return True else: os.mkdir(path, 0o755) @@ -218,13 +226,13 @@ def _package_conf_ordering(conf, clean=True, keep_backup=False): shutil.copy(file_path, file_path + '.bak') backup_files.append(file_path + '.bak') - if cp[0] == '/' or cp.split('/') > 2: - with salt.utils.fopen(file_path) as fp_: + if cp[0] == '/' or len(cp.split('/')) > 2: + with salt.utils.files.fopen(file_path) as fp_: rearrange.extend(fp_.readlines()) os.remove(file_path) else: new_contents = '' - with salt.utils.fopen(file_path, 'r+') as file_handler: + with salt.utils.files.fopen(file_path, 'r+') as file_handler: for line in file_handler: try: atom = line.strip().split()[0] @@ -362,9 +370,9 @@ def append_to_package_conf(conf, atom='', flags=None, string='', overwrite=False pass try: - file_handler = salt.utils.fopen(complete_file_path, 'r+') # pylint: disable=resource-leakage + file_handler = salt.utils.files.fopen(complete_file_path, 'r+') # pylint: disable=resource-leakage except IOError: - file_handler = salt.utils.fopen(complete_file_path, 'w+') # pylint: disable=resource-leakage + file_handler = salt.utils.files.fopen(complete_file_path, 'w+') # pylint: disable=resource-leakage new_contents = '' added = False @@ -465,7 +473,7 @@ def get_flags_from_package_conf(conf, atom): flags = [] try: - with salt.utils.fopen(package_file) as fp_: + with salt.utils.files.fopen(package_file) as fp_: for line in fp_: line = line.strip() line_package = line.split()[0] @@ -563,7 +571,7 @@ def is_present(conf, atom): match_list = set(_porttree().dbapi.xmatch("match-all", atom)) try: - with salt.utils.fopen(package_file) as fp_: + with salt.utils.files.fopen(package_file) as fp_: for line in fp_: line = line.strip() line_package = line.split()[0] diff --git a/salt/modules/postfix.py b/salt/modules/postfix.py index c0d763ee372..10c57722a59 100644 --- a/salt/modules/postfix.py +++ b/salt/modules/postfix.py @@ -18,7 +18,11 @@ import re import logging # Import salt libs -import salt.utils +import salt.utils.files +import salt.utils.path + +# Import 3rd-party libs +from salt.ext import six SWWS = re.compile(r'^\s') @@ -32,7 +36,7 @@ def __virtual__(): ''' Only load the module if Postfix is installed ''' - if salt.utils.which('postfix'): + if salt.utils.path.which('postfix'): return True return (False, 'postfix execution module not loaded: postfix not installed.') @@ -50,7 +54,7 @@ def _parse_master(path=MASTER_CF): Returns a dict of the active config lines, and a list of the entire file, in order. These compliment each other. ''' - with salt.utils.fopen(path, 'r') as fh_: + with salt.utils.files.fopen(path, 'r') as fh_: full_conf = fh_.read() # Condense the file based on line continuations, but keep order, comments @@ -223,7 +227,7 @@ def _parse_main(path=MAIN_CF): * Keys defined in the file may be referred to as variables further down in the file. ''' - with salt.utils.fopen(path, 'r') as fh_: + with salt.utils.files.fopen(path, 'r') as fh_: full_conf = fh_.read() # Condense the file based on line continuations, but keep order, comments @@ -238,7 +242,7 @@ def _parse_main(path=MAIN_CF): # This should only happen at the top of the file conf_list.append(line) continue - if not isinstance(conf_list[-1], str): + if not isinstance(conf_list[-1], six.string_types): conf_list[-1] = '' # This line is a continuation of the previous line conf_list[-1] = '\n'.join([conf_list[-1], line]) @@ -306,7 +310,7 @@ def _write_conf(conf, path=MAIN_CF): ''' Write out configuration file. ''' - with salt.utils.fopen(path, 'w') as fh_: + with salt.utils.files.fopen(path, 'w') as fh_: for line in conf: if isinstance(line, dict): fh_.write(' '.join(line)) diff --git a/salt/modules/postgres.py b/salt/modules/postgres.py index 0f16da9484f..b8b2a12908f 100644 --- a/salt/modules/postgres.py +++ b/salt/modules/postgres.py @@ -47,14 +47,16 @@ except ImportError: HAS_CSV = False # Import salt libs -import salt.utils import salt.utils.files import salt.utils.itertools +import salt.utils.odict +import salt.utils.path +import salt.utils.stringutils from salt.exceptions import CommandExecutionError, SaltInvocationError from salt.utils.versions import LooseVersion as _LooseVersion # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import zip # pylint: disable=import-error,redefined-builtin from salt.ext.six.moves import StringIO @@ -120,7 +122,7 @@ def __virtual__(): if not HAS_CSV: return False for util in utils: - if not salt.utils.which(util): + if not salt.utils.path.which(util): if not _find_pg_binary(util): return (False, '{0} was not found'.format(util)) return True @@ -133,10 +135,10 @@ def _find_pg_binary(util): Helper function to locate various psql related binaries ''' pg_bin_dir = __salt__['config.option']('postgres.bins_dir') - util_bin = salt.utils.which(util) + util_bin = salt.utils.path.which(util) if not util_bin: if pg_bin_dir: - return salt.utils.which(os.path.join(pg_bin_dir, util)) + return salt.utils.path.which(os.path.join(pg_bin_dir, util)) else: return util_bin @@ -171,7 +173,7 @@ def _run_psql(cmd, runas=None, password=None, host=None, port=None, user=None): password = __salt__['config.option']('postgres.pass') if password is not None: pgpassfile = salt.utils.files.mkstemp(text=True) - with salt.utils.fopen(pgpassfile, 'w') as fp_: + with salt.utils.files.fopen(pgpassfile, 'w') as fp_: fp_.write('{0}:{1}:*:{2}:{3}'.format( 'localhost' if not host or host.startswith('/') else host, port if port else '*', @@ -227,7 +229,7 @@ def _run_initdb(name, if password is not None: pgpassfile = salt.utils.files.mkstemp(text=True) - with salt.utils.fopen(pgpassfile, 'w') as fp_: + with salt.utils.files.fopen(pgpassfile, 'w') as fp_: fp_.write('{0}'.format(password)) __salt__['file.chown'](pgpassfile, runas, '') cmd.extend([ @@ -884,8 +886,8 @@ def user_list(user=None, host=None, port=None, maintenance_db=None, for date_key in ('expiry time',): try: retrow[date_key] = datetime.datetime.strptime( - row['date_key'], '%Y-%m-%d %H:%M:%S') - except (ValueError, KeyError): + row[date_key], '%Y-%m-%d %H:%M:%S') + except ValueError: retrow[date_key] = None retrow['defaults variables'] = row['defaults variables'] if return_password: @@ -1005,7 +1007,7 @@ def _maybe_encrypt_password(role, password = str(password) if encrypted and password and not password.startswith('md5'): password = "md5{0}".format( - hashlib.md5(salt.utils.to_bytes('{0}{1}'.format(password, role))).hexdigest()) + hashlib.md5(salt.utils.stringutils.to_bytes('{0}{1}'.format(password, role))).hexdigest()) return password @@ -1023,6 +1025,7 @@ def _role_cmd_args(name, groups=None, replication=None, rolepassword=None, + valid_until=None, db_role=None): if createuser is not None and superuser is None: superuser = createuser @@ -1039,6 +1042,7 @@ def _role_cmd_args(name, encrypted = _DEFAULT_PASSWORDS_ENCRYPTION skip_passwd = False escaped_password = '' + escaped_valid_until = '' if not ( rolepassword is not None # first is passwd set @@ -1056,6 +1060,10 @@ def _role_cmd_args(name, _maybe_encrypt_password(name, rolepassword.replace('\'', '\'\''), encrypted=encrypted)) + if isinstance(valid_until, six.string_types) and bool(valid_until): + escaped_valid_until = '\'{0}\''.format( + valid_until.replace('\'', '\'\''), + ) skip_superuser = False if bool(db_role) and bool(superuser) == bool(db_role['superuser']): skip_superuser = True @@ -1079,6 +1087,10 @@ def _role_cmd_args(name, {'flag': 'PASSWORD', 'test': bool(rolepassword), 'skip': skip_passwd, 'addtxt': escaped_password}, + {'flag': 'VALID UNTIL', + 'test': bool(valid_until), + 'skip': valid_until is None, + 'addtxt': escaped_valid_until}, ) for data in flags: sub_cmd = _add_role_flag(sub_cmd, **data) @@ -1108,6 +1120,7 @@ def _role_create(name, inherit=None, replication=None, rolepassword=None, + valid_until=None, typ_='role', groups=None, runas=None): @@ -1136,7 +1149,8 @@ def _role_create(name, superuser=superuser, groups=groups, replication=replication, - rolepassword=rolepassword + rolepassword=rolepassword, + valid_until=valid_until )) ret = _psql_prepare_and_run(['-c', sub_cmd], runas=runas, host=host, user=user, port=port, @@ -1162,6 +1176,7 @@ def user_create(username, superuser=None, replication=None, rolepassword=None, + valid_until=None, groups=None, runas=None): ''' @@ -1173,7 +1188,7 @@ def user_create(username, salt '*' postgres.user_create 'username' user='user' \\ host='hostname' port='port' password='password' \\ - rolepassword='rolepassword' + rolepassword='rolepassword' valid_until='valid_until' ''' return _role_create(username, typ_='user', @@ -1192,6 +1207,7 @@ def user_create(username, superuser=superuser, replication=replication, rolepassword=rolepassword, + valid_until=valid_until, groups=groups, runas=runas) @@ -1213,6 +1229,7 @@ def _role_update(name, superuser=None, replication=None, rolepassword=None, + valid_until=None, groups=None, runas=None): ''' @@ -1248,6 +1265,7 @@ def _role_update(name, groups=groups, replication=replication, rolepassword=rolepassword, + valid_until=valid_until, db_role=role )) ret = _psql_prepare_and_run(['-c', sub_cmd], @@ -1274,6 +1292,7 @@ def user_update(username, connlimit=None, replication=None, rolepassword=None, + valid_until=None, groups=None, runas=None): ''' @@ -1285,7 +1304,7 @@ def user_update(username, salt '*' postgres.user_update 'username' user='user' \\ host='hostname' port='port' password='password' \\ - rolepassword='rolepassword' + rolepassword='rolepassword' valid_until='valid_until' ''' return _role_update(username, user=user, @@ -1304,6 +1323,7 @@ def user_update(username, superuser=superuser, replication=replication, rolepassword=rolepassword, + valid_until=valid_until, groups=groups, runas=runas) @@ -1954,7 +1974,7 @@ def schema_create(dbname, name, owner=None, ''' # check if schema exists - if schema_exists(dbname, name, + if schema_exists(dbname, name, user=user, db_user=db_user, db_password=db_password, db_host=db_host, db_port=db_port): log.info('\'{0}\' already exists in \'{1}\''.format(name, dbname)) @@ -2009,7 +2029,7 @@ def schema_remove(dbname, name, ''' # check if schema exists - if not schema_exists(dbname, name, + if not schema_exists(dbname, name, user=None, db_user=db_user, db_password=db_password, db_host=db_host, db_port=db_port): log.info('Schema \'{0}\' does not exist in \'{1}\''.format(name, dbname)) @@ -2023,7 +2043,7 @@ def schema_remove(dbname, name, maintenance_db=dbname, host=db_host, user=db_user, port=db_port, password=db_password) - if not schema_exists(dbname, name, + if not schema_exists(dbname, name, user, db_user=db_user, db_password=db_password, db_host=db_host, db_port=db_port): return True @@ -2032,7 +2052,7 @@ def schema_remove(dbname, name, return False -def schema_exists(dbname, name, +def schema_exists(dbname, name, user=None, db_user=None, db_password=None, db_host=None, db_port=None): ''' @@ -2050,6 +2070,9 @@ def schema_exists(dbname, name, name Schema name we look for + user + The system user the operation should be performed on behalf of + db_user database username if different from config or default @@ -2064,14 +2087,14 @@ def schema_exists(dbname, name, ''' return bool( - schema_get(dbname, name, + schema_get(dbname, name, user=user, db_user=db_user, db_host=db_host, db_port=db_port, db_password=db_password)) -def schema_get(dbname, name, +def schema_get(dbname, name, user=None, db_user=None, db_password=None, db_host=None, db_port=None): ''' @@ -2089,6 +2112,9 @@ def schema_get(dbname, name, name Schema name we look for + user + The system user the operation should be performed on behalf of + db_user database username if different from config or default @@ -2101,7 +2127,7 @@ def schema_get(dbname, name, db_port Database port if different from config or default ''' - all_schemas = schema_list(dbname, + all_schemas = schema_list(dbname, user=user, db_user=db_user, db_host=db_host, db_port=db_port, @@ -2113,7 +2139,7 @@ def schema_get(dbname, name, return False -def schema_list(dbname, +def schema_list(dbname, user=None, db_user=None, db_password=None, db_host=None, db_port=None): ''' @@ -2128,6 +2154,9 @@ def schema_list(dbname, dbname Database name we query on + user + The system user the operation should be performed on behalf of + db_user database username if different from config or default @@ -2152,7 +2181,7 @@ def schema_list(dbname, 'LEFT JOIN pg_roles ON pg_roles.oid = pg_namespace.nspowner ' ])) - rows = psql_query(query, + rows = psql_query(query, runas=user, host=db_host, user=db_user, port=db_port, diff --git a/salt/modules/poudriere.py b/salt/modules/poudriere.py index 6dd9f46c532..ac4215c3b9e 100644 --- a/salt/modules/poudriere.py +++ b/salt/modules/poudriere.py @@ -9,7 +9,8 @@ import os import logging # Import salt libs -import salt.utils +import salt.utils.files +import salt.utils.path log = logging.getLogger(__name__) @@ -18,7 +19,7 @@ def __virtual__(): ''' Module load on freebsd only and if poudriere installed ''' - if __grains__['os'] == 'FreeBSD' and salt.utils.which('poudriere'): + if __grains__['os'] == 'FreeBSD' and salt.utils.path.which('poudriere'): return 'poudriere' else: return (False, 'The poudriere execution module failed to load: only available on FreeBSD with the poudriere binary in the path.') @@ -115,7 +116,7 @@ def parse_config(config_file=None): config_file = _config_file() ret = {} if _check_config_exists(config_file): - with salt.utils.fopen(config_file) as ifile: + with salt.utils.files.fopen(config_file) as ifile: for line in ifile: key, val = line.split('=') ret[key] = val diff --git a/salt/modules/proxy.py b/salt/modules/proxy.py index d5e88634e42..2bdd0c92837 100644 --- a/salt/modules/proxy.py +++ b/salt/modules/proxy.py @@ -12,8 +12,8 @@ from __future__ import absolute_import import logging import re -# Import salt libs -import salt.utils +# Import Salt libs +import salt.utils.platform log = logging.getLogger(__name__) __virtualname__ = 'proxy' @@ -23,7 +23,7 @@ def __virtual__(): ''' Only work on Mac OS and Windows ''' - if salt.utils.is_darwin() or salt.utils.is_windows(): + if salt.utils.platform.is_darwin() or salt.utils.platform.is_windows(): return True return (False, 'Module proxy: module only works on Windows or MacOS systems') diff --git a/salt/modules/ps.py b/salt/modules/ps.py index 48087b11fc8..b2625c48114 100644 --- a/salt/modules/ps.py +++ b/salt/modules/ps.py @@ -17,7 +17,8 @@ import re from salt.exceptions import SaltInvocationError, CommandExecutionError # Import third party libs -import salt.ext.six as six +import salt.utils.decorators.path +from salt.ext import six # pylint: disable=import-error try: import salt.utils.psutil_compat as psutil @@ -655,6 +656,7 @@ def lsof(name): return ret +@salt.utils.decorators.path.which('netstat') def netstat(name): ''' Retrieve the netstat information of the given process name. @@ -676,6 +678,31 @@ def netstat(name): return ret +@salt.utils.decorators.path.which('ss') +def ss(name): + ''' + Retrieve the ss information of the given process name. + + CLI Example: + + .. code-block:: bash + + salt '*' ps.ss apache2 + + .. versionadded:: 2016.11.6 + + ''' + sanitize_name = str(name) + ss_infos = __salt__['cmd.run']("ss -neap") + found_infos = [] + ret = [] + for info in ss_infos.splitlines(): + if info.find(sanitize_name) != -1: + found_infos.append(info) + ret.extend([sanitize_name, found_infos]) + return ret + + def psaux(name): ''' Retrieve information corresponding to a "ps aux" filtered diff --git a/salt/modules/publish.py b/salt/modules/publish.py index cd43ead5f53..680e23dcd24 100644 --- a/salt/modules/publish.py +++ b/salt/modules/publish.py @@ -13,6 +13,7 @@ import salt.crypt import salt.payload import salt.transport import salt.utils.args +import salt.utils.versions from salt.exceptions import SaltReqTimeoutError, SaltInvocationError log = logging.getLogger(__name__) @@ -252,7 +253,7 @@ def publish(tgt, # remember to remove the expr_form argument from this function when # performing the cleanup on this deprecation. if expr_form is not None: - salt.utils.warn_until( + salt.utils.versions.warn_until( 'Fluorine', 'the target type should be passed using the \'tgt_type\' ' 'argument instead of \'expr_form\'. Support for using ' @@ -302,7 +303,7 @@ def full_data(tgt, # remember to remove the expr_form argument from this function when # performing the cleanup on this deprecation. if expr_form is not None: - salt.utils.warn_until( + salt.utils.versions.warn_until( 'Fluorine', 'the target type should be passed using the \'tgt_type\' ' 'argument instead of \'expr_form\'. Support for using ' diff --git a/salt/modules/puppet.py b/salt/modules/puppet.py index 0b49f9e3653..58b3963c8cd 100644 --- a/salt/modules/puppet.py +++ b/salt/modules/puppet.py @@ -11,12 +11,15 @@ import os import datetime # Import salt libs -import salt.utils +import salt.utils.args +import salt.utils.files +import salt.utils.path +import salt.utils.platform from salt.exceptions import CommandExecutionError # Import 3rd-party libs import yaml -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import range log = logging.getLogger(__name__) @@ -26,7 +29,7 @@ def __virtual__(): Only load if puppet is installed ''' unavailable_exes = ', '.join(exe for exe in ('facter', 'puppet') - if salt.utils.which(exe) is None) + if salt.utils.path.which(exe) is None) if unavailable_exes: return (False, ('The puppet execution module cannot be loaded: ' @@ -61,7 +64,7 @@ class _Puppet(object): self.kwargs = {'color': 'false'} # e.g. --tags=apache::server self.args = [] # e.g. --noop - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): self.vardir = 'C:\\ProgramData\\PuppetLabs\\puppet\\var' self.rundir = 'C:\\ProgramData\\PuppetLabs\\puppet\\run' self.confdir = 'C:\\ProgramData\\PuppetLabs\\puppet\\etc' @@ -164,7 +167,7 @@ def run(*args, **kwargs): # args will exist as an empty list even if none have been provided puppet.arguments(buildargs) - puppet.kwargs.update(salt.utils.clean_kwargs(**kwargs)) + puppet.kwargs.update(salt.utils.args.clean_kwargs(**kwargs)) ret = __salt__['cmd.run_all'](repr(puppet), python_shell=puppet.useshell) if ret['retcode'] in [0, 2]: @@ -243,7 +246,7 @@ def disable(message=None): if os.path.isfile(puppet.disabled_lockfile): return False else: - with salt.utils.fopen(puppet.disabled_lockfile, 'w') as lockfile: + with salt.utils.files.fopen(puppet.disabled_lockfile, 'w') as lockfile: try: # Puppet chokes when no valid json is found str = '{{"disabled_message":"{0}"}}'.format(message) if message is not None else '{}' @@ -275,7 +278,7 @@ def status(): if os.path.isfile(puppet.run_lockfile): try: - with salt.utils.fopen(puppet.run_lockfile, 'r') as fp_: + with salt.utils.files.fopen(puppet.run_lockfile, 'r') as fp_: pid = int(fp_.read()) os.kill(pid, 0) # raise an OSError if process doesn't exist except (OSError, ValueError): @@ -285,7 +288,7 @@ def status(): if os.path.isfile(puppet.agent_pidfile): try: - with salt.utils.fopen(puppet.agent_pidfile, 'r') as fp_: + with salt.utils.files.fopen(puppet.agent_pidfile, 'r') as fp_: pid = int(fp_.read()) os.kill(pid, 0) # raise an OSError if process doesn't exist except (OSError, ValueError): @@ -312,7 +315,7 @@ def summary(): puppet = _Puppet() try: - with salt.utils.fopen(puppet.lastrunfile, 'r') as fp_: + with salt.utils.files.fopen(puppet.lastrunfile, 'r') as fp_: report = yaml.safe_load(fp_.read()) result = {} @@ -345,9 +348,10 @@ def summary(): def plugin_sync(): ''' - Runs a plugin synch between the puppet master and agent + Runs a plugin sync between the puppet master and agent CLI Example: + .. code-block:: bash salt '*' puppet.plugin_sync diff --git a/salt/modules/purefa.py b/salt/modules/purefa.py new file mode 100644 index 00000000000..14beb37bef2 --- /dev/null +++ b/salt/modules/purefa.py @@ -0,0 +1,1242 @@ +# -*- coding: utf-8 -*- + +## +# Copyright 2017 Pure Storage Inc +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +''' + +Management of Pure Storage FlashArray + +Installation Prerequisites +-------------------------- +- You will need the ``purestorage`` python package in your python installation + path that is running salt. + + .. code-block:: bash + + pip install purestorage + +:maintainer: Simon Dodsley (simon@purestorage.com) +:maturity: new +:requires: purestorage +:platform: all + +.. versionadded:: Oxygen + +''' + +# Import Python libs +from __future__ import absolute_import +import os +import platform +from datetime import datetime + +# Import Salt libs +from salt.exceptions import CommandExecutionError + +# Import 3rd party modules +try: + import purestorage + HAS_PURESTORAGE = True +except ImportError: + HAS_PURESTORAGE = False + +__docformat__ = 'restructuredtext en' + +VERSION = '1.0.0' +USER_AGENT_BASE = 'Salt' + +__virtualname__ = 'purefa' + +# Default symbols to use for passwords. Avoids visually confusing characters. +# ~6 bits per symbol +DEFAULT_PASSWORD_SYMBOLS = ('23456789', # Removed: 0,1 + 'ABCDEFGHJKLMNPQRSTUVWXYZ', # Removed: I, O + 'abcdefghijkmnopqrstuvwxyz') # Removed: l + + +def __virtual__(): + ''' + Determine whether or not to load this module + ''' + if HAS_PURESTORAGE: + return __virtualname__ + return (False, 'purefa execution module not loaded: purestorage python library not available.') + + +def _get_system(): + ''' + Get Pure Storage FlashArray configuration + + 1) From the minion config + pure_tags: + fa: + san_ip: management vip or hostname for the FlashArray + api_token: A valid api token for the FlashArray being managed + 2) From environment (PUREFA_IP and PUREFA_API) + 3) From the pillar (PUREFA_IP and PUREFA_API) + + ''' + agent = {'base': USER_AGENT_BASE, + 'class': __name__, + 'version': VERSION, + 'platform': platform.platform() + } + + user_agent = '{base} {class}/{version} ({platform})'.format(**agent) + + try: + array = __opts__['pure_tags']['fa'].get('san_ip') + api = __opts__['pure_tags']['fa'].get('api_token') + if array and api: + system = purestorage.FlashArray(array, api_token=api, user_agent=user_agent) + except (KeyError, NameError, TypeError): + try: + san_ip = os.environ.get('PUREFA_IP') + api_token = os.environ.get('PUREFA_API') + system = purestorage.FlashArray(san_ip, + api_token=api_token, + user_agent=user_agent) + except (ValueError, KeyError, NameError): + try: + system = purestorage.FlashArray(__pillar__['PUREFA_IP'], + api_token=__pillar__['PUREFA_API'], + user_agent=user_agent) + except (KeyError, NameError): + raise CommandExecutionError('No Pure Storage FlashArray credentials found.') + + try: + system.get() + except Exception: + raise CommandExecutionError('Pure Storage FlashArray authentication failed.') + return system + + +def _get_volume(name, array): + '''Private function to check volume''' + try: + return array.get_volume(name) + except purestorage.PureError: + return None + + +def _get_snapshot(name, suffix, array): + '''Private function to check snapshot''' + snapshot = name + '.' + suffix + try: + for snap in array.get_volume(name, snap=True): + if snap['name'] == snapshot: + return snapshot + except purestorage.PureError: + return None + + +def _get_deleted_volume(name, array): + '''Private function to check deleted volume''' + try: + return array.get_volume(name, pending='true') + except purestorage.PureError: + return None + + +def _get_pgroup(name, array): + '''Private function to check protection group''' + pgroup = None + for temp in array.list_pgroups(): + if temp['name'] == name: + pgroup = temp + break + return pgroup + + +def _get_deleted_pgroup(name, array): + '''Private function to check deleted protection group''' + try: + return array.get_pgroup(name, pending='true') + except purestorage.PureError: + return None + + +def _get_hgroup(name, array): + '''Private function to check hostgroup''' + hostgroup = None + for temp in array.list_hgroups(): + if temp['name'] == name: + hostgroup = temp + break + return hostgroup + + +def _get_host(name, array): + '''Private function to check host''' + host = None + for temp in array.list_hosts(): + if temp['name'] == name: + host = temp + break + return host + + +def snap_create(name, suffix=None): + ''' + + Create a volume snapshot on a Pure Storage FlashArray. + + Will return False is volume selected to snap does not exist. + + .. versionadded:: 2017.7.3 + + name : string + name of volume to snapshot + suffix : string + if specificed forces snapshot name suffix. If not specified defaults to timestamp. + + CLI Example: + + .. code-block:: bash + + salt '*' purefa.snap_create foo + salt '*' purefa.snap_create foo suffix=bar + + ''' + array = _get_system() + if suffix is None: + suffix = 'snap-' + str((datetime.utcnow() - datetime(1970, 1, 1, 0, 0, 0, 0)).total_seconds()) + suffix = suffix.replace('.', '') + if _get_volume(name, array) is not None: + try: + array.create_snapshot(name, suffix=suffix) + return True + except purestorage.PureError: + return False + else: + return False + + +def snap_delete(name, suffix=None, eradicate=False): + ''' + + Delete a volume snapshot on a Pure Storage FlashArray. + + Will return False if selected snapshot does not exist. + + .. versionadded:: 2017.7.3 + + name : string + name of volume + suffix : string + name of snapshot + eradicate : boolean + Eradicate snapshot after deletion if True. Default is False + + CLI Example: + + .. code-block:: bash + + salt '*' purefa.snap_delete foo suffix=snap eradicate=True + + ''' + array = _get_system() + if _get_snapshot(name, suffix, array) is not None: + try: + snapname = name + '.' + suffix + array.destroy_volume(snapname) + except purestorage.PureError: + return False + if eradicate is True: + try: + array.eradicate_volume(snapname) + return True + except purestorage.PureError: + return False + else: + return True + else: + return False + + +def snap_eradicate(name, suffix=None): + ''' + + Eradicate a deleted volume snapshot on a Pure Storage FlashArray. + + Will retunr False is snapshot is not in a deleted state. + + .. versionadded:: 2017.7.3 + + name : string + name of volume + suffix : string + name of snapshot + + CLI Example: + + .. code-block:: bash + + salt '*' purefa.snap_delete foo suffix=snap eradicate=True + + ''' + array = _get_system() + if _get_snapshot(name, suffix, array) is not None: + snapname = name + '.' + suffix + try: + array.eradicate_volume(snapname) + return True + except purestorage.PureError: + return False + else: + return False + + +def volume_create(name, size=None): + ''' + + Create a volume on a Pure Storage FlashArray. + + Will return False if volume already exists. + + .. versionadded:: 2017.7.3 + + name : string + name of volume (truncated to 63 characters) + size : string + if specificed capacity of volume. If not specified default to 1G. + Refer to Pure Storage documentation for formatting rules. + + CLI Example: + + .. code-block:: bash + + salt '*' purefa.volume_create foo + salt '*' purefa.volume_create foo size=10T + + ''' + if len(name) > 63: + name = name[0:63] + array = _get_system() + if _get_volume(name, array) is None: + if size is None: + size = '1G' + try: + array.create_volume(name, size) + return True + except purestorage.PureError: + return False + else: + return False + + +def volume_delete(name, eradicate=False): + ''' + + Delete a volume on a Pure Storage FlashArray. + + Will return False if volume doesn't exist is already in a deleted state. + + .. versionadded:: 2017.7.3 + + name : string + name of volume + eradicate : boolean + Eradicate volume after deletion if True. Default is False + + CLI Example: + + .. code-block:: bash + + salt '*' purefa.volume_delete foo eradicate=True + + ''' + array = _get_system() + if _get_volume(name, array) is not None: + try: + array.destroy_volume(name) + except purestorage.PureError: + return False + if eradicate is True: + try: + array.eradicate_volume(name) + return True + except purestorage.PureError: + return False + else: + return True + else: + return False + + +def volume_eradicate(name): + ''' + + Eradicate a deleted volume on a Pure Storage FlashArray. + + Will return False is volume is not in a deleted state. + + .. versionadded:: 2017.7.3 + + name : string + name of volume + + CLI Example: + + .. code-block:: bash + + salt '*' purefa.volume_eradicate foo + + ''' + array = _get_system() + if _get_deleted_volume(name, array) is not None: + try: + array.eradicate_volume(name) + return True + except purestorage.PureError: + return False + else: + return False + + +def volume_extend(name, size): + ''' + + Extend an existing volume on a Pure Storage FlashArray. + + Will return False if new size is less than or equal to existing size. + + .. versionadded:: 2017.7.3 + + name : string + name of volume + size : string + New capacity of volume. + Refer to Pure Storage documentation for formatting rules. + + CLI Example: + + .. code-block:: bash + + salt '*' purefa.volume_extend foo 10T + + ''' + array = _get_system() + vol = _get_volume(name, array) + if vol is not None: + if __utils__['stringutils.human_to_bytes'](size) > vol['size']: + try: + array.extend_volume(name, size) + return True + except purestorage.PureError: + return False + else: + return False + else: + return False + + +def snap_volume_create(name, target, overwrite=False): + ''' + + Create R/W volume from snapshot on a Pure Storage FlashArray. + + Will return False if target volume already exists and + overwrite is not specified, or selected snapshot doesn't exist. + + .. versionadded:: 2017.7.3 + + name : string + name of volume snapshot + target : string + name of clone volume + overwrite : boolean + overwrite clone if already exists (default: False) + + CLI Example: + + .. code-block:: bash + + salt '*' purefa.snap_volume_create foo.bar clone overwrite=True + + ''' + array = _get_system() + source, suffix = name.split('.') + if _get_snapshot(source, suffix, array) is not None: + if _get_volume(target, array) is None: + try: + array.copy_volume(name, target) + return True + except purestorage.PureError: + return False + else: + if overwrite: + try: + array.copy_volume(name, target, overwrite=overwrite) + return True + except purestorage.PureError: + return False + else: + return False + else: + return False + + +def volume_clone(name, target, overwrite=False): + ''' + + Clone an existing volume on a Pure Storage FlashArray. + + Will return False if source volume doesn't exist, or + target volume already exists and overwrite not specified. + + .. versionadded:: 2017.7.3 + + name : string + name of volume + target : string + name of clone volume + overwrite : boolean + overwrite clone if already exists (default: False) + + CLI Example: + + .. code-block:: bash + + salt '*' purefa.volume_clone foo bar overwrite=True + + ''' + array = _get_system() + if _get_volume(name, array) is not None: + if _get_volume(target, array) is None: + try: + array.copy_volume(name, target) + return True + except purestorage.PureError: + return False + else: + if overwrite: + try: + array.copy_volume(name, target, overwrite=overwrite) + return True + except purestorage.PureError: + return False + else: + return False + else: + return False + + +def volume_attach(name, host): + ''' + + Attach a volume to a host on a Pure Storage FlashArray. + + Host and volume must exist or else will return False. + + .. versionadded:: 2017.7.3 + + name : string + name of volume + host : string + name of host + + CLI Example: + + .. code-block:: bash + + salt '*' purefa.volume_attach foo bar + + ''' + array = _get_system() + if _get_volume(name, array) is not None and _get_host(host, array) is not None: + try: + array.connect_host(host, name) + return True + except purestorage.PureError: + return False + else: + return False + + +def volume_detach(name, host): + ''' + + Detach a volume from a host on a Pure Storage FlashArray. + + Will return False if either host or volume do not exist, or + if selected volume isn't already connected to the host. + + .. versionadded:: 2017.7.3 + + name : string + name of volume + host : string + name of host + + CLI Example: + + .. code-block:: bash + + salt '*' purefa.volume_detach foo bar + + ''' + array = _get_system() + if _get_volume(name, array) is None or _get_host(host, array) is None: + return False + elif _get_volume(name, array) is not None and _get_host(host, array) is not None: + try: + array.disconnect_host(host, name) + return True + except purestorage.PureError: + return False + + +def host_create(name, iqn=None, wwn=None): + ''' + + Add a host on a Pure Storage FlashArray. + + Will return False if host already exists, or the iSCSI or + Fibre Channel parameters are not in a valid format. + See Pure Storage FlashArray documentation. + + .. versionadded:: 2017.7.3 + + name : string + name of host (truncated to 63 characters) + iqn : string + iSCSI IQN of host + wwn : string + Fibre Channel WWN of host + + CLI Example: + + .. code-block:: bash + + salt '*' purefa.host_create foo iqn='' wwn='' + + ''' + array = _get_system() + if len(name) > 63: + name = name[0:63] + if _get_host(name, array) is None: + try: + array.create_host(name) + except purestorage.PureError: + return False + if iqn is not None: + try: + array.set_host(name, addiqnlist=[iqn]) + except purestorage.PureError: + array.delete_host(name) + return False + if wwn is not None: + try: + array.set_host(name, addwwnlist=[wwn]) + except purestorage.PureError: + array.delete_host(name) + return False + else: + return False + + return True + + +def host_update(name, iqn=None, wwn=None): + ''' + + Update a hosts port definitions on a Pure Storage FlashArray. + + Will return False if new port definitions are already in use + by another host, or are not in a valid format. + See Pure Storage FlashArray documentation. + + .. versionadded:: 2017.7.3 + + name : string + name of host + iqn : string + Additional iSCSI IQN of host + wwn : string + Additional Fibre Channel WWN of host + + CLI Example: + + .. code-block:: bash + + salt '*' purefa.host_update foo iqn='' wwn='' + + ''' + array = _get_system() + if _get_host(name, array) is not None: + if iqn is not None: + try: + array.set_host(name, addiqnlist=[iqn]) + except purestorage.PureError: + return False + if wwn is not None: + try: + array.set_host(name, addwwnlist=[wwn]) + except purestorage.PureError: + return False + return True + else: + return False + + +def host_delete(name): + ''' + + Delete a host on a Pure Storage FlashArray (detaches all volumes). + + Will return False if the host doesn't exist. + + .. versionadded:: 2017.7.3 + + name : string + name of host + + CLI Example: + + .. code-block:: bash + + salt '*' purefa.host_delete foo + + ''' + array = _get_system() + if _get_host(name, array) is not None: + for vol in array.list_host_connections(name): + try: + array.disconnect_host(name, vol['vol']) + except purestorage.PureError: + return False + try: + array.delete_host(name) + return True + except purestorage.PureError: + return False + else: + return False + + +def hg_create(name, host=None, volume=None): + ''' + + Create a hostgroup on a Pure Storage FlashArray. + + Will return False if hostgroup already exists, or if + named host or volume do not exist. + + .. versionadded:: 2017.7.3 + + name : string + name of hostgroup (truncated to 63 characters) + host : string + name of host to add to hostgroup + volume : string + name of volume to add to hostgroup + + CLI Example: + + .. code-block:: bash + + salt '*' purefa.hg_create foo host=bar volume=vol + + ''' + array = _get_system() + if len(name) > 63: + name = name[0:63] + if _get_hgroup(name, array) is None: + try: + array.create_hgroup(name) + except purestorage.PureError: + return False + if host is not None: + if _get_host(host, array): + try: + array.set_hgroup(name, addhostlist=[host]) + except purestorage.PureError: + return False + else: + hg_delete(name) + return False + if volume is not None: + if _get_volume(volume, array): + try: + array.connect_hgroup(name, volume) + except purestorage.PureError: + hg_delete(name) + return False + else: + hg_delete(name) + return False + return True + else: + return False + + +def hg_update(name, host=None, volume=None): + ''' + + Adds entries to a hostgroup on a Pure Storage FlashArray. + + Will return False is hostgroup doesn't exist, or host + or volume do not exist. + + .. versionadded:: 2017.7.3 + + name : string + name of hostgroup + host : string + name of host to add to hostgroup + volume : string + name of volume to add to hostgroup + + CLI Example: + + .. code-block:: bash + + salt '*' purefa.hg_update foo host=bar volume=vol + + ''' + array = _get_system() + if _get_hgroup(name, array) is not None: + if host is not None: + if _get_host(host, array): + try: + array.set_hgroup(name, addhostlist=[host]) + except purestorage.PureError: + return False + else: + return False + if volume is not None: + if _get_volume(volume, array): + try: + array.connect_hgroup(name, volume) + except purestorage.PureError: + return False + else: + return False + return True + else: + return False + + +def hg_delete(name): + ''' + + Delete a hostgroup on a Pure Storage FlashArray (removes all volumes and hosts). + + Will return False is hostgroup is already in a deleted state. + + .. versionadded:: 2017.7.3 + + name : string + name of hostgroup + + CLI Example: + + .. code-block:: bash + + salt '*' purefa.hg_delete foo + + ''' + array = _get_system() + if _get_hgroup(name, array) is not None: + for vol in array.list_hgroup_connections(name): + try: + array.disconnect_hgroup(name, vol['vol']) + except purestorage.PureError: + return False + host = array.get_hgroup(name) + try: + array.set_hgroup(name, remhostlist=host['hosts']) + array.delete_hgroup(name) + return True + except purestorage.PureError: + return False + else: + return False + + +def hg_remove(name, volume=None, host=None): + ''' + + Remove a host and/or volume from a hostgroup on a Pure Storage FlashArray. + + Will return False is hostgroup does not exist, or named host or volume are + not in the hostgroup. + + .. versionadded:: 2017.7.3 + + name : string + name of hostgroup + volume : string + name of volume to remove from hostgroup + host : string + name of host to remove from hostgroup + + CLI Example: + + .. code-block:: bash + + salt '*' purefa.hg_remove foo volume=test host=bar + + ''' + array = _get_system() + if _get_hgroup(name, array) is not None: + if volume is not None: + if _get_volume(volume, array): + for temp in array.list_hgroup_connections(name): + if temp['vol'] == volume: + try: + array.disconnect_hgroup(name, volume) + return True + except purestorage.PureError: + return False + return False + else: + return False + if host is not None: + if _get_host(host, array): + temp = _get_host(host, array) + if temp['hgroup'] == name: + try: + array.set_hgroup(name, remhostlist=[host]) + return True + except purestorage.PureError: + return False + else: + return False + else: + return False + if host is None and volume is None: + return False + else: + return False + + +def pg_create(name, hostgroup=None, host=None, volume=None, enabled=True): + ''' + + Create a protection group on a Pure Storage FlashArray. + + Will return False is the following cases: + * Protection Grop already exists + * Protection Group in a deleted state + * More than one type is specified - protection groups are for only + hostgroups, hosts or volumes + * Named type for protection group does not exist + + .. versionadded:: 2017.7.3 + + name : string + name of protection group + hostgroup : string + name of hostgroup to add to protection group + host : string + name of host to add to protection group + volume : string + name of volume to add to protection group + + CLI Example: + + .. code-block:: bash + + salt '*' purefa.pg_create foo [hostgroup=foo | host=bar | volume=vol] enabled=[true | false] + + ''' + array = _get_system() + if hostgroup is None and host is None and volume is None: + if _get_pgroup(name, array) is None: + try: + array.create_pgroup(name) + except purestorage.PureError: + return False + try: + array.set_pgroup(name, snap_enabled=enabled) + return True + except purestorage.PureError: + pg_delete(name) + return False + else: + return False + elif __utils__['value.xor'](hostgroup, host, volume): + if _get_pgroup(name, array) is None: + try: + array.create_pgroup(name) + except purestorage.PureError: + return False + try: + array.set_pgroup(name, snap_enabled=enabled) + except purestorage.PureError: + pg_delete(name) + return False + if hostgroup is not None: + if _get_hgroup(hostgroup, array) is not None: + try: + array.set_pgroup(name, addhgrouplist=[hostgroup]) + return True + except purestorage.PureError: + pg_delete(name) + return False + else: + pg_delete(name) + return False + elif host is not None: + if _get_host(host, array) is not None: + try: + array.set_pgroup(name, addhostlist=[host]) + return True + except purestorage.PureError: + pg_delete(name) + return False + else: + pg_delete(name) + return False + elif volume is not None: + if _get_volume(volume, array) is not None: + try: + array.set_pgroup(name, addvollist=[volume]) + return True + except purestorage.PureError: + pg_delete(name) + return False + else: + pg_delete(name) + return False + else: + return False + else: + return False + + +def pg_update(name, hostgroup=None, host=None, volume=None): + ''' + + Update a protection group on a Pure Storage FlashArray. + + Will return False in the following cases: + * Protection group does not exist + * Incorrect type selected for current protection group type + * Specified type does not exist + + .. versionadded:: 2017.7.3 + + name : string + name of protection group + hostgroup : string + name of hostgroup to add to protection group + host : string + name of host to add to protection group + volume : string + name of volume to add to protection group + + CLI Example: + + .. code-block:: bash + + salt '*' purefa.pg_update foo [hostgroup=foo | host=bar | volume=vol] + + ''' + array = _get_system() + pgroup = _get_pgroup(name, array) + if pgroup is not None: + if hostgroup is not None and pgroup['hgroups'] is not None: + if _get_hgroup(hostgroup, array) is not None: + try: + array.add_hgroup(hostgroup, name) + return True + except purestorage.PureError: + return False + else: + return False + elif host is not None and pgroup['hosts'] is not None: + if _get_host(host, array) is not None: + try: + array.add_host(host, name) + return True + except purestorage.PureError: + return False + else: + return False + elif volume is not None and pgroup['volumes'] is not None: + if _get_volume(volume, array) is not None: + try: + array.add_volume(volume, name) + return True + except purestorage.PureError: + return False + else: + return False + else: + if pgroup['hgroups'] is None and pgroup['hosts'] is None and pgroup['volumes'] is None: + if hostgroup is not None: + if _get_hgroup(hostgroup, array) is not None: + try: + array.set_pgroup(name, addhgrouplist=[hostgroup]) + return True + except purestorage.PureError: + return False + else: + return False + elif host is not None: + if _get_host(host, array) is not None: + try: + array.set_pgroup(name, addhostlist=[host]) + return True + except purestorage.PureError: + return False + else: + return False + elif volume is not None: + if _get_volume(volume, array) is not None: + try: + array.set_pgroup(name, addvollist=[volume]) + return True + except purestorage.PureError: + return False + else: + return False + else: + return False + else: + return False + + +def pg_delete(name, eradicate=False): + ''' + + Delete a protecton group on a Pure Storage FlashArray. + + Will return False if protection group is already in a deleted state. + + .. versionadded:: 2017.7.3 + + name : string + name of protection group + + CLI Example: + + .. code-block:: bash + + salt '*' purefa.pg_delete foo + + ''' + array = _get_system() + if _get_pgroup(name, array) is not None: + try: + array.destroy_pgroup(name) + except purestorage.PureError: + return False + if eradicate is True: + try: + array.eradicate_pgroup(name) + return True + except purestorage.PureError: + return False + else: + return True + else: + return False + + +def pg_eradicate(name): + ''' + + Eradicate a deleted protecton group on a Pure Storage FlashArray. + + Will return False if protection group is not in a deleted state. + + .. versionadded:: 2017.7.3 + + name : string + name of protection group + + CLI Example: + + .. code-block:: bash + + salt '*' purefa.pg_eradicate foo + + ''' + array = _get_system() + if _get_deleted_pgroup(name, array) is not None: + try: + array.eradicate_pgroup(name) + return True + except purestorage.PureError: + return False + else: + return False + + +def pg_remove(name, hostgroup=None, host=None, volume=None): + ''' + + Remove a hostgroup, host or volume from a protection group on a Pure Storage FlashArray. + + Will return False in the following cases: + * Protection group does not exist + * Specified type is not currently associated with the protection group + + .. versionadded:: 2017.7.3 + + name : string + name of hostgroup + hostgroup : string + name of hostgroup to remove from protection group + host : string + name of host to remove from hostgroup + volume : string + name of volume to remove from hostgroup + + CLI Example: + + .. code-block:: bash + + salt '*' purefa.pg_remove foo [hostgroup=bar | host=test | volume=bar] + + ''' + array = _get_system() + pgroup = _get_pgroup(name, array) + if pgroup is not None: + if hostgroup is not None and pgroup['hgroups'] is not None: + if _get_hgroup(hostgroup, array) is not None: + try: + array.remove_hgroup(hostgroup, name) + return True + except purestorage.PureError: + return False + else: + return False + elif host is not None and pgroup['hosts'] is not None: + if _get_host(host, array) is not None: + try: + array.remove_host(host, name) + return True + except purestorage.PureError: + return False + else: + return False + elif volume is not None and pgroup['volumes'] is not None: + if _get_volume(volume, array) is not None: + try: + array.remove_volume(volume, name) + return True + except purestorage.PureError: + return False + else: + return False + else: + return False + else: + return False diff --git a/salt/modules/pw_group.py b/salt/modules/pw_group.py index 182abb54704..143b3a1495e 100644 --- a/salt/modules/pw_group.py +++ b/salt/modules/pw_group.py @@ -15,6 +15,7 @@ import logging # Import salt libs import salt.utils +import salt.utils.args log = logging.getLogger(__name__) @@ -49,7 +50,7 @@ def add(name, gid=None, **kwargs): salt '*' group.add foo 3456 ''' - kwargs = salt.utils.clean_kwargs(**kwargs) + kwargs = salt.utils.args.clean_kwargs(**kwargs) if salt.utils.is_true(kwargs.pop('system', False)): log.warning('pw_group module does not support the \'system\' argument') if kwargs: diff --git a/salt/modules/pw_user.py b/salt/modules/pw_user.py index 41fbd181cfa..c63b97b7b1b 100644 --- a/salt/modules/pw_user.py +++ b/salt/modules/pw_user.py @@ -44,12 +44,13 @@ except ImportError: HAS_PWD = False # Import 3rd party libs -import salt.ext.six as six +from salt.ext import six # Import salt libs import salt.utils +import salt.utils.args +import salt.utils.locales from salt.exceptions import CommandExecutionError -from salt.utils import locales log = logging.getLogger(__name__) @@ -82,10 +83,10 @@ def _get_gecos(name): # Assign empty strings for any unspecified trailing GECOS fields while len(gecos_field) < 4: gecos_field.append('') - return {'fullname': locales.sdecode(gecos_field[0]), - 'roomnumber': locales.sdecode(gecos_field[1]), - 'workphone': locales.sdecode(gecos_field[2]), - 'homephone': locales.sdecode(gecos_field[3])} + return {'fullname': salt.utils.locales.sdecode(gecos_field[0]), + 'roomnumber': salt.utils.locales.sdecode(gecos_field[1]), + 'workphone': salt.utils.locales.sdecode(gecos_field[2]), + 'homephone': salt.utils.locales.sdecode(gecos_field[3])} def _build_gecos(gecos_dict): @@ -141,7 +142,7 @@ def add(name, salt '*' user.add name ''' - kwargs = salt.utils.clean_kwargs(**kwargs) + kwargs = salt.utils.args.clean_kwargs(**kwargs) if salt.utils.is_true(kwargs.pop('system', False)): log.warning('pw_user module does not support the \'system\' argument') if kwargs: diff --git a/salt/modules/qemu_img.py b/salt/modules/qemu_img.py index 9e37646309c..8c947d78c9d 100644 --- a/salt/modules/qemu_img.py +++ b/salt/modules/qemu_img.py @@ -14,13 +14,14 @@ import os # Import salt libs import salt.utils +import salt.utils.path def __virtual__(): ''' Only load if qemu-img is installed ''' - if salt.utils.which('qemu-img'): + if salt.utils.path.which('qemu-img'): return 'qemu_img' return (False, 'The qemu_img execution module cannot be loaded: the qemu-img binary is not in the path.') diff --git a/salt/modules/qemu_nbd.py b/salt/modules/qemu_nbd.py index b30135ce45d..f5ffa3b36b3 100644 --- a/salt/modules/qemu_nbd.py +++ b/salt/modules/qemu_nbd.py @@ -16,10 +16,11 @@ import logging # Import salt libs import salt.utils +import salt.utils.path import salt.crypt # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six # Set up logging log = logging.getLogger(__name__) @@ -29,7 +30,7 @@ def __virtual__(): ''' Only load if qemu-img and qemu-nbd are installed ''' - if salt.utils.which('qemu-nbd'): + if salt.utils.path.which('qemu-nbd'): return 'qemu_nbd' return (False, 'The qemu_nbd execution module cannot be loaded: the qemu-nbd binary is not in the path.') @@ -49,7 +50,7 @@ def connect(image): '{0} does not exist'.format(image)) return '' - if salt.utils.which('sfdisk'): + if salt.utils.path.which('sfdisk'): fdisk = 'sfdisk -d' else: fdisk = 'fdisk -l' diff --git a/salt/modules/quota.py b/salt/modules/quota.py index 610e5cdf168..e7de0b5e147 100644 --- a/salt/modules/quota.py +++ b/salt/modules/quota.py @@ -8,7 +8,8 @@ from __future__ import absolute_import import logging # Import salt libs -import salt.utils +import salt.utils.path +import salt.utils.platform from salt.exceptions import CommandExecutionError, SaltInvocationError log = logging.getLogger(__name__) @@ -24,9 +25,14 @@ def __virtual__(): ''' Only work on POSIX-like systems with setquota binary available ''' - if not salt.utils.is_windows() and salt.utils.which('setquota'): + if not salt.utils.platform.is_windows() \ + and salt.utils.path.which('setquota'): return 'quota' - return (False, 'The quota execution module cannot be loaded: the module is only available on POSIX-like systems with the setquota binary available.') + return ( + False, + 'The quota execution module cannot be loaded: the module is only ' + 'available on POSIX-like systems with the setquota binary available.' + ) def report(mount): diff --git a/salt/modules/rabbitmq.py b/salt/modules/rabbitmq.py index b55f2dd1737..e6338e7a55f 100644 --- a/salt/modules/rabbitmq.py +++ b/salt/modules/rabbitmq.py @@ -18,7 +18,9 @@ import string # Import salt libs import salt.utils import salt.utils.itertools -import salt.ext.six as six +import salt.utils.path +import salt.utils.platform +from salt.ext import six from salt.exceptions import SaltInvocationError from salt.ext.six.moves import range from salt.exceptions import CommandExecutionError @@ -36,7 +38,7 @@ def __virtual__(): global RABBITMQCTL global RABBITMQ_PLUGINS - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): from salt.ext.six.moves import winreg key = None try: @@ -71,8 +73,8 @@ def __virtual__(): if key is not None: winreg.CloseKey(key) else: - RABBITMQCTL = salt.utils.which('rabbitmqctl') - RABBITMQ_PLUGINS = salt.utils.which('rabbitmq-plugins') + RABBITMQCTL = salt.utils.path.which('rabbitmqctl') + RABBITMQ_PLUGINS = salt.utils.path.which('rabbitmq-plugins') if not RABBITMQCTL: return (False, 'Module rabbitmq: module only works when RabbitMQ is installed') @@ -135,6 +137,7 @@ def _safe_output(line): ''' return not any([ line.startswith('Listing') and line.endswith('...'), + line.startswith('Listing') and '\t' not in line, '...done' in line, line.startswith('WARNING:') ]) @@ -218,7 +221,7 @@ def list_users(runas=None): # Windows runas currently requires a password. # Due to this, don't use a default value for # runas in Windows. - if runas is None and not salt.utils.is_windows(): + if runas is None and not salt.utils.platform.is_windows(): runas = salt.utils.get_user() res = __salt__['cmd.run_all']( [RABBITMQCTL, 'list_users', '-q'], @@ -241,7 +244,7 @@ def list_vhosts(runas=None): salt '*' rabbitmq.list_vhosts ''' - if runas is None and not salt.utils.is_windows(): + if runas is None and not salt.utils.platform.is_windows(): runas = salt.utils.get_user() res = __salt__['cmd.run_all']( [RABBITMQCTL, 'list_vhosts', '-q'], @@ -261,7 +264,7 @@ def user_exists(name, runas=None): salt '*' rabbitmq.user_exists rabbit_user ''' - if runas is None and not salt.utils.is_windows(): + if runas is None and not salt.utils.platform.is_windows(): runas = salt.utils.get_user() return name in list_users(runas=runas) @@ -276,7 +279,7 @@ def vhost_exists(name, runas=None): salt '*' rabbitmq.vhost_exists rabbit_host ''' - if runas is None and not salt.utils.is_windows(): + if runas is None and not salt.utils.platform.is_windows(): runas = salt.utils.get_user() return name in list_vhosts(runas=runas) @@ -299,10 +302,10 @@ def add_user(name, password=None, runas=None): password = ''.join(random.SystemRandom().choice( string.ascii_uppercase + string.digits) for x in range(15)) - if runas is None and not salt.utils.is_windows(): + if runas is None and not salt.utils.platform.is_windows(): runas = salt.utils.get_user() - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): # On Windows, if the password contains a special character # such as '|', normal execution will fail. For example: # cmd: rabbitmq.add_user abc "asdf|def" @@ -347,7 +350,7 @@ def delete_user(name, runas=None): salt '*' rabbitmq.delete_user rabbit_user ''' - if runas is None and not salt.utils.is_windows(): + if runas is None and not salt.utils.platform.is_windows(): runas = salt.utils.get_user() res = __salt__['cmd.run_all']( [RABBITMQCTL, 'delete_user', name], @@ -368,9 +371,9 @@ def change_password(name, password, runas=None): salt '*' rabbitmq.change_password rabbit_user password ''' - if runas is None and not salt.utils.is_windows(): + if runas is None and not salt.utils.platform.is_windows(): runas = salt.utils.get_user() - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): # On Windows, if the password contains a special character # such as '|', normal execution will fail. For example: # cmd: rabbitmq.add_user abc "asdf|def" @@ -404,7 +407,7 @@ def clear_password(name, runas=None): salt '*' rabbitmq.clear_password rabbit_user ''' - if runas is None and not salt.utils.is_windows(): + if runas is None and not salt.utils.platform.is_windows(): runas = salt.utils.get_user() res = __salt__['cmd.run_all']( [RABBITMQCTL, 'clear_password', name], @@ -429,7 +432,7 @@ def check_password(name, password, runas=None): ''' # try to get the rabbitmq-version - adapted from _get_rabbitmq_plugin - if runas is None and not salt.utils.is_windows(): + if runas is None and not salt.utils.platform.is_windows(): runas = salt.utils.get_user() try: @@ -448,7 +451,7 @@ def check_password(name, password, runas=None): # rabbitmq introduced a native api to check a username and password in version 3.5.7. if tuple(version) >= (3, 5, 7): - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): # On Windows, if the password contains a special character # such as '|', normal execution will fail. For example: # cmd: rabbitmq.add_user abc "asdf|def" @@ -504,7 +507,7 @@ def add_vhost(vhost, runas=None): salt '*' rabbitmq add_vhost '' ''' - if runas is None and not salt.utils.is_windows(): + if runas is None and not salt.utils.platform.is_windows(): runas = salt.utils.get_user() res = __salt__['cmd.run_all']( [RABBITMQCTL, 'add_vhost', vhost], @@ -525,7 +528,7 @@ def delete_vhost(vhost, runas=None): salt '*' rabbitmq.delete_vhost '' ''' - if runas is None and not salt.utils.is_windows(): + if runas is None and not salt.utils.platform.is_windows(): runas = salt.utils.get_user() res = __salt__['cmd.run_all']( [RABBITMQCTL, 'delete_vhost', vhost], @@ -545,7 +548,7 @@ def set_permissions(vhost, user, conf='.*', write='.*', read='.*', runas=None): salt '*' rabbitmq.set_permissions 'myvhost' 'myuser' ''' - if runas is None and not salt.utils.is_windows(): + if runas is None and not salt.utils.platform.is_windows(): runas = salt.utils.get_user() res = __salt__['cmd.run_all']( [RABBITMQCTL, 'set_permissions', '-p', @@ -566,7 +569,7 @@ def list_permissions(vhost, runas=None): salt '*' rabbitmq.list_permissions '/myvhost' ''' - if runas is None and not salt.utils.is_windows(): + if runas is None and not salt.utils.platform.is_windows(): runas = salt.utils.get_user() res = __salt__['cmd.run_all']( [RABBITMQCTL, 'list_permissions', '-q', '-p', vhost], @@ -586,7 +589,7 @@ def list_user_permissions(name, runas=None): salt '*' rabbitmq.list_user_permissions 'user'. ''' - if runas is None and not salt.utils.is_windows(): + if runas is None and not salt.utils.platform.is_windows(): runas = salt.utils.get_user() res = __salt__['cmd.run_all']( [RABBITMQCTL, 'list_user_permissions', name, '-q'], @@ -605,14 +608,14 @@ def set_user_tags(name, tags, runas=None): salt '*' rabbitmq.set_user_tags 'myadmin' 'administrator' ''' - if runas is None and not salt.utils.is_windows(): + if runas is None and not salt.utils.platform.is_windows(): runas = salt.utils.get_user() - if tags and isinstance(tags, (list, tuple)): - tags = ' '.join(tags) + if not isinstance(tags, (list, tuple)): + tags = [tags] res = __salt__['cmd.run_all']( - [RABBITMQCTL, 'set_user_tags', name, tags], + [RABBITMQCTL, 'set_user_tags', name] + list(tags), runas=runas, python_shell=False) msg = "Tag(s) set" @@ -629,7 +632,7 @@ def status(runas=None): salt '*' rabbitmq.status ''' - if runas is None and not salt.utils.is_windows(): + if runas is None and not salt.utils.platform.is_windows(): runas = salt.utils.get_user() res = __salt__['cmd.run_all']( [RABBITMQCTL, 'status'], @@ -649,7 +652,7 @@ def cluster_status(runas=None): salt '*' rabbitmq.cluster_status ''' - if runas is None and not salt.utils.is_windows(): + if runas is None and not salt.utils.platform.is_windows(): runas = salt.utils.get_user() res = __salt__['cmd.run_all']( [RABBITMQCTL, 'cluster_status'], @@ -674,7 +677,7 @@ def join_cluster(host, user='rabbit', ram_node=None, runas=None): cmd.append('--ram') cmd.append('{0}@{1}'.format(user, host)) - if runas is None and not salt.utils.is_windows(): + if runas is None and not salt.utils.platform.is_windows(): runas = salt.utils.get_user() stop_app(runas) res = __salt__['cmd.run_all'](cmd, runas=runas, python_shell=False) @@ -693,7 +696,7 @@ def stop_app(runas=None): salt '*' rabbitmq.stop_app ''' - if runas is None and not salt.utils.is_windows(): + if runas is None and not salt.utils.platform.is_windows(): runas = salt.utils.get_user() res = __salt__['cmd.run_all']( [RABBITMQCTL, 'stop_app'], @@ -713,7 +716,7 @@ def start_app(runas=None): salt '*' rabbitmq.start_app ''' - if runas is None and not salt.utils.is_windows(): + if runas is None and not salt.utils.platform.is_windows(): runas = salt.utils.get_user() res = __salt__['cmd.run_all']( [RABBITMQCTL, 'start_app'], @@ -733,7 +736,7 @@ def reset(runas=None): salt '*' rabbitmq.reset ''' - if runas is None and not salt.utils.is_windows(): + if runas is None and not salt.utils.platform.is_windows(): runas = salt.utils.get_user() res = __salt__['cmd.run_all']( [RABBITMQCTL, 'reset'], @@ -753,7 +756,7 @@ def force_reset(runas=None): salt '*' rabbitmq.force_reset ''' - if runas is None and not salt.utils.is_windows(): + if runas is None and not salt.utils.platform.is_windows(): runas = salt.utils.get_user() res = __salt__['cmd.run_all']( [RABBITMQCTL, 'force_reset'], @@ -773,7 +776,7 @@ def list_queues(runas=None, *args): salt '*' rabbitmq.list_queues messages consumers ''' - if runas is None and not salt.utils.is_windows(): + if runas is None and not salt.utils.platform.is_windows(): runas = salt.utils.get_user() cmd = [RABBITMQCTL, 'list_queues', '-q'] cmd.extend(args) @@ -795,7 +798,7 @@ def list_queues_vhost(vhost, runas=None, *args): salt '*' rabbitmq.list_queues messages consumers ''' - if runas is None and not salt.utils.is_windows(): + if runas is None and not salt.utils.platform.is_windows(): runas = salt.utils.get_user() cmd = [RABBITMQCTL, 'list_queues', '-q', '-p', vhost] cmd.extend(args) @@ -818,7 +821,7 @@ def list_policies(vhost="/", runas=None): salt '*' rabbitmq.list_policies' ''' ret = {} - if runas is None and not salt.utils.is_windows(): + if runas is None and not salt.utils.platform.is_windows(): runas = salt.utils.get_user() res = __salt__['cmd.run_all']( [RABBITMQCTL, 'list_policies', '-q', '-p', vhost], @@ -860,7 +863,7 @@ def set_policy(vhost, name, pattern, definition, priority=None, runas=None): salt '*' rabbitmq.set_policy / HA '.*' '{"ha-mode":"all"}' ''' - if runas is None and not salt.utils.is_windows(): + if runas is None and not salt.utils.platform.is_windows(): runas = salt.utils.get_user() if isinstance(definition, dict): definition = json.dumps(definition) @@ -889,7 +892,7 @@ def delete_policy(vhost, name, runas=None): salt '*' rabbitmq.delete_policy / HA' ''' - if runas is None and not salt.utils.is_windows(): + if runas is None and not salt.utils.platform.is_windows(): runas = salt.utils.get_user() res = __salt__['cmd.run_all']( [RABBITMQCTL, 'clear_policy', '-p', vhost, name], @@ -911,7 +914,7 @@ def policy_exists(vhost, name, runas=None): salt '*' rabbitmq.policy_exists / HA ''' - if runas is None and not salt.utils.is_windows(): + if runas is None and not salt.utils.platform.is_windows(): runas = salt.utils.get_user() policies = list_policies(runas=runas) return bool(vhost in policies and name in policies[vhost]) @@ -927,7 +930,7 @@ def list_available_plugins(runas=None): salt '*' rabbitmq.list_available_plugins ''' - if runas is None and not salt.utils.is_windows(): + if runas is None and not salt.utils.platform.is_windows(): runas = salt.utils.get_user() cmd = [_get_rabbitmq_plugin(), 'list', '-m'] ret = __salt__['cmd.run_all'](cmd, python_shell=False, runas=runas) @@ -945,7 +948,7 @@ def list_enabled_plugins(runas=None): salt '*' rabbitmq.list_enabled_plugins ''' - if runas is None and not salt.utils.is_windows(): + if runas is None and not salt.utils.platform.is_windows(): runas = salt.utils.get_user() cmd = [_get_rabbitmq_plugin(), 'list', '-m', '-e'] ret = __salt__['cmd.run_all'](cmd, python_shell=False, runas=runas) @@ -963,7 +966,7 @@ def plugin_is_enabled(name, runas=None): salt '*' rabbitmq.plugin_is_enabled rabbitmq_plugin_name ''' - if runas is None and not salt.utils.is_windows(): + if runas is None and not salt.utils.platform.is_windows(): runas = salt.utils.get_user() return name in list_enabled_plugins(runas) @@ -978,7 +981,7 @@ def enable_plugin(name, runas=None): salt '*' rabbitmq.enable_plugin foo ''' - if runas is None and not salt.utils.is_windows(): + if runas is None and not salt.utils.platform.is_windows(): runas = salt.utils.get_user() cmd = [_get_rabbitmq_plugin(), 'enable', name] ret = __salt__['cmd.run_all'](cmd, runas=runas, python_shell=False) @@ -995,7 +998,7 @@ def disable_plugin(name, runas=None): salt '*' rabbitmq.disable_plugin foo ''' - if runas is None and not salt.utils.is_windows(): + if runas is None and not salt.utils.platform.is_windows(): runas = salt.utils.get_user() cmd = [_get_rabbitmq_plugin(), 'disable', name] ret = __salt__['cmd.run_all'](cmd, runas=runas, python_shell=False) diff --git a/salt/modules/raet_publish.py b/salt/modules/raet_publish.py index 99aa0629267..294a1d84c30 100644 --- a/salt/modules/raet_publish.py +++ b/salt/modules/raet_publish.py @@ -12,6 +12,7 @@ import logging import salt.payload import salt.transport import salt.utils.args +import salt.utils.versions from salt.exceptions import SaltReqTimeoutError log = logging.getLogger(__name__) @@ -166,7 +167,7 @@ def publish(tgt, # remember to remove the expr_form argument from this function when # performing the cleanup on this deprecation. if expr_form is not None: - salt.utils.warn_until( + salt.utils.versions.warn_until( 'Fluorine', 'the target type should be passed using the \'tgt_type\' ' 'argument instead of \'expr_form\'. Support for using ' @@ -214,7 +215,7 @@ def full_data(tgt, # remember to remove the expr_form argument from this function when # performing the cleanup on this deprecation. if expr_form is not None: - salt.utils.warn_until( + salt.utils.versions.warn_until( 'Fluorine', 'the target type should be passed using the \'tgt_type\' ' 'argument instead of \'expr_form\'. Support for using ' diff --git a/salt/modules/rbac_solaris.py b/salt/modules/rbac_solaris.py index 853386cb288..5027d4e2c81 100644 --- a/salt/modules/rbac_solaris.py +++ b/salt/modules/rbac_solaris.py @@ -8,7 +8,8 @@ from __future__ import absolute_import import logging # Import Salt libs -import salt.utils +import salt.utils.files +import salt.utils.path log = logging.getLogger(__name__) @@ -20,7 +21,7 @@ def __virtual__(): ''' Provides rbac if we are running on a solaris like system ''' - if __grains__['kernel'] == 'SunOS' and salt.utils.which('profiles'): + if __grains__['kernel'] == 'SunOS' and salt.utils.path.which('profiles'): return __virtualname__ return ( False, @@ -47,14 +48,14 @@ def profile_list(default_only=False): default_profiles = ['All'] ## lookup default profile(s) - with salt.utils.fopen('/etc/security/policy.conf', 'r') as policy_conf: + with salt.utils.files.fopen('/etc/security/policy.conf', 'r') as policy_conf: for policy in policy_conf: policy = policy.split('=') if policy[0].strip() == 'PROFS_GRANTED': default_profiles.extend(policy[1].strip().split(',')) ## read prof_attr file (profname:res1:res2:desc:attr) - with salt.utils.fopen('/etc/security/prof_attr', 'r') as prof_attr: + with salt.utils.files.fopen('/etc/security/prof_attr', 'r') as prof_attr: for profile in prof_attr: profile = profile.split(':') @@ -92,7 +93,7 @@ def profile_get(user, default_hidden=True): user_profiles = [] ## read user_attr file (user:qualifier:res1:res2:attr) - with salt.utils.fopen('/etc/user_attr', 'r') as user_attr: + with salt.utils.files.fopen('/etc/user_attr', 'r') as user_attr: for profile in user_attr: profile = profile.strip().split(':') @@ -249,7 +250,7 @@ def role_list(): roles = {} ## read user_attr file (user:qualifier:res1:res2:attr) - with salt.utils.fopen('/etc/user_attr', 'r') as user_attr: + with salt.utils.files.fopen('/etc/user_attr', 'r') as user_attr: for role in user_attr: role = role.split(':') @@ -291,7 +292,7 @@ def role_get(user): user_roles = [] ## read user_attr file (user:qualifier:res1:res2:attr) - with salt.utils.fopen('/etc/user_attr', 'r') as user_attr: + with salt.utils.files.fopen('/etc/user_attr', 'r') as user_attr: for role in user_attr: role = role.strip().strip().split(':') @@ -442,7 +443,7 @@ def auth_list(): auths = {} ## read auth_attr file (name:res1:res2:short_desc:long_desc:attr) - with salt.utils.fopen('/etc/security/auth_attr', 'r') as auth_attr: + with salt.utils.files.fopen('/etc/security/auth_attr', 'r') as auth_attr: for auth in auth_attr: auth = auth.split(':') @@ -476,7 +477,7 @@ def auth_get(user, computed=True): user_auths = [] ## read user_attr file (user:qualifier:res1:res2:attr) - with salt.utils.fopen('/etc/user_attr', 'r') as user_attr: + with salt.utils.files.fopen('/etc/user_attr', 'r') as user_attr: for auth in user_attr: auth = auth.strip().split(':') diff --git a/salt/modules/rbenv.py b/salt/modules/rbenv.py index cdae6e9d9af..bcbefa15824 100644 --- a/salt/modules/rbenv.py +++ b/salt/modules/rbenv.py @@ -18,10 +18,12 @@ import logging # Import Salt libs import salt.utils +import salt.utils.args +import salt.utils.platform from salt.exceptions import SaltInvocationError # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six # Set up logger log = logging.getLogger(__name__) @@ -40,18 +42,18 @@ def __virtual__(): ''' Only work on POSIX-like systems ''' - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): return (False, 'The rbenv execution module failed to load: only available on non-Windows systems.') return True def _shlex_split(s): - # from python:salt.utils.shlex_split: passing None for s will read + # from python:salt.utils.args.shlex_split: passing None for s will read # the string to split from standard input. if s is None: - ret = salt.utils.shlex_split('') + ret = salt.utils.args.shlex_split('') else: - ret = salt.utils.shlex_split(s) + ret = salt.utils.args.shlex_split(s) return ret @@ -381,9 +383,9 @@ def do(cmdline, runas=None, env=None): env['PATH'] = '{0}/shims:{1}'.format(path, os.environ['PATH']) try: - cmdline = salt.utils.shlex_split(cmdline) + cmdline = salt.utils.args.shlex_split(cmdline) except AttributeError: - cmdline = salt.utils.shlex_split(str(cmdline)) + cmdline = salt.utils.args.shlex_split(str(cmdline)) result = __salt__['cmd.run_all']( cmdline, @@ -417,9 +419,9 @@ def do_with_ruby(ruby, cmdline, runas=None): raise SaltInvocationError('Command must be specified') try: - cmdline = salt.utils.shlex_split(cmdline) + cmdline = salt.utils.args.shlex_split(cmdline) except AttributeError: - cmdline = salt.utils.shlex_split(str(cmdline)) + cmdline = salt.utils.args.shlex_split(str(cmdline)) env = {} if ruby: diff --git a/salt/modules/rdp.py b/salt/modules/rdp.py index 800fa2d3877..3840570729a 100644 --- a/salt/modules/rdp.py +++ b/salt/modules/rdp.py @@ -8,9 +8,9 @@ from __future__ import absolute_import import logging import re -# Import salt libs +# Import Salt libs +import salt.utils.platform from salt.utils.decorators import depends -import salt.utils try: from pywintypes import error as PyWinError @@ -26,7 +26,7 @@ def __virtual__(): ''' Only works on Windows systems ''' - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): return 'rdp' return (False, 'Module only works on Windows.') diff --git a/salt/modules/redismod.py b/salt/modules/redismod.py index ee15a45a03c..a95e1b9f3f4 100644 --- a/salt/modules/redismod.py +++ b/salt/modules/redismod.py @@ -18,6 +18,8 @@ Module to provide redis functionality to Salt # Import Python libs from __future__ import absolute_import from salt.ext.six.moves import zip +from salt.ext import six +from datetime import datetime # Import third party libs try: @@ -513,8 +515,14 @@ def lastsave(host=None, port=None, db=None, password=None): salt '*' redis.lastsave ''' + # Use of %s to get the timestamp is not supported by Python. The reason it + # works is because it's passed to the system strftime which may not support + # it. See: https://stackoverflow.com/a/11743262 server = _connect(host, port, db, password) - return int(server.lastsave().strftime("%s")) + if six.PY2: + return int((server.lastsave() - datetime(1970, 1, 1)).total_seconds()) + else: + return int(server.lastsave().timestamp()) def llen(key, host=None, port=None, db=None, password=None): diff --git a/salt/modules/reg.py b/salt/modules/reg.py index ca56f702bad..04f1cdbba5f 100644 --- a/salt/modules/reg.py +++ b/salt/modules/reg.py @@ -47,8 +47,8 @@ try: except ImportError: HAS_WINDOWS_MODULES = False -# Import salt libs -import salt.utils +# Import Salt libs +import salt.utils.platform from salt.exceptions import CommandExecutionError PY2 = sys.version_info[0] == 2 @@ -62,7 +62,7 @@ def __virtual__(): ''' Only works on Windows systems with the _winreg python module ''' - if not salt.utils.is_windows(): + if not salt.utils.platform.is_windows(): return (False, 'reg execution module failed to load: ' 'The module will only run on Windows systems') @@ -150,7 +150,9 @@ class Registry(object): # pylint: disable=R0903 _winreg.REG_DWORD: 'REG_DWORD', _winreg.REG_EXPAND_SZ: 'REG_EXPAND_SZ', _winreg.REG_MULTI_SZ: 'REG_MULTI_SZ', - _winreg.REG_SZ: 'REG_SZ' + _winreg.REG_SZ: 'REG_SZ', + # REG_QWORD isn't in the winreg library + 11: 'REG_QWORD' } self.opttype_reverse = { _winreg.REG_OPTION_NON_VOLATILE: 'REG_OPTION_NON_VOLATILE', @@ -753,7 +755,7 @@ def import_file(source, use_32bit_registry=False): ''' Import registry settings from a Windows ``REG`` file by invoking ``REG.EXE``. - .. versionadded:: Nitrogen + .. versionadded:: Oxygen Usage: diff --git a/salt/modules/rest_package.py b/salt/modules/rest_package.py index 8baeabab08c..a0bb1abed86 100644 --- a/salt/modules/rest_package.py +++ b/salt/modules/rest_package.py @@ -7,6 +7,7 @@ from __future__ import absolute_import # Import python libs import logging import salt.utils +import salt.utils.platform log = logging.getLogger(__name__) @@ -20,12 +21,21 @@ def __virtual__(): Only work on systems that are a proxy minion ''' try: - if salt.utils.is_proxy() and __opts__['proxy']['proxytype'] == 'rest_sample': + if salt.utils.platform.is_proxy() \ + and __opts__['proxy']['proxytype'] == 'rest_sample': return __virtualname__ except KeyError: - return (False, 'The rest_package execution module failed to load. Check the proxy key in pillar.') + return ( + False, + 'The rest_package execution module failed to load. Check the ' + 'proxy key in pillar.' + ) - return (False, 'The rest_package execution module failed to load: only works on a rest_sample proxy minion.') + return ( + False, + 'The rest_package execution module failed to load: only works on a ' + 'rest_sample proxy minion.' + ) def list_pkgs(versions_as_list=False, **kwargs): diff --git a/salt/modules/rest_service.py b/salt/modules/rest_service.py index fd06a7ef51d..78260596ca0 100644 --- a/salt/modules/rest_service.py +++ b/salt/modules/rest_service.py @@ -2,14 +2,15 @@ ''' Provide the service module for the proxy-minion REST sample ''' -# Import python libs +# Import Python libs from __future__ import absolute_import -import salt.utils - import logging import fnmatch import re +# Import Salt libs +import salt.utils.platform + log = logging.getLogger(__name__) __func_alias__ = { @@ -26,12 +27,21 @@ def __virtual__(): Only work on systems that are a proxy minion ''' try: - if salt.utils.is_proxy() and __opts__['proxy']['proxytype'] == 'rest_sample': + if salt.utils.platform.is_proxy() \ + and __opts__['proxy']['proxytype'] == 'rest_sample': return __virtualname__ except KeyError: - return (False, 'The rest_service execution module failed to load. Check the proxy key in pillar.') + return ( + False, + 'The rest_service execution module failed to load. Check the ' + 'proxy key in pillar.' + ) - return (False, 'The rest_service execution module failed to load: only works on a rest_sample proxy minion.') + return ( + False, + 'The rest_service execution module failed to load: only works on a ' + 'rest_sample proxy minion.' + ) def get_all(): diff --git a/salt/modules/restartcheck.py b/salt/modules/restartcheck.py index 8cd359d4763..0680b012740 100644 --- a/salt/modules/restartcheck.py +++ b/salt/modules/restartcheck.py @@ -19,7 +19,7 @@ import subprocess import sys # Import salt libs -import salt.utils +import salt.utils.files HAS_PSUTIL = False try: @@ -126,7 +126,7 @@ def _deleted_files(): try: pinfo = proc.as_dict(attrs=['pid', 'name']) try: - maps = salt.utils.fopen('/proc/{0}/maps'.format(pinfo['pid'])) # pylint: disable=resource-leakage + maps = salt.utils.files.fopen('/proc/{0}/maps'.format(pinfo['pid'])) # pylint: disable=resource-leakage dirpath = '/proc/' + str(pinfo['pid']) + '/fd/' listdir = os.listdir(dirpath) except (OSError, IOError): @@ -408,7 +408,7 @@ def restartcheck(ignorelist=None, blacklist=None, excludepid=None, verbose=True) pth.find('.wants') == -1: is_oneshot = False try: - servicefile = salt.utils.fopen(pth) # pylint: disable=resource-leakage + servicefile = salt.utils.files.fopen(pth) # pylint: disable=resource-leakage except IOError: continue sysfold_len = len(systemd_folder) diff --git a/salt/modules/rh_ip.py b/salt/modules/rh_ip.py index e9dd2da6ae9..d29392f07af 100644 --- a/salt/modules/rh_ip.py +++ b/salt/modules/rh_ip.py @@ -14,10 +14,12 @@ import jinja2 import jinja2.exceptions # Import salt libs -import salt.utils +import salt.utils.files +import salt.utils.stringutils import salt.utils.templates import salt.utils.validate.net -import salt.ext.six as six +from salt.exceptions import CommandExecutionError +from salt.ext import six # Set up logging log = logging.getLogger(__name__) @@ -46,7 +48,7 @@ def __virtual__(): _ETHTOOL_CONFIG_OPTS = [ 'autoneg', 'speed', 'duplex', 'rx', 'tx', 'sg', 'tso', 'ufo', - 'gso', 'gro', 'lro' + 'gso', 'gro', 'lro', 'advertise' ] _RH_CONFIG_OPTS = [ 'domain', 'peerdns', 'peerntp', 'defroute', @@ -158,6 +160,18 @@ def _parse_ethtool_opts(opts, iface): else: _raise_error_iface(iface, opts['speed'], valid) + if 'advertise' in opts: + valid = [ + '0x001', '0x002', '0x004', '0x008', '0x010', '0x020', + '0x20000', '0x8000', '0x1000', '0x40000', '0x80000', + '0x200000', '0x400000', '0x800000', '0x1000000', + '0x2000000', '0x4000000' + ] + if str(opts['advertise']) in valid: + config.update({'advertise': opts['advertise']}) + else: + _raise_error_iface(iface, 'advertise', valid) + valid = _CONFIG_TRUE + _CONFIG_FALSE for option in ('rx', 'tx', 'sg', 'tso', 'ufo', 'gso', 'gro', 'lro'): if option in opts: @@ -268,7 +282,11 @@ def _parse_settings_bond_0(opts, iface, bond_def): function will log what the Interface, Setting and what it was expecting. ''' - bond = {'mode': '0'} + + # balance-rr shares miimon settings with balance-xor + bond = _parse_settings_bond_1(opts, iface, bond_def) + + bond.update({'mode': '0'}) # ARP targets in n.n.n.n form valid = ['list of ips (up to 16)'] @@ -285,7 +303,7 @@ def _parse_settings_bond_0(opts, iface, bond_def): _raise_error_iface(iface, 'arp_ip_target', valid) else: _raise_error_iface(iface, 'arp_ip_target', valid) - else: + elif 'miimon' not in opts: _raise_error_iface(iface, 'arp_ip_target', valid) if 'arp_interval' in opts: @@ -650,14 +668,24 @@ def _parse_settings_eth(opts, iface_type, enabled, iface): bypassfirewall = False else: _raise_error_iface(iface, opts[opt], valid) + + bridgectls = [ + 'net.bridge.bridge-nf-call-ip6tables', + 'net.bridge.bridge-nf-call-iptables', + 'net.bridge.bridge-nf-call-arptables', + ] + if bypassfirewall: - __salt__['sysctl.persist']('net.bridge.bridge-nf-call-ip6tables', '0') - __salt__['sysctl.persist']('net.bridge.bridge-nf-call-iptables', '0') - __salt__['sysctl.persist']('net.bridge.bridge-nf-call-arptables', '0') + sysctl_value = 0 else: - __salt__['sysctl.persist']('net.bridge.bridge-nf-call-ip6tables', '1') - __salt__['sysctl.persist']('net.bridge.bridge-nf-call-iptables', '1') - __salt__['sysctl.persist']('net.bridge.bridge-nf-call-arptables', '1') + sysctl_value = 1 + + for sysctl in bridgectls: + try: + __salt__['sysctl.persist'](sysctl, sysctl_value) + except CommandExecutionError: + log.warning('Failed to set sysctl: {0}'.format(sysctl)) + else: if 'bridge' in opts: result['bridge'] = opts['bridge'] @@ -807,7 +835,7 @@ def _parse_network_settings(opts, current): try: opts['networking'] = current['networking'] # If networking option is quoted, use its quote type - quote_type = salt.utils.is_quoted(opts['networking']) + quote_type = salt.utils.stringutils.is_quoted(opts['networking']) _log_default_network('networking', current['networking']) except ValueError: _raise_error_network('networking', valid) @@ -817,7 +845,7 @@ def _parse_network_settings(opts, current): true_val = '{0}yes{0}'.format(quote_type) false_val = '{0}no{0}'.format(quote_type) - networking = salt.utils.dequote(opts['networking']) + networking = salt.utils.stringutils.dequote(opts['networking']) if networking in valid: if networking in _CONFIG_TRUE: result['networking'] = true_val @@ -835,12 +863,12 @@ def _parse_network_settings(opts, current): if opts['hostname']: result['hostname'] = '{1}{0}{1}'.format( - salt.utils.dequote(opts['hostname']), quote_type) + salt.utils.stringutils.dequote(opts['hostname']), quote_type) else: _raise_error_network('hostname', ['server1.example.com']) if 'nozeroconf' in opts: - nozeroconf = salt.utils.dequote(opts['nozerconf']) + nozeroconf = salt.utils.stringutils.dequote(opts['nozeroconf']) if nozeroconf in valid: if nozeroconf in _CONFIG_TRUE: result['nozeroconf'] = true_val @@ -852,7 +880,7 @@ def _parse_network_settings(opts, current): for opt in opts: if opt not in ['networking', 'hostname', 'nozeroconf']: result[opt] = '{1}{0}{1}'.format( - salt.utils.dequote(opts[opt]), quote_type) + salt.utils.stringutils.dequote(opts[opt]), quote_type) return result @@ -888,7 +916,7 @@ def _read_file(path): Reads and returns the contents of a file ''' try: - with salt.utils.fopen(path, 'rb') as rfh: + with salt.utils.files.fopen(path, 'rb') as rfh: contents = rfh.read() if six.PY3: contents = contents.encode(__salt_system_encoding__) @@ -913,7 +941,7 @@ def _write_file_iface(iface, data, folder, pattern): msg = msg.format(filename, folder) log.error(msg) raise AttributeError(msg) - with salt.utils.fopen(filename, 'w') as fp_: + with salt.utils.files.fopen(filename, 'w') as fp_: fp_.write(data) @@ -921,7 +949,7 @@ def _write_file_network(data, filename): ''' Writes a file to disk ''' - with salt.utils.fopen(filename, 'w') as fp_: + with salt.utils.files.fopen(filename, 'w') as fp_: fp_.write(data) @@ -985,7 +1013,10 @@ def build_interface(iface, iface_type, enabled, **settings): salt '*' ip.build_interface eth0 eth ''' if __grains__['os'] == 'Fedora': - rh_major = '6' + if __grains__['osmajorrelease'] >= 18: + rh_major = '7' + else: + rh_major = '6' else: rh_major = __grains__['osrelease'][:1] diff --git a/salt/modules/rh_service.py b/salt/modules/rh_service.py index cd66952900d..95d5048791c 100644 --- a/salt/modules/rh_service.py +++ b/salt/modules/rh_service.py @@ -19,7 +19,7 @@ import fnmatch import re # Import salt libs -import salt.utils +import salt.utils.path log = logging.getLogger(__name__) @@ -32,7 +32,7 @@ __virtualname__ = 'service' # Import upstart module if needed HAS_UPSTART = False -if salt.utils.which('initctl'): +if salt.utils.path.which('initctl'): try: # Don't re-invent the wheel, import the helper functions from the # upstart module. diff --git a/salt/modules/riak.py b/salt/modules/riak.py index 5d61fb150cf..cb4b3d24f0f 100644 --- a/salt/modules/riak.py +++ b/salt/modules/riak.py @@ -5,14 +5,14 @@ Riak Salt Module from __future__ import absolute_import # Import salt libs -import salt.utils +import salt.utils.path def __virtual__(): ''' Only available on systems with Riak installed. ''' - if salt.utils.which('riak'): + if salt.utils.path.which('riak'): return True return (False, 'The riak execution module failed to load: the riak binary is not in the path.') @@ -22,7 +22,7 @@ def __execute_cmd(name, cmd): Execute Riak commands ''' return __salt__['cmd.run_all']( - '{0} {1}'.format(salt.utils.which(name), cmd) + '{0} {1}'.format(salt.utils.path.which(name), cmd) ) diff --git a/salt/modules/rpm.py b/salt/modules/rpm.py index 0e4d889304c..0419092251b 100644 --- a/salt/modules/rpm.py +++ b/salt/modules/rpm.py @@ -11,10 +11,11 @@ import re import datetime # Import Salt libs -import salt.utils +import salt.utils.decorators.path import salt.utils.itertools -import salt.utils.decorators as decorators +import salt.utils.path import salt.utils.pkg.rpm +import salt.utils.versions # pylint: disable=import-error,redefined-builtin from salt.ext.six.moves import zip from salt.ext import six @@ -44,7 +45,7 @@ def __virtual__(): ''' Confine this module to rpm based systems ''' - if not salt.utils.which('rpm'): + if not salt.utils.path.which('rpm'): return (False, 'The rpm execution module failed to load: rpm binary is not in the path.') try: os_grain = __grains__['os'].lower() @@ -420,9 +421,9 @@ def owner(*paths): return ret -@decorators.which('rpm2cpio') -@decorators.which('cpio') -@decorators.which('diff') +@salt.utils.decorators.path.which('rpm2cpio') +@salt.utils.decorators.path.which('cpio') +@salt.utils.decorators.path.which('diff') def diff(package, path): ''' Return a formatted diff between current file and original in a package. @@ -658,7 +659,7 @@ def version_cmp(ver1, ver2, ignore_epoch=False): log.debug('rpmUtils.miscutils.compareEVR is not available') if cmp_func is None: - if salt.utils.which('rpmdev-vercmp'): + if salt.utils.path.which('rpmdev-vercmp'): # rpmdev-vercmp always uses epochs, even when zero def _ensure_epoch(ver): def _prepend(ver): @@ -687,7 +688,7 @@ def version_cmp(ver1, ver2, ignore_epoch=False): elif result['retcode'] == 12: return -1 else: - # We'll need to fall back to salt.utils.version_cmp() + # We'll need to fall back to salt.utils.versions.version_cmp() log.warning( 'Failed to interpret results of rpmdev-vercmp output. ' 'This is probably a bug, and should be reported. ' @@ -695,7 +696,7 @@ def version_cmp(ver1, ver2, ignore_epoch=False): result['retcode'], result['stdout'] ) else: - # We'll need to fall back to salt.utils.version_cmp() + # We'll need to fall back to salt.utils.versions.version_cmp() log.warning( 'rpmdevtools is not installed, please install it for ' 'more accurate version comparisons' @@ -705,8 +706,8 @@ def version_cmp(ver1, ver2, ignore_epoch=False): # otherwise would be equal, ignore the release. This can happen if # e.g. you are checking if a package version 3.2 is satisfied by # 3.2-1. - (ver1_e, ver1_v, ver1_r) = salt.utils.str_version_to_evr(ver1) - (ver2_e, ver2_v, ver2_r) = salt.utils.str_version_to_evr(ver2) + (ver1_e, ver1_v, ver1_r) = salt.utils.pkg.rpm.version_to_evr(ver1) + (ver2_e, ver2_v, ver2_r) = salt.utils.pkg.rpm.version_to_evr(ver2) if not ver1_r or not ver2_r: ver1_r = ver2_r = '' @@ -727,7 +728,7 @@ def version_cmp(ver1, ver2, ignore_epoch=False): # We would already have normalized the versions at the beginning of this # function if ignore_epoch=True, so avoid unnecessary work and just pass # False for this value. - return salt.utils.version_cmp(ver1, ver2, ignore_epoch=False) + return salt.utils.versions.version_cmp(ver1, ver2, ignore_epoch=False) def checksum(*paths): diff --git a/salt/modules/rpmbuild.py b/salt/modules/rpmbuild.py index 7b7ab71bc7c..a6687db5241 100644 --- a/salt/modules/rpmbuild.py +++ b/salt/modules/rpmbuild.py @@ -23,9 +23,15 @@ import traceback import functools # Import salt libs -from salt.ext.six.moves.urllib.parse import urlparse as _urlparse # pylint: disable=no-name-in-module,import-error from salt.exceptions import SaltInvocationError -import salt.utils +import salt.utils # Can be removed when chugid_and_umask is moved +import salt.utils.files +import salt.utils.path +import salt.utils.vt + +# Import 3rd-party libs +from salt.ext import six +from salt.ext.six.moves.urllib.parse import urlparse as _urlparse # pylint: disable=no-name-in-module,import-error HAS_LIBS = False @@ -48,7 +54,7 @@ def __virtual__(): missing_util = False utils_reqd = ['gpg', 'rpm', 'rpmbuild', 'mock', 'createrepo'] for named_util in utils_reqd: - if not salt.utils.which(named_util): + if not salt.utils.path.which(named_util): missing_util = True break @@ -76,7 +82,7 @@ def _create_rpmmacros(): os.makedirs(mockdir) rpmmacros = os.path.join(home, '.rpmmacros') - with salt.utils.fopen(rpmmacros, 'w') as afile: + with salt.utils.files.fopen(rpmmacros, 'w') as afile: afile.write('%_topdir {0}\n'.format(rpmbuilddir)) afile.write('%signature gpg\n') afile.write('%_source_filedigest_algorithm 8\n') @@ -186,7 +192,7 @@ def make_src_pkg(dest_dir, spec, sources, env=None, template=None, saltenv='base _create_rpmmacros() tree_base = _mk_tree() spec_path = _get_spec(tree_base, spec, template, saltenv) - if isinstance(sources, str): + if isinstance(sources, six.string_types): sources = sources.split(',') for src in sources: _get_src(tree_base, src, saltenv) diff --git a/salt/modules/rsync.py b/salt/modules/rsync.py index 22ddca7aaeb..e536b063e45 100644 --- a/salt/modules/rsync.py +++ b/salt/modules/rsync.py @@ -12,9 +12,12 @@ from __future__ import absolute_import # Import python libs import errno import logging +import re +import tempfile # Import salt libs -import salt.utils +import salt.utils.files +import salt.utils.path from salt.exceptions import CommandExecutionError, SaltInvocationError log = logging.getLogger(__name__) @@ -26,7 +29,7 @@ def __virtual__(): ''' Only load module if rsync binary is present ''' - if salt.utils.which('rsync'): + if salt.utils.path.which('rsync'): return __virtualname__ return (False, 'The rsync execution module cannot be loaded: ' 'the rsync binary is not in the path.') @@ -69,7 +72,8 @@ def rsync(src, excludefrom=None, dryrun=False, rsh=None, - additional_opts=None): + additional_opts=None, + saltenv='base'): ''' .. versionchanged:: 2016.3.0 Return data now contains just the output of the rsync command, instead @@ -122,6 +126,9 @@ def rsync(src, additional_opts Any additional rsync options, should be specified as a list. + saltenv + Specify a salt fileserver environment to be used. + CLI Example: .. code-block:: bash @@ -154,6 +161,38 @@ def rsync(src, if not src or not dst: raise SaltInvocationError('src and dst cannot be empty') + tmp_src = None + if src.startswith('salt://'): + _src = src + _path = re.sub('salt://', '', _src) + src_is_dir = False + if _path in __salt__['cp.list_master_dirs'](saltenv=saltenv): + src_is_dir = True + + if src_is_dir: + tmp_src = tempfile.mkdtemp() + dir_src = __salt__['cp.get_dir'](_src, + tmp_src, + saltenv) + if dir_src: + src = tmp_src + # Ensure src ends in / so we + # get the contents not the tmpdir + # itself. + if not src.endswith('/'): + src = '{0}/'.format(src) + else: + raise CommandExecutionError('{0} does not exist'.format(src)) + else: + tmp_src = salt.utils.mkstemp() + file_src = __salt__['cp.get_file'](_src, + tmp_src, + saltenv) + if file_src: + src = tmp_src + else: + raise CommandExecutionError('{0} does not exist'.format(src)) + option = _check(delete, force, update, @@ -172,6 +211,9 @@ def rsync(src, return __salt__['cmd.run_all'](cmd, python_shell=False) except (IOError, OSError) as exc: raise CommandExecutionError(exc.strerror) + finally: + if tmp_src: + __salt__['file.remove'](tmp_src) def version(): @@ -221,7 +263,7 @@ def config(conf_path='/etc/rsyncd.conf'): ''' ret = '' try: - with salt.utils.fopen(conf_path, 'r') as fp_: + with salt.utils.files.fopen(conf_path, 'r') as fp_: for line in fp_: ret += line except IOError as exc: diff --git a/salt/modules/runit.py b/salt/modules/runit.py index 083ac86917a..f994aae6ccd 100644 --- a/salt/modules/runit.py +++ b/salt/modules/runit.py @@ -57,7 +57,8 @@ log = logging.getLogger(__name__) # Import salt libs from salt.exceptions import CommandExecutionError -import salt.utils +import salt.utils.files +import salt.utils.path # Function alias to not shadow built-ins. __func_alias__ = { @@ -80,7 +81,8 @@ for service_dir in VALID_SERVICE_DIRS: AVAIL_SVR_DIRS = [] # Define the module's virtual name -__virtualname__ = 'service' +__virtualname__ = 'runit' +__virtual_aliases__ = ('runit',) def __virtual__(): @@ -91,8 +93,12 @@ def __virtual__(): if __grains__.get('init') == 'runit': if __grains__['os'] == 'Void': add_svc_avail_path('/etc/sv') + global __virtualname__ + __virtualname__ = 'service' return __virtualname__ - return False + if salt.utils.path.which('sv'): + return __virtualname__ + return (False, 'Runit not available. Please install sv') def _service_path(name): @@ -600,7 +606,7 @@ def enable(name, start=False, **kwargs): log.trace('need a temporary file {0}'.format(down_file)) if not os.path.exists(down_file): try: - salt.utils.fopen(down_file, "w").close() # pylint: disable=resource-leakage + salt.utils.files.fopen(down_file, "w").close() # pylint: disable=resource-leakage except IOError: log.error('Unable to create file {0}'.format(down_file)) return False @@ -675,7 +681,7 @@ def disable(name, stop=False, **kwargs): if not os.path.exists(down_file): try: - salt.utils.fopen(down_file, "w").close() # pylint: disable=resource-leakage + salt.utils.files.fopen(down_file, "w").close() # pylint: disable=resource-leakage except IOError: log.error('Unable to create file {0}'.format(down_file)) return False diff --git a/salt/modules/rvm.py b/salt/modules/rvm.py index 4a42ed498b7..a23f8e050fb 100644 --- a/salt/modules/rvm.py +++ b/salt/modules/rvm.py @@ -10,7 +10,7 @@ import os import logging # Import salt libs -import salt.utils +import salt.utils.args from salt.exceptions import CommandExecutionError log = logging.getLogger(__name__) @@ -462,7 +462,7 @@ def do(ruby, command, runas=None, cwd=None, env=None): # pylint: disable=C0103 salt '*' rvm.do 2.0.0 ''' try: - command = salt.utils.shlex_split(command) + command = salt.utils.args.shlex_split(command) except AttributeError: - command = salt.utils.shlex_split(str(command)) + command = salt.utils.args.shlex_split(str(command)) return _rvm_do(ruby, command, runas=runas, cwd=cwd, env=env) diff --git a/salt/modules/salt_proxy.py b/salt/modules/salt_proxy.py index d0967af9dc7..4c7c1a4e361 100644 --- a/salt/modules/salt_proxy.py +++ b/salt/modules/salt_proxy.py @@ -7,14 +7,16 @@ Module to deploy and manage salt-proxy processes on a minion. ''' +# Import Python libs from __future__ import absolute_import - -import salt.ext.six.moves - import os import logging -import salt.utils +# Import Salt libs +import salt.utils.files + +# Import 3rd-party libs +import salt.ext.six.moves log = logging.getLogger(__name__) @@ -30,7 +32,7 @@ def _write_proxy_conf(proxyfile): if proxyfile: log.debug('Writing proxy conf file') - with salt.utils.fopen(proxyfile, 'w') as proxy_conf: + with salt.utils.files.fopen(proxyfile, 'w') as proxy_conf: proxy_conf.write('master = {0}' .format(__grains__['master'])) msg = 'Wrote proxy file {0}'.format(proxyfile) diff --git a/salt/modules/saltcheck.py b/salt/modules/saltcheck.py new file mode 100644 index 00000000000..f59d2d7e91f --- /dev/null +++ b/salt/modules/saltcheck.py @@ -0,0 +1,602 @@ +# -*- coding: utf-8 -*- +''' +A module for testing the logic of states and highstates + +Saltcheck provides unittest like functionality requiring only the knowledge of salt module execution and yaml. + +In order to run state and highstate saltcheck tests a sub-folder of a state must be creaed and named "saltcheck-tests". + +Tests for a state should be created in files ending in *.tst and placed in the saltcheck-tests folder. + +Multiple tests can be created in a file. +Multiple *.tst files can be created in the saltcheck-tests folder. +The "id" of a test works in the same manner as in salt state files. +They should be unique and descriptive. + +Example file system layout: +/srv/salt/apache/ + init.sls + config.sls + saltcheck-tests/ + pkg_and_mods.tst + config.tst + + +Saltcheck Test Syntax: + +Unique-ID: + module_and_function: + args: + kwargs: + assertion: + expected-return: + + +Example test 1: + +echo-test-hello: + module_and_function: test.echo + args: + - "hello" + kwargs: + assertion: assertEqual + expected-return: 'hello' + + +:codeauthor: William Cannon +:maturity: new +''' +from __future__ import absolute_import +import logging +import os +import time +import yaml +try: + import salt.utils + import salt.client + import salt.exceptions +except ImportError: + pass + +log = logging.getLogger(__name__) + +__virtualname__ = 'saltcheck' + + +def __virtual__(): + ''' + Check dependencies - may be useful in future + ''' + return __virtualname__ + + +def update_master_cache(): + ''' + Updates the master cache onto the minion - transfers all salt-check-tests + Should be done one time before running tests, and if tests are updated + Can be automated by setting "auto_update_master_cache: True" in minion config + + CLI Example: + salt '*' saltcheck.update_master_cache + ''' + __salt__['cp.cache_master']() + return True + + +def run_test(**kwargs): + ''' + Execute one saltcheck test and return result + + :param keyword arg test: + CLI Example:: + salt '*' saltcheck.run_test + test='{"module_and_function": "test.echo", + "assertion": "assertEqual", + "expected-return": "This works!", + "args":["This works!"] }' + ''' + # salt converts the string to a dictionary auto-magically + scheck = SaltCheck() + test = kwargs.get('test', None) + if test and isinstance(test, dict): + return scheck.run_test(test) + else: + return "Test must be a dictionary" + + +def run_state_tests(state): + ''' + Execute all tests for a salt state and return results + Nested states will also be tested + + :param str state: the name of a user defined state + + CLI Example:: + salt '*' saltcheck.run_state_tests postfix + ''' + scheck = SaltCheck() + paths = scheck.get_state_search_path_list() + stl = StateTestLoader(search_paths=paths) + results = {} + sls_list = _get_state_sls(state) + for state_name in sls_list: + mypath = stl.convert_sls_to_path(state_name) + stl.add_test_files_for_sls(mypath) + stl.load_test_suite() + results_dict = {} + for key, value in stl.test_dict.items(): + result = scheck.run_test(value) + results_dict[key] = result + results[state_name] = results_dict + passed = 0 + failed = 0 + missing_tests = 0 + for state in results: + if len(results[state].items()) == 0: + missing_tests = missing_tests + 1 + else: + for dummy, val in results[state].items(): + log.info("dummy={}, val={}".format(dummy, val)) + if val.startswith('Pass'): + passed = passed + 1 + if val.startswith('Fail'): + failed = failed + 1 + out_list = [] + for key, value in results.items(): + out_list.append({key: value}) + out_list.sort() + out_list.append({"TEST RESULTS": {'Passed': passed, 'Failed': failed, 'Missing Tests': missing_tests}}) + return out_list + + +def run_highstate_tests(): + ''' + Execute all tests for a salt highstate and return results + + CLI Example:: + salt '*' saltcheck.run_highstate_tests + ''' + scheck = SaltCheck() + paths = scheck.get_state_search_path_list() + stl = StateTestLoader(search_paths=paths) + results = {} + sls_list = _get_top_states() + all_states = [] + for top_state in sls_list: + sls_list = _get_state_sls(top_state) + for state in sls_list: + if state not in all_states: + all_states.append(state) + + for state_name in all_states: + mypath = stl.convert_sls_to_path(state_name) + stl.add_test_files_for_sls(mypath) + stl.load_test_suite() + results_dict = {} + for key, value in stl.test_dict.items(): + result = scheck.run_test(value) + results_dict[key] = result + results[state_name] = results_dict + passed = 0 + failed = 0 + missing_tests = 0 + for state in results: + if len(results[state].items()) == 0: + missing_tests = missing_tests + 1 + else: + for dummy, val in results[state].items(): + log.info("dummy={}, val={}".format(dummy, val)) + if val.startswith('Pass'): + passed = passed + 1 + if val.startswith('Fail'): + failed = failed + 1 + out_list = [] + for key, value in results.items(): + out_list.append({key: value}) + out_list.sort() + out_list.append({"TEST RESULTS": {'Passed': passed, 'Failed': failed, 'Missing Tests': missing_tests}}) + return out_list + + +def _is_valid_module(module): + '''return a list of all modules available on minion''' + modules = __salt__['sys.list_modules']() + return bool(module in modules) + + +def _get_auto_update_cache_value(): + '''return the config value of auto_update_master_cache''' + __salt__['config.get']('auto_update_master_cache') + return True + + +def _is_valid_function(module_name, function): + '''Determine if a function is valid for a module''' + try: + functions = __salt__['sys.list_functions'](module_name) + except salt.exceptions.SaltException: + functions = ["unable to look up functions"] + return "{0}.{1}".format(module_name, function) in functions + + +def _get_top_states(): + ''' equivalent to a salt cli: salt web state.show_top''' + alt_states = [] + try: + returned = __salt__['state.show_top']() + for i in returned['base']: + alt_states.append(i) + except Exception: + raise + # log.info("top states: {}".format(alt_states)) + return alt_states + + +def _get_state_sls(state): + ''' equivalent to a salt cli: salt web state.show_low_sls STATE''' + sls_list_state = [] + try: + returned = __salt__['state.show_low_sls'](state) + for i in returned: + if i['__sls__'] not in sls_list_state: + sls_list_state.append(i['__sls__']) + except Exception: + raise + return sls_list_state + + +class SaltCheck(object): + ''' + This class implements the saltcheck + ''' + + def __init__(self): + # self.sls_list_top = [] + self.sls_list_state = [] + self.modules = [] + self.results_dict = {} + self.results_dict_summary = {} + self.assertions_list = '''assertEqual assertNotEqual + assertTrue assertFalse + assertIn assertNotIn + assertGreater + assertGreaterEqual + assertLess assertLessEqual'''.split() + self.auto_update_master_cache = _get_auto_update_cache_value + # self.salt_lc = salt.client.Caller(mopts=__opts__) + self.salt_lc = salt.client.Caller() + if self.auto_update_master_cache: + update_master_cache() + + def __is_valid_test(self, test_dict): + '''Determine if a test contains: + a test name, + a valid module and function, + a valid assertion, + an expected return value''' + tots = 0 # need total of >= 6 to be a valid test + m_and_f = test_dict.get('module_and_function', None) + assertion = test_dict.get('assertion', None) + expected_return = test_dict.get('expected-return', None) + log.info("__is_valid_test has test: {}".format(test_dict)) + if m_and_f: + tots += 1 + module, function = m_and_f.split('.') + if _is_valid_module(module): + tots += 1 + if _is_valid_function(module, function): + tots += 1 + log.info("__is_valid_test has valid m_and_f") + if assertion: + tots += 1 + if assertion in self.assertions_list: + tots += 1 + log.info("__is_valid_test has valid_assertion") + if expected_return: + tots += 1 + log.info("__is_valid_test has valid_expected_return") + log.info("__is_valid_test score: {}".format(tots)) + return tots >= 6 + + def call_salt_command(self, + fun, + args, + kwargs): + '''Generic call of salt Caller command''' + value = False + try: + if args and kwargs: + value = self.salt_lc.cmd(fun, *args, **kwargs) + elif args and not kwargs: + value = self.salt_lc.cmd(fun, *args) + elif not args and kwargs: + value = self.salt_lc.cmd(fun, **kwargs) + else: + value = self.salt_lc.cmd(fun) + except salt.exceptions.SaltException: + raise + except Exception: + raise + return value + + def run_test(self, test_dict): + '''Run a single saltcheck test''' + if self.__is_valid_test(test_dict): + mod_and_func = test_dict['module_and_function'] + args = test_dict.get('args', None) + kwargs = test_dict.get('kwargs', None) + assertion = test_dict['assertion'] + expected_return = test_dict['expected-return'] + actual_return = self.call_salt_command(mod_and_func, args, kwargs) + if assertion != "assertIn": + expected_return = self.cast_expected_to_returned_type(expected_return, actual_return) + if assertion == "assertEqual": + value = self.__assert_equal(expected_return, actual_return) + elif assertion == "assertNotEqual": + value = self.__assert_not_equal(expected_return, actual_return) + elif assertion == "assertTrue": + value = self.__assert_true(expected_return) + elif assertion == "assertFalse": + value = self.__assert_false(expected_return) + elif assertion == "assertIn": + value = self.__assert_in(expected_return, actual_return) + elif assertion == "assertNotIn": + value = self.__assert_not_in(expected_return, actual_return) + elif assertion == "assertGreater": + value = self.__assert_greater(expected_return, actual_return) + elif assertion == "assertGreaterEqual": + value = self.__assert_greater_equal(expected_return, actual_return) + elif assertion == "assertLess": + value = self.__assert_less(expected_return, actual_return) + elif assertion == "assertLessEqual": + value = self.__assert_less_equal(expected_return, actual_return) + else: + value = "Fail - bas assertion" + else: + return "Fail - invalid test" + return value + + @staticmethod + def cast_expected_to_returned_type(expected, returned): + ''' + Determine the type of variable returned + Cast the expected to the type of variable returned + ''' + ret_type = type(returned) + new_expected = expected + if expected == "False" and ret_type == bool: + expected = False + try: + new_expected = ret_type(expected) + except ValueError: + log.info("Unable to cast expected into type of returned") + log.info("returned = {}".format(returned)) + log.info("type of returned = {}".format(type(returned))) + log.info("expected = {}".format(expected)) + log.info("type of expected = {}".format(type(expected))) + return new_expected + + @staticmethod + def __assert_equal(expected, returned): + ''' + Test if two objects are equal + ''' + result = "Pass" + + try: + assert (expected == returned), "{0} is not equal to {1}".format(expected, returned) + except AssertionError as err: + result = "Fail: " + str(err) + return result + + @staticmethod + def __assert_not_equal(expected, returned): + ''' + Test if two objects are not equal + ''' + result = "Pass" + try: + assert (expected != returned), "{0} is equal to {1}".format(expected, returned) + except AssertionError as err: + result = "Fail: " + str(err) + return result + + @staticmethod + def __assert_true(returned): + ''' + Test if an boolean is True + ''' + result = "Pass" + try: + assert (returned is True), "{0} not True".format(returned) + except AssertionError as err: + result = "Fail: " + str(err) + return result + + @staticmethod + def __assert_false(returned): + ''' + Test if an boolean is False + ''' + result = "Pass" + if isinstance(returned, str): + try: + returned = bool(returned) + except ValueError: + raise + try: + assert (returned is False), "{0} not False".format(returned) + except AssertionError as err: + result = "Fail: " + str(err) + return result + + @staticmethod + def __assert_in(expected, returned): + ''' + Test if a value is in the list of returned values + ''' + result = "Pass" + try: + assert (expected in returned), "{0} not False".format(returned) + except AssertionError as err: + result = "Fail: " + str(err) + return result + + @staticmethod + def __assert_not_in(expected, returned): + ''' + Test if a value is not in the list of returned values + ''' + result = "Pass" + try: + assert (expected not in returned), "{0} not False".format(returned) + except AssertionError as err: + result = "Fail: " + str(err) + return result + + @staticmethod + def __assert_greater(expected, returned): + ''' + Test if a value is greater than the returned value + ''' + result = "Pass" + try: + assert (expected > returned), "{0} not False".format(returned) + except AssertionError as err: + result = "Fail: " + str(err) + return result + + @staticmethod + def __assert_greater_equal(expected, returned): + ''' + Test if a value is greater than or equal to the returned value + ''' + result = "Pass" + try: + assert (expected >= returned), "{0} not False".format(returned) + except AssertionError as err: + result = "Fail: " + str(err) + return result + + @staticmethod + def __assert_less(expected, returned): + ''' + Test if a value is less than the returned value + ''' + result = "Pass" + try: + assert (expected < returned), "{0} not False".format(returned) + except AssertionError as err: + result = "Fail: " + str(err) + return result + + @staticmethod + def __assert_less_equal(expected, returned): + ''' + Test if a value is less than or equal to the returned value + ''' + result = "Pass" + try: + assert (expected <= returned), "{0} not False".format(returned) + except AssertionError as err: + result = "Fail: " + str(err) + return result + + @staticmethod + def get_state_search_path_list(): + '''For the state file system, return a + list of paths to search for states''' + # state cache should be updated before running this method + search_list = [] + cachedir = __opts__.get('cachedir', None) + environment = __opts__['environment'] + if environment: + path = cachedir + os.sep + "files" + os.sep + environment + search_list.append(path) + path = cachedir + os.sep + "files" + os.sep + "base" + search_list.append(path) + return search_list + + +class StateTestLoader(object): + ''' + Class loads in test files for a state + e.g. state_dir/saltcheck-tests/[1.tst, 2.tst, 3.tst] + ''' + + def __init__(self, search_paths): + self.search_paths = search_paths + self.path_type = None + self.test_files = [] # list of file paths + self.test_dict = {} + + def load_test_suite(self): + '''load tests either from one file, or a set of files''' + self.test_dict = {} + for myfile in self.test_files: + self.load_file(myfile) + self.test_files = [] + + def load_file(self, filepath): + ''' + loads in one test file + ''' + try: + with salt.utils.files.fopen(filepath, 'r') as myfile: + # with open(filepath, 'r') as myfile: + contents_yaml = yaml.load(myfile) + for key, value in contents_yaml.items(): + self.test_dict[key] = value + except: + raise + return + + def gather_files(self, filepath): + '''gather files for a test suite''' + self.test_files = [] + log.info("gather_files: {}".format(time.time())) + filepath = filepath + os.sep + 'saltcheck-tests' + rootdir = filepath + # for dirname, subdirlist, filelist in os.walk(rootdir): + for dirname, dummy, filelist in os.walk(rootdir): + for fname in filelist: + if fname.endswith('.tst'): + start_path = dirname + os.sep + fname + full_path = os.path.abspath(start_path) + self.test_files.append(full_path) + return + + @staticmethod + def convert_sls_to_paths(sls_list): + '''Converting sls to paths''' + new_sls_list = [] + for sls in sls_list: + sls = sls.replace(".", os.sep) + new_sls_list.append(sls) + return new_sls_list + + @staticmethod + def convert_sls_to_path(sls): + '''Converting sls to paths''' + sls = sls.replace(".", os.sep) + return sls + + def add_test_files_for_sls(self, sls_path): + '''Adding test files''' + for path in self.search_paths: + full_path = path + os.sep + sls_path + rootdir = full_path + if os.path.isdir(full_path): + log.info("searching path= {}".format(full_path)) + # for dirname, subdirlist, filelist in os.walk(rootdir, topdown=True): + for dirname, subdirlist, dummy in os.walk(rootdir, topdown=True): + if "saltcheck-tests" in subdirlist: + self.gather_files(dirname) + log.info("test_files list: {}".format(self.test_files)) + log.info("found subdir match in = {}".format(dirname)) + else: + log.info("did not find subdir match in = {}".format(dirname)) + del subdirlist[:] + else: + log.info("path is not a directory= {}".format(full_path)) + return diff --git a/salt/modules/saltutil.py b/salt/modules/saltutil.py index bb8136c20ea..869fb1ac39c 100644 --- a/salt/modules/saltutil.py +++ b/salt/modules/saltutil.py @@ -50,9 +50,11 @@ import salt.utils import salt.utils.args import salt.utils.event import salt.utils.extmods +import salt.utils.files import salt.utils.minion import salt.utils.process import salt.utils.url +import salt.utils.versions import salt.wheel HAS_PSUTIL = True @@ -78,7 +80,8 @@ def _get_top_file_envs(): return __context__['saltutil._top_file_envs'] except KeyError: try: - st_ = salt.state.HighState(__opts__) + st_ = salt.state.HighState(__opts__, + initial_pillar=__pillar__) top = st_.get_top() if top: envs = list(st_.top_matches(top).keys()) or 'base' @@ -105,7 +108,7 @@ def _sync(form, saltenv=None, extmod_whitelist=None, extmod_blacklist=None): # Dest mod_dir is touched? trigger reload if requested if touched: mod_file = os.path.join(__opts__['cachedir'], 'module_refresh') - with salt.utils.fopen(mod_file, 'a+') as ofile: + with salt.utils.files.fopen(mod_file, 'a+') as ofile: ofile.write('') if form == 'grains' and \ __opts__.get('grains_cache') and \ @@ -189,10 +192,14 @@ def sync_beacons(saltenv=None, refresh=True, extmod_whitelist=None, extmod_black Sync beacons from ``salt://_beacons`` to the minion - saltenv : base + saltenv The fileserver environment from which to sync. To sync from more than one environment, pass a comma-separated list. + If not passed, then all environments configured in the :ref:`top files + ` will be checked for beacons to sync. If no top files are + found, then the ``base`` environment will be synced. + refresh : True If ``True``, refresh the available beacons on the minion. This refresh will be performed even if no new beacons are synced. Set to ``False`` @@ -224,10 +231,14 @@ def sync_sdb(saltenv=None, extmod_whitelist=None, extmod_blacklist=None): Sync sdb modules from ``salt://_sdb`` to the minion - saltenv : base + saltenv The fileserver environment from which to sync. To sync from more than one environment, pass a comma-separated list. + If not passed, then all environments configured in the :ref:`top files + ` will be checked for sdb modules to sync. If no top files + are found, then the ``base`` environment will be synced. + refresh : False This argument has no affect and is included for consistency with the other sync functions. @@ -256,10 +267,14 @@ def sync_modules(saltenv=None, refresh=True, extmod_whitelist=None, extmod_black Sync execution modules from ``salt://_modules`` to the minion - saltenv : base + saltenv The fileserver environment from which to sync. To sync from more than one environment, pass a comma-separated list. + If not passed, then all environments configured in the :ref:`top files + ` will be checked for execution modules to sync. If no top + files are found, then the ``base`` environment will be synced. + refresh : True If ``True``, refresh the available execution modules on the minion. This refresh will be performed even if no new execution modules are @@ -308,10 +323,14 @@ def sync_states(saltenv=None, refresh=True, extmod_whitelist=None, extmod_blackl Sync state modules from ``salt://_states`` to the minion - saltenv : base + saltenv The fileserver environment from which to sync. To sync from more than one environment, pass a comma-separated list. + If not passed, then all environments configured in the :ref:`top files + ` will be checked for state modules to sync. If no top + files are found, then the ``base`` environment will be synced. + refresh : True If ``True``, refresh the available states on the minion. This refresh will be performed even if no new state modules are synced. Set to @@ -357,10 +376,10 @@ def refresh_grains(**kwargs): salt '*' saltutil.refresh_grains ''' - kwargs = salt.utils.clean_kwargs(**kwargs) + kwargs = salt.utils.args.clean_kwargs(**kwargs) _refresh_pillar = kwargs.pop('refresh_pillar', True) if kwargs: - salt.utils.invalid_kwargs(kwargs) + salt.utils.args.invalid_kwargs(kwargs) # Modules and pillar need to be refreshed in case grains changes affected # them, and the module refresh process reloads the grains and assigns the # newly-reloaded grains to each execution module's __grains__ dunder. @@ -376,10 +395,14 @@ def sync_grains(saltenv=None, refresh=True, extmod_whitelist=None, extmod_blackl Sync grains modules from ``salt://_grains`` to the minion - saltenv : base + saltenv The fileserver environment from which to sync. To sync from more than one environment, pass a comma-separated list. + If not passed, then all environments configured in the :ref:`top files + ` will be checked for grains modules to sync. If no top + files are found, then the ``base`` environment will be synced. + refresh : True If ``True``, refresh the available execution modules and recompile pillar data for the minion. This refresh will be performed even if no @@ -413,10 +436,14 @@ def sync_renderers(saltenv=None, refresh=True, extmod_whitelist=None, extmod_bla Sync renderers from ``salt://_renderers`` to the minion - saltenv : base + saltenv The fileserver environment from which to sync. To sync from more than one environment, pass a comma-separated list. + If not passed, then all environments configured in the :ref:`top files + ` will be checked for renderers to sync. If no top files + are found, then the ``base`` environment will be synced. + refresh : True If ``True``, refresh the available execution modules on the minion. This refresh will be performed even if no new renderers are synced. @@ -449,10 +476,14 @@ def sync_returners(saltenv=None, refresh=True, extmod_whitelist=None, extmod_bla Sync beacons from ``salt://_returners`` to the minion - saltenv : base + saltenv The fileserver environment from which to sync. To sync from more than one environment, pass a comma-separated list. + If not passed, then all environments configured in the :ref:`top files + ` will be checked for returners to sync. If no top files + are found, then the ``base`` environment will be synced. + refresh : True If ``True``, refresh the available execution modules on the minion. This refresh will be performed even if no new returners are synced. Set @@ -483,10 +514,14 @@ def sync_proxymodules(saltenv=None, refresh=False, extmod_whitelist=None, extmod Sync proxy modules from ``salt://_proxy`` to the minion - saltenv : base + saltenv The fileserver environment from which to sync. To sync from more than one environment, pass a comma-separated list. + If not passed, then all environments configured in the :ref:`top files + ` will be checked for proxy modules to sync. If no top + files are found, then the ``base`` environment will be synced. + refresh : True If ``True``, refresh the available execution modules on the minion. This refresh will be performed even if no new proxy modules are synced. @@ -518,10 +553,14 @@ def sync_engines(saltenv=None, refresh=False, extmod_whitelist=None, extmod_blac Sync engine modules from ``salt://_engines`` to the minion - saltenv : base + saltenv The fileserver environment from which to sync. To sync from more than one environment, pass a comma-separated list. + If not passed, then all environments configured in the :ref:`top files + ` will be checked for engines to sync. If no top files are + found, then the ``base`` environment will be synced. + refresh : True If ``True``, refresh the available execution modules on the minion. This refresh will be performed even if no new engine modules are synced. @@ -550,10 +589,14 @@ def sync_output(saltenv=None, refresh=True, extmod_whitelist=None, extmod_blackl ''' Sync outputters from ``salt://_output`` to the minion - saltenv : base + saltenv The fileserver environment from which to sync. To sync from more than one environment, pass a comma-separated list. + If not passed, then all environments configured in the :ref:`top files + ` will be checked for outputters to sync. If no top files + are found, then the ``base`` environment will be synced. + refresh : True If ``True``, refresh the available execution modules on the minion. This refresh will be performed even if no new outputters are synced. @@ -622,10 +665,14 @@ def sync_utils(saltenv=None, refresh=True, extmod_whitelist=None, extmod_blackli Sync utility modules from ``salt://_utils`` to the minion - saltenv : base + saltenv The fileserver environment from which to sync. To sync from more than one environment, pass a comma-separated list. + If not passed, then all environments configured in the :ref:`top files + ` will be checked for utility modules to sync. If no top + files are found, then the ``base`` environment will be synced. + refresh : True If ``True``, refresh the available execution modules on the minion. This refresh will be performed even if no new utility modules are @@ -681,10 +728,14 @@ def sync_log_handlers(saltenv=None, refresh=True, extmod_whitelist=None, extmod_ Sync log handlers from ``salt://_log_handlers`` to the minion - saltenv : base + saltenv The fileserver environment from which to sync. To sync from more than one environment, pass a comma-separated list. + If not passed, then all environments configured in the :ref:`top files + ` will be checked for log handlers to sync. If no top files + are found, then the ``base`` environment will be synced. + refresh : True If ``True``, refresh the available execution modules on the minion. This refresh will be performed even if no new log handlers are synced. @@ -1050,7 +1101,7 @@ def find_cached_job(jid): else: return 'Local jobs cache directory {0} not found'.format(job_dir) path = os.path.join(job_dir, 'return.p') - with salt.utils.fopen(path, 'rb') as fp_: + with salt.utils.files.fopen(path, 'rb') as fp_: buf = fp_.read() fp_.close() if buf: @@ -1235,7 +1286,7 @@ def _get_ssh_or_api_client(cfgfile, ssh=False): def _exec(client, tgt, fun, arg, timeout, tgt_type, ret, kwarg, **kwargs): if 'expr_form' in kwargs: - salt.utils.warn_until( + salt.utils.versions.warn_until( 'Fluorine', 'The target type should be passed using the \'tgt_type\' ' 'argument instead of \'expr_form\'. Support for using ' @@ -1395,7 +1446,7 @@ def runner(name, arg=None, kwarg=None, full_return=False, saltenv='base', jid=No kwarg = {} jid = kwargs.pop('__orchestration_jid__', jid) saltenv = kwargs.pop('__env__', saltenv) - kwargs = salt.utils.clean_kwargs(**kwargs) + kwargs = salt.utils.args.clean_kwargs(**kwargs) if kwargs: kwarg.update(kwargs) diff --git a/salt/modules/schedule.py b/salt/modules/schedule.py index d67b9ac561a..da186888530 100644 --- a/salt/modules/schedule.py +++ b/salt/modules/schedule.py @@ -14,11 +14,12 @@ import os import yaml # Import salt libs -import salt.utils +import salt.utils.event +import salt.utils.files import salt.utils.odict # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six __proxyenabled__ = ['*'] @@ -128,15 +129,6 @@ def list_(show_all=False, continue if '_seconds' in schedule[job]: - # if _seconds is greater than zero - # then include the original back in seconds. - # otherwise remove seconds from the listing as the - # original item didn't include it. - if schedule[job]['_seconds'] > 0: - schedule[job]['seconds'] = schedule[job]['_seconds'] - elif 'seconds' in schedule[job]: - del schedule[job]['seconds'] - # remove _seconds from the listing del schedule[job]['_seconds'] @@ -809,7 +801,7 @@ def reload_(): # move this file into an configurable opt sfn = '{0}/{1}/schedule.conf'.format(__opts__['config_dir'], os.path.dirname(__opts__['default_include'])) if os.path.isfile(sfn): - with salt.utils.fopen(sfn, 'rb') as fp_: + with salt.utils.files.fopen(sfn, 'rb') as fp_: try: schedule = yaml.safe_load(fp_.read()) except yaml.YAMLError as exc: diff --git a/salt/modules/scsi.py b/salt/modules/scsi.py index 5bbcd00ec2b..eb0deccbbb5 100644 --- a/salt/modules/scsi.py +++ b/salt/modules/scsi.py @@ -7,6 +7,7 @@ from __future__ import absolute_import import os.path import logging import salt.utils +import salt.utils.path log = logging.getLogger(__name__) @@ -34,7 +35,7 @@ def ls_(get_size=True): .. versionadded:: 2015.5.10 ''' - if not salt.utils.which('lsscsi'): + if not salt.utils.path.which('lsscsi'): __context__['retcode'] = 1 return 'scsi.ls not available - lsscsi command not found' diff --git a/salt/modules/seed.py b/salt/modules/seed.py index 7d77a031af6..6b5723847ec 100644 --- a/salt/modules/seed.py +++ b/salt/modules/seed.py @@ -13,8 +13,8 @@ import tempfile # Import salt libs import salt.crypt -import salt.utils import salt.utils.cloud +import salt.utils.files import salt.config import salt.syspaths import uuid @@ -31,7 +31,7 @@ __func_alias__ = { def _file_or_content(file_): if os.path.exists(file_): - with salt.utils.fopen(file_) as fic: + with salt.utils.files.fopen(file_) as fic: return fic.read() return file_ @@ -220,7 +220,7 @@ def mkconfig(config=None, # Write the new minion's config to a tmp file tmp_config = os.path.join(tmp, 'minion') - with salt.utils.fopen(tmp_config, 'w+') as fp_: + with salt.utils.files.fopen(tmp_config, 'w+') as fp_: fp_.write(salt.utils.cloud.salt_config_to_yaml(config)) # Generate keys for the minion @@ -230,16 +230,16 @@ def mkconfig(config=None, if preseeded: log.debug('Writing minion.pub to {0}'.format(pubkeyfn)) log.debug('Writing minion.pem to {0}'.format(privkeyfn)) - with salt.utils.fopen(pubkeyfn, 'w') as fic: + with salt.utils.files.fopen(pubkeyfn, 'w') as fic: fic.write(_file_or_content(pub_key)) - with salt.utils.fopen(privkeyfn, 'w') as fic: + with salt.utils.files.fopen(privkeyfn, 'w') as fic: fic.write(_file_or_content(priv_key)) os.chmod(pubkeyfn, 0o600) os.chmod(privkeyfn, 0o600) else: salt.crypt.gen_keys(tmp, 'minion', 2048) if approve_key and not preseeded: - with salt.utils.fopen(pubkeyfn) as fp_: + with salt.utils.files.fopen(pubkeyfn) as fp_: pubkey = fp_.read() __salt__['pillar.ext']({'virtkey': [id_, pubkey]}) @@ -274,7 +274,7 @@ def _check_resolv(mpt): if not os.path.isfile(resolv): replace = True if not replace: - with salt.utils.fopen(resolv, 'rb') as fp_: + with salt.utils.files.fopen(resolv, 'rb') as fp_: conts = fp_.read() if 'nameserver' not in conts: replace = True diff --git a/salt/modules/selinux.py b/salt/modules/selinux.py index df91a08e814..437e428fb08 100644 --- a/salt/modules/selinux.py +++ b/salt/modules/selinux.py @@ -17,12 +17,13 @@ import os import re # Import salt libs -import salt.utils +import salt.utils.files +import salt.utils.path import salt.utils.decorators as decorators from salt.exceptions import CommandExecutionError, SaltInvocationError # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six _SELINUX_FILETYPES = { @@ -46,7 +47,7 @@ def __virtual__(): # Iterate over all of the commands this module uses and make sure # each of them are available in the standard PATH to prevent breakage for cmd in required_cmds: - if not salt.utils.which(cmd): + if not salt.utils.path.which(cmd): return (False, cmd + ' is not in the path') # SELinux only makes sense on Linux *obviously* if __grains__['kernel'] == 'Linux': @@ -94,7 +95,7 @@ def getenforce(): return 'Disabled' try: enforce = os.path.join(_selinux_fs_path, 'enforce') - with salt.utils.fopen(enforce, 'r') as _fp: + with salt.utils.files.fopen(enforce, 'r') as _fp: if _fp.readline().strip() == '0': return 'Permissive' else: @@ -115,7 +116,7 @@ def getconfig(): ''' try: config = '/etc/selinux/config' - with salt.utils.fopen(config, 'r') as _fp: + with salt.utils.files.fopen(config, 'r') as _fp: for line in _fp: if line.strip().startswith('SELINUX='): return line.split('=')[1].capitalize().strip() @@ -158,7 +159,7 @@ def setenforce(mode): if getenforce() != 'Disabled': enforce = os.path.join(selinux_fs_path(), 'enforce') try: - with salt.utils.fopen(enforce, 'w') as _fp: + with salt.utils.files.fopen(enforce, 'w') as _fp: _fp.write(mode) except (IOError, OSError) as exc: msg = 'Could not write SELinux enforce file: {0}' @@ -166,10 +167,10 @@ def setenforce(mode): config = '/etc/selinux/config' try: - with salt.utils.fopen(config, 'r') as _cf: + with salt.utils.files.fopen(config, 'r') as _cf: conf = _cf.read() try: - with salt.utils.fopen(config, 'w') as _cf: + with salt.utils.files.fopen(config, 'w') as _cf: conf = re.sub(r"\nSELINUX=.*\n", "\nSELINUX=" + modestring + "\n", conf) _cf.write(conf) except (IOError, OSError) as exc: @@ -300,7 +301,7 @@ def install_semod(module_path): salt '*' selinux.install_semod [salt://]path/to/module.pp - .. versionadded:: develop + .. versionadded:: 2016.11.6 ''' if module_path.find('salt://') == 0: module_path = __salt__['cp.cache_file'](module_path) @@ -318,7 +319,7 @@ def remove_semod(module): salt '*' selinux.remove_semod module_name - .. versionadded:: develop + .. versionadded:: 2016.11.6 ''' cmd = 'semodule -r {0}'.format(module) return not __salt__['cmd.retcode'](cmd) @@ -403,7 +404,7 @@ def _context_string_to_dict(context): return ret -def _filetype_id_to_string(filetype='a'): +def filetype_id_to_string(filetype='a'): ''' Translates SELinux filetype single-letter representation to a more human-readable version (which is also used in `semanage fcontext -l`). @@ -444,7 +445,7 @@ def fcontext_get_policy(name, filetype=None, sel_type=None, sel_user=None, sel_l 'sel_role': '[^:]+', # se_role for file context is always object_r 'sel_type': sel_type or '[^:]+', 'sel_level': sel_level or '[^:]+'} - cmd_kwargs['filetype'] = '[[:alpha:] ]+' if filetype is None else _filetype_id_to_string(filetype) + cmd_kwargs['filetype'] = '[[:alpha:] ]+' if filetype is None else filetype_id_to_string(filetype) cmd = 'semanage fcontext -l | egrep ' + \ "'^{filespec}{spacer}{filetype}{spacer}{sel_user}:{sel_role}:{sel_type}:{sel_level}$'".format(**cmd_kwargs) current_entry_text = __salt__['cmd.shell'](cmd) diff --git a/salt/modules/sensors.py b/salt/modules/sensors.py index 77b19071492..2da4dcee707 100644 --- a/salt/modules/sensors.py +++ b/salt/modules/sensors.py @@ -6,18 +6,18 @@ Read lm-sensors ''' from __future__ import absolute_import -#Import python libs +# Import python libs import logging -#import salt libs -import salt.utils +# import Salt libs +import salt.utils.path log = logging.getLogger(__name__) def __virtual__(): - if salt.utils.which('sensors'): + if salt.utils.path.which('sensors'): return True return (False, 'sensors does not exist in the path') diff --git a/salt/modules/serverdensity_device.py b/salt/modules/serverdensity_device.py index ae08d1da4e8..fcbabcdfb38 100644 --- a/salt/modules/serverdensity_device.py +++ b/salt/modules/serverdensity_device.py @@ -14,7 +14,7 @@ import os import tempfile # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import map # pylint: disable=import-error,no-name-in-module,redefined-builtin from salt.exceptions import CommandExecutionError diff --git a/salt/modules/servicenow.py b/salt/modules/servicenow.py index 18422c80312..d4c5c460dc9 100644 --- a/salt/modules/servicenow.py +++ b/salt/modules/servicenow.py @@ -131,7 +131,7 @@ def non_structured_query(table, query=None, **kwargs): client = _get_client() client.table = table # underlying lib doesn't use six or past.basestring, - # does isinstance(x,str) + # does isinstance(x, str) # http://bit.ly/1VkMmpE if query is None: # try and assemble a query by keyword diff --git a/salt/modules/shadow.py b/salt/modules/shadow.py index 70a1bbcc322..785af50dd3b 100644 --- a/salt/modules/shadow.py +++ b/salt/modules/shadow.py @@ -19,7 +19,9 @@ except ImportError: pass # Import salt libs -import salt.utils +import salt.utils # Can be removed when is_true is moved +import salt.utils.files +import salt.utils.stringutils from salt.exceptions import CommandExecutionError try: import salt.utils.pycrypto @@ -288,9 +290,9 @@ def set_password(name, password, use_usermod=False): if not os.path.isfile(s_file): return ret lines = [] - with salt.utils.fopen(s_file, 'rb') as fp_: + with salt.utils.files.fopen(s_file, 'rb') as fp_: for line in fp_: - line = salt.utils.to_str(line) + line = salt.utils.stringutils.to_str(line) comps = line.strip().split(':') if comps[0] != name: lines.append(line) @@ -300,7 +302,7 @@ def set_password(name, password, use_usermod=False): comps[2] = str(changed_date.days) line = ':'.join(comps) lines.append('{0}\n'.format(line)) - with salt.utils.fopen(s_file, 'w+') as fp_: + with salt.utils.files.fopen(s_file, 'w+') as fp_: fp_.writelines(lines) uinfo = info(name) return uinfo['passwd'] == password diff --git a/salt/modules/slsutil.py b/salt/modules/slsutil.py index 564c558871f..af8008bb1c8 100644 --- a/salt/modules/slsutil.py +++ b/salt/modules/slsutil.py @@ -2,11 +2,15 @@ ''' Utility functions for use with or in SLS files ''' + +# Import Python libs from __future__ import absolute_import +# Import Salt libs import salt.exceptions import salt.loader import salt.template +import salt.utils.args import salt.utils.dictupdate @@ -134,3 +138,67 @@ def renderer(path=None, string=None, default_renderer='jinja|yaml', **kwargs): __opts__['renderer_blacklist'], __opts__['renderer_whitelist'], **kwargs) + + +def _get_serialize_fn(serializer, fn_name): + serializers = salt.loader.serializers(__opts__) + fns = getattr(serializers, serializer, None) + fn = getattr(fns, fn_name, None) + + if not fns: + raise salt.exceptions.CommandExecutionError( + "Serializer '{0}' not found.".format(serializer)) + + if not fn: + raise salt.exceptions.CommandExecutionError( + "Serializer '{0}' does not implement {1}.".format(serializer, + fn_name)) + + return fn + + +def serialize(serializer, obj, **mod_kwargs): + ''' + Serialize a Python object using a :py:mod:`serializer module + ` + + CLI Example: + + .. code-block:: bash + + salt '*' --no-parse=obj slsutil.serialize 'json' obj="{'foo': 'Foo!'} + + Jinja Example: + + .. code-block:: jinja + + {% set json_string = salt.slsutil.serialize('json', + {'foo': 'Foo!'}) %} + ''' + kwargs = salt.utils.args.clean_kwargs(**mod_kwargs) + return _get_serialize_fn(serializer, 'serialize')(obj, **kwargs) + + +def deserialize(serializer, stream_or_string, **mod_kwargs): + ''' + Deserialize a Python object using a :py:mod:`serializer module + ` + + CLI Example: + + .. code-block:: bash + + salt '*' slsutil.deserialize 'json' '{"foo": "Foo!"}' + salt '*' --no-parse=stream_or_string slsutil.deserialize 'json' \\ + stream_or_string='{"foo": "Foo!"}' + + Jinja Example: + + .. code-block:: jinja + + {% set python_object = salt.slsutil.deserialize('json', + '{"foo": "Foo!"}') %} + ''' + kwargs = salt.utils.args.clean_kwargs(**mod_kwargs) + return _get_serialize_fn(serializer, 'deserialize')(stream_or_string, + **kwargs) diff --git a/salt/modules/smartos_imgadm.py b/salt/modules/smartos_imgadm.py index 25bae318b8c..5b13ce3c00c 100644 --- a/salt/modules/smartos_imgadm.py +++ b/salt/modules/smartos_imgadm.py @@ -9,7 +9,8 @@ import logging import json # Import Salt libs -import salt.utils +import salt.utils.path +import salt.utils.platform import salt.utils.decorators as decorators log = logging.getLogger(__name__) @@ -30,7 +31,7 @@ def _check_imgadm(): ''' Looks to see if imgadm is present on the system ''' - return salt.utils.which('imgadm') + return salt.utils.path.which('imgadm') def _exit_status(retcode): @@ -69,7 +70,7 @@ def __virtual__(): ''' Provides imgadm only on SmartOS ''' - if salt.utils.is_smartos_globalzone() and _check_imgadm(): + if salt.utils.platform.is_smartos_globalzone() and _check_imgadm(): return __virtualname__ return ( False, diff --git a/salt/modules/smartos_nictagadm.py b/salt/modules/smartos_nictagadm.py index d0fad1cf194..a2ad7a0769f 100644 --- a/salt/modules/smartos_nictagadm.py +++ b/salt/modules/smartos_nictagadm.py @@ -15,7 +15,8 @@ from __future__ import absolute_import import logging # Import Salt libs -import salt.utils +import salt.utils.path +import salt.utils.platform import salt.utils.decorators as decorators log = logging.getLogger(__name__) @@ -34,21 +35,22 @@ def _check_nictagadm(): ''' Looks to see if nictagadm is present on the system ''' - return salt.utils.which('nictagadm') + return salt.utils.path.which('nictagadm') def _check_dladm(): ''' Looks to see if dladm is present on the system ''' - return salt.utils.which('dladm') + return salt.utils.path.which('dladm') def __virtual__(): ''' Provides nictagadm on SmartOS ''' - if salt.utils.is_smartos_globalzone() and _check_nictagadm() and _check_dladm(): + if salt.utils.platform.is_smartos_globalzone() \ + and _check_nictagadm() and _check_dladm(): return __virtualname__ return ( False, diff --git a/salt/modules/smartos_virt.py b/salt/modules/smartos_virt.py index e762da9c234..6c4c964e953 100644 --- a/salt/modules/smartos_virt.py +++ b/salt/modules/smartos_virt.py @@ -8,8 +8,9 @@ from __future__ import absolute_import import logging # Import Salt libs +import salt.utils.path +import salt.utils.platform from salt.exceptions import CommandExecutionError -import salt.utils log = logging.getLogger(__name__) @@ -21,7 +22,8 @@ def __virtual__(): ''' Provides virt on SmartOS ''' - if salt.utils.is_smartos_globalzone() and salt.utils.which('vmadm'): + if salt.utils.platform.is_smartos_globalzone() \ + and salt.utils.path.which('vmadm'): return __virtualname__ return ( False, diff --git a/salt/modules/smartos_vmadm.py b/salt/modules/smartos_vmadm.py index 90f0d835998..7b173c9fbb9 100644 --- a/salt/modules/smartos_vmadm.py +++ b/salt/modules/smartos_vmadm.py @@ -14,11 +14,17 @@ except ImportError: from pipes import quote as _quote_args # Import Salt libs -import salt.ext.six as six import salt.utils +import salt.utils.args +import salt.utils.path +import salt.utils.platform import salt.utils.decorators as decorators +import salt.utils.files from salt.utils.odict import OrderedDict +# Import 3rd-party libs +from salt.ext import six + log = logging.getLogger(__name__) # Function aliases @@ -35,21 +41,21 @@ def _check_vmadm(): ''' Looks to see if vmadm is present on the system ''' - return salt.utils.which('vmadm') + return salt.utils.path.which('vmadm') def _check_zfs(): ''' Looks to see if zfs is present on the system ''' - return salt.utils.which('zfs') + return salt.utils.path.which('zfs') def __virtual__(): ''' Provides vmadm on SmartOS ''' - if salt.utils.is_smartos_globalzone() and _check_vmadm(): + if salt.utils.platform.is_smartos_globalzone() and _check_vmadm(): return __virtualname__ return ( False, @@ -127,7 +133,7 @@ def _create_update_from_cfg(mode='create', uuid=None, vmcfg=None): # write json file vmadm_json_file = __salt__['temp.file'](prefix='vmadm-') - with salt.utils.fopen(vmadm_json_file, 'w') as vmadm_json: + with salt.utils.files.fopen(vmadm_json_file, 'w') as vmadm_json: vmadm_json.write(json.dumps(vmcfg)) # vmadm validate create|update [-f ] @@ -166,7 +172,7 @@ def _create_update_from_cfg(mode='create', uuid=None, vmcfg=None): return ret else: # cleanup json file (only when succesful to help troubleshooting) - salt.utils.safe_rm(vmadm_json_file) + salt.utils.files.safe_rm(vmadm_json_file) # return uuid if res['stderr'].startswith('Successfully created VM'): @@ -783,7 +789,7 @@ def create(from_file=None, **kwargs): ret = {} # prepare vmcfg vmcfg = {} - kwargs = salt.utils.clean_kwargs(**kwargs) + kwargs = salt.utils.args.clean_kwargs(**kwargs) for k, v in six.iteritems(kwargs): vmcfg[k] = v @@ -818,7 +824,7 @@ def update(vm, from_file=None, key='uuid', **kwargs): vmadm = _check_vmadm() # prepare vmcfg vmcfg = {} - kwargs = salt.utils.clean_kwargs(**kwargs) + kwargs = salt.utils.args.clean_kwargs(**kwargs) for k, v in six.iteritems(kwargs): vmcfg[k] = v diff --git a/salt/modules/smbios.py b/salt/modules/smbios.py index a7023bed574..4b4156962a0 100644 --- a/salt/modules/smbios.py +++ b/salt/modules/smbios.py @@ -18,8 +18,7 @@ import uuid import re # Import salt libs -# import salt.log -import salt.utils +import salt.utils.path # Solve the Chicken and egg problem where grains need to run before any # of the modules are loaded and are generally available for any usage. @@ -30,7 +29,7 @@ from salt.ext.six.moves import zip # pylint: disable=import-error,redefined-bui log = logging.getLogger(__name__) -DMIDECODER = salt.utils.which_bin(['dmidecode', 'smbios']) +DMIDECODER = salt.utils.path.which_bin(['dmidecode', 'smbios']) def __virtual__(): diff --git a/salt/modules/snapper.py b/salt/modules/snapper.py index 06f8ed6eb28..23384bfa9ff 100644 --- a/salt/modules/snapper.py +++ b/salt/modules/snapper.py @@ -26,7 +26,7 @@ except ImportError: HAS_PWD = False from salt.exceptions import CommandExecutionError -import salt.utils +import salt.utils.files try: @@ -796,7 +796,7 @@ def diff(config='root', filename=None, num_pre=None, num_post=None): if os.path.isfile(pre_file): pre_file_exists = True - with salt.utils.fopen(pre_file) as rfh: + with salt.utils.files.fopen(pre_file) as rfh: pre_file_content = rfh.readlines() else: pre_file_content = [] @@ -804,7 +804,7 @@ def diff(config='root', filename=None, num_pre=None, num_post=None): if os.path.isfile(post_file): post_file_exists = True - with salt.utils.fopen(post_file) as rfh: + with salt.utils.files.fopen(post_file) as rfh: post_file_content = rfh.readlines() else: post_file_content = [] diff --git a/salt/modules/solaris_fmadm.py b/salt/modules/solaris_fmadm.py index 71cc3fd1b86..585bb019766 100644 --- a/salt/modules/solaris_fmadm.py +++ b/salt/modules/solaris_fmadm.py @@ -14,7 +14,8 @@ from __future__ import absolute_import import logging # Import Salt libs -import salt.utils +import salt.utils.path +import salt.utils.platform import salt.utils.decorators as decorators from salt.utils.odict import OrderedDict @@ -34,21 +35,21 @@ def _check_fmadm(): ''' Looks to see if fmadm is present on the system ''' - return salt.utils.which('fmadm') + return salt.utils.path.which('fmadm') def _check_fmdump(): ''' Looks to see if fmdump is present on the system ''' - return salt.utils.which('fmdump') + return salt.utils.path.which('fmdump') def __virtual__(): ''' Provides fmadm only on Solaris ''' - if salt.utils.is_sunos() and \ + if salt.utils.platform.is_sunos() and \ _check_fmadm() and _check_fmdump(): return __virtualname__ return ( diff --git a/salt/modules/solaris_shadow.py b/salt/modules/solaris_shadow.py index ac50e5dd792..b9454105165 100644 --- a/salt/modules/solaris_shadow.py +++ b/salt/modules/solaris_shadow.py @@ -24,7 +24,7 @@ except ImportError: pass # We're most likely on a Windows machine. # Import salt libs -import salt.utils +import salt.utils.files from salt.exceptions import CommandExecutionError try: import salt.utils.pycrypto @@ -117,7 +117,7 @@ def info(name): s_file = '/etc/shadow' if not os.path.isfile(s_file): return ret - with salt.utils.fopen(s_file, 'rb') as ifile: + with salt.utils.files.fopen(s_file, 'rb') as ifile: for line in ifile: comps = line.strip().split(':') if comps[0] == name: @@ -280,7 +280,7 @@ def set_password(name, password): if not os.path.isfile(s_file): return ret lines = [] - with salt.utils.fopen(s_file, 'rb') as ifile: + with salt.utils.files.fopen(s_file, 'rb') as ifile: for line in ifile: comps = line.strip().split(':') if comps[0] != name: @@ -289,7 +289,7 @@ def set_password(name, password): comps[1] = password line = ':'.join(comps) lines.append('{0}\n'.format(line)) - with salt.utils.fopen(s_file, 'w+') as ofile: + with salt.utils.files.fopen(s_file, 'w+') as ofile: ofile.writelines(lines) uinfo = info(name) return uinfo['passwd'] == password diff --git a/salt/modules/solaris_system.py b/salt/modules/solaris_system.py index 5335146e460..0638b94fa4a 100644 --- a/salt/modules/solaris_system.py +++ b/salt/modules/solaris_system.py @@ -6,9 +6,12 @@ This module is assumes we are using solaris-like shutdown .. versionadded:: 2016.3.0 ''' +# Import Python libs from __future__ import absolute_import -import salt.utils +# Import Salt libs +import salt.utils.path +import salt.utils.platform # Define the module's virtual name __virtualname__ = 'system' @@ -18,9 +21,13 @@ def __virtual__(): ''' Only supported on Solaris-like systems ''' - if not salt.utils.is_sunos() or not salt.utils.which('shutdown'): - return (False, 'The system execution module failed to load: ' - 'only available on Solaris-like ystems with shutdown command.') + if not salt.utils.platform.is_sunos() \ + or not salt.utils.path.which('shutdown'): + return ( + False, + 'The system execution module failed to load: only available on ' + 'Solaris-like ystems with shutdown command.' + ) return __virtualname__ diff --git a/salt/modules/solaris_user.py b/salt/modules/solaris_user.py index f135d1dcf4f..e52c82c1377 100644 --- a/salt/modules/solaris_user.py +++ b/salt/modules/solaris_user.py @@ -23,7 +23,7 @@ import logging # Import salt libs import salt.utils -import salt.ext.six as six +from salt.ext import six from salt.exceptions import CommandExecutionError log = logging.getLogger(__name__) diff --git a/salt/modules/solarisips.py b/salt/modules/solarisips.py index b3f48a61cda..1e87a46144c 100644 --- a/salt/modules/solarisips.py +++ b/salt/modules/solarisips.py @@ -44,6 +44,7 @@ import logging # Import salt libs import salt.utils +import salt.utils.path import salt.utils.pkg from salt.exceptions import CommandExecutionError @@ -56,9 +57,9 @@ def __virtual__(): ''' Set the virtual pkg module if the os is Solaris 11 ''' - if __grains__['os'] == 'Solaris' \ + if __grains__['os_family'] == 'Solaris' \ and float(__grains__['kernelrelease']) > 5.10 \ - and salt.utils.which('pkg'): + and salt.utils.path.which('pkg'): return __virtualname__ return (False, 'The solarisips execution module failed to load: only available ' diff --git a/salt/modules/solarispkg.py b/salt/modules/solarispkg.py index f51667b6eb1..84588976d77 100644 --- a/salt/modules/solarispkg.py +++ b/salt/modules/solarispkg.py @@ -30,7 +30,7 @@ def __virtual__(): ''' Set the virtual pkg module if the os is Solaris ''' - if __grains__['os'] == 'Solaris' and float(__grains__['kernelrelease']) <= 5.10: + if __grains__['os_family'] == 'Solaris' and float(__grains__['kernelrelease']) <= 5.10: return __virtualname__ return (False, 'The solarispkg execution module failed to load: only available ' diff --git a/salt/modules/solr.py b/salt/modules/solr.py index 46f8dfd1ef3..4306e77af86 100644 --- a/salt/modules/solr.py +++ b/salt/modules/solr.py @@ -66,7 +66,7 @@ import os # Import 3rd-party libs # pylint: disable=no-name-in-module,import-error -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves.urllib.request import ( urlopen as _urlopen, HTTPBasicAuthHandler as _HTTPBasicAuthHandler, @@ -77,7 +77,7 @@ from salt.ext.six.moves.urllib.request import ( # pylint: enable=no-name-in-module,import-error # Import salt libs -import salt.utils +import salt.utils.path # ######################### PRIVATE METHODS ############################## @@ -89,9 +89,9 @@ def __virtual__(): Return: str/bool ''' - if salt.utils.which('solr'): + if salt.utils.path.which('solr'): return 'solr' - if salt.utils.which('apache-solr'): + if salt.utils.path.which('apache-solr'): return 'solr' return (False, 'The solr execution module failed to load: requires both the solr and apache-solr binaries in the path.') diff --git a/salt/modules/splunk.py b/salt/modules/splunk.py index 7fd589264d9..0d64fb30b12 100644 --- a/salt/modules/splunk.py +++ b/salt/modules/splunk.py @@ -28,7 +28,8 @@ import hmac import base64 import subprocess -# Import third party libs +# Import 3rd-party libs +from salt.ext import six HAS_LIBS = False try: import splunklib.client @@ -280,7 +281,7 @@ def update_user(email, profile="splunk", **kwargs): if k.lower() == 'name': continue if k.lower() == 'roles': - if isinstance(v, str): + if isinstance(v, six.string_types): v = v.split(',') if set(roles) != set(v): kwargs['roles'] = list(set(v)) diff --git a/salt/modules/splunk_search.py b/salt/modules/splunk_search.py index 150dd4f708c..c78192a0555 100644 --- a/salt/modules/splunk_search.py +++ b/salt/modules/splunk_search.py @@ -27,7 +27,7 @@ import yaml import urllib # Import third party libs -import salt.ext.six as six +from salt.ext import six HAS_LIBS = False try: import splunklib.client diff --git a/salt/modules/ssh.py b/salt/modules/ssh.py index 4969f2b8428..cba20355346 100644 --- a/salt/modules/ssh.py +++ b/salt/modules/ssh.py @@ -19,17 +19,19 @@ import re import subprocess # Import salt libs -import salt.ext.six as six +from salt.ext import six import salt.utils +import salt.utils.decorators.path import salt.utils.files -import salt.utils.decorators as decorators +import salt.utils.path +import salt.utils.platform from salt.exceptions import ( SaltInvocationError, CommandExecutionError, ) # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import range log = logging.getLogger(__name__) @@ -41,7 +43,7 @@ if six.PY3: def __virtual__(): # TODO: This could work on windows with some love - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): return (False, 'The module cannot be loaded on windows.') return True @@ -149,20 +151,27 @@ def _replace_auth_key( try: # open the file for both reading AND writing - with salt.utils.fopen(full, 'r') as _fh: + with salt.utils.files.fopen(full, 'r') as _fh: for line in _fh: + # We don't need any whitespace-only containing lines or arbitrary doubled newlines + line = line.strip() + if line == '': + continue + line += '\n' + if line.startswith('#'): # Commented Line lines.append(line) continue comps = re.findall(r'((.*)\s)?(ssh-[a-z0-9-]+|ecdsa-[a-z0-9-]+)\s([a-zA-Z0-9+/]+={0,2})(\s(.*))?', line) if len(comps) > 0 and len(comps[0]) > 3 and comps[0][3] == key: + # Found our key, replace it lines.append(auth_line) else: lines.append(line) _fh.close() # Re-open the file writable after properly closing it - with salt.utils.fopen(full, 'w') as _fh: + with salt.utils.files.fopen(full, 'w') as _fh: # Write out any changes _fh.writelines(lines) except (IOError, OSError) as exc: @@ -171,7 +180,7 @@ def _replace_auth_key( ) -def _validate_keys(key_file): +def _validate_keys(key_file, fingerprint_hash_type): ''' Return a dict containing validated keys in the passed file ''' @@ -179,8 +188,14 @@ def _validate_keys(key_file): linere = re.compile(r'^(.*?)\s?((?:ssh\-|ecds)[\w-]+\s.+)$') try: - with salt.utils.fopen(key_file, 'r') as _fh: + with salt.utils.files.fopen(key_file, 'r') as _fh: for line in _fh: + # We don't need any whitespace-only containing lines or arbitrary doubled newlines + line = line.strip() + if line == '': + continue + line += '\n' + if line.startswith('#'): # Commented Line continue @@ -207,7 +222,7 @@ def _validate_keys(key_file): enc = comps[0] key = comps[1] comment = ' '.join(comps[2:]) - fingerprint = _fingerprint(key) + fingerprint = _fingerprint(key, fingerprint_hash_type) if fingerprint is None: continue @@ -223,7 +238,7 @@ def _validate_keys(key_file): return ret -def _fingerprint(public_key, fingerprint_hash_type=None): +def _fingerprint(public_key, fingerprint_hash_type): ''' Return a public key fingerprint based on its base64-encoded representation @@ -246,9 +261,6 @@ def _fingerprint(public_key, fingerprint_hash_type=None): if fingerprint_hash_type: hash_type = fingerprint_hash_type.lower() else: - # Set fingerprint_hash_type to md5 as default - log.warning('Public Key hashing currently defaults to "md5". This will ' - 'change to "sha256" in the 2017.7.0 release.') hash_type = 'sha256' try: @@ -341,7 +353,7 @@ def host_keys(keydir=None, private=True, certs=True): if m.group('pub'): kname += m.group('pub') try: - with salt.utils.fopen(os.path.join(keydir, fn_), 'r') as _fh: + with salt.utils.files.fopen(os.path.join(keydir, fn_), 'r') as _fh: # As of RFC 4716 "a key file is a text file, containing a # sequence of lines", although some SSH implementations # (e.g. OpenSSH) manage their own format(s). Please see @@ -358,7 +370,9 @@ def host_keys(keydir=None, private=True, certs=True): return keys -def auth_keys(user=None, config='.ssh/authorized_keys'): +def auth_keys(user=None, + config='.ssh/authorized_keys', + fingerprint_hash_type=None): ''' Return the authorized keys for users @@ -388,7 +402,7 @@ def auth_keys(user=None, config='.ssh/authorized_keys'): pass if full and os.path.isfile(full): - keys[u] = _validate_keys(full) + keys[u] = _validate_keys(full, fingerprint_hash_type) if old_output_when_one_user: if user[0] in keys: @@ -402,7 +416,8 @@ def auth_keys(user=None, config='.ssh/authorized_keys'): def check_key_file(user, source, config='.ssh/authorized_keys', - saltenv='base'): + saltenv='base', + fingerprint_hash_type=None): ''' Check a keyfile from a source destination against the local keys and return the keys to change @@ -416,7 +431,7 @@ def check_key_file(user, keyfile = __salt__['cp.cache_file'](source, saltenv) if not keyfile: return {} - s_keys = _validate_keys(keyfile) + s_keys = _validate_keys(keyfile, fingerprint_hash_type) if not s_keys: err = 'No keys detected in {0}. Is file properly ' \ 'formatted?'.format(source) @@ -432,12 +447,19 @@ def check_key_file(user, s_keys[key]['enc'], s_keys[key]['comment'], s_keys[key]['options'], - config) + config=config, + fingerprint_hash_type=fingerprint_hash_type) return ret -def check_key(user, key, enc, comment, options, config='.ssh/authorized_keys', - cache_keys=None): +def check_key(user, + key, + enc, + comment, + options, + config='.ssh/authorized_keys', + cache_keys=None, + fingerprint_hash_type=None): ''' Check to see if a key needs updating, returns "update", "add" or "exists" @@ -450,7 +472,9 @@ def check_key(user, key, enc, comment, options, config='.ssh/authorized_keys', if cache_keys is None: cache_keys = [] enc = _refine_enc(enc) - current = auth_keys(user, config) + current = auth_keys(user, + config=config, + fingerprint_hash_type=fingerprint_hash_type) nline = _format_auth_line(key, enc, comment, options) # Removing existing keys from the auth_keys isn't really a good idea @@ -479,9 +503,10 @@ def check_key(user, key, enc, comment, options, config='.ssh/authorized_keys', def rm_auth_key_from_file(user, - source, - config='.ssh/authorized_keys', - saltenv='base'): + source, + config='.ssh/authorized_keys', + saltenv='base', + fingerprint_hash_type=None): ''' Remove an authorized key from the specified user's authorized key file, using a file as source @@ -498,7 +523,7 @@ def rm_auth_key_from_file(user, 'Failed to pull key file from salt file server' ) - s_keys = _validate_keys(lfile) + s_keys = _validate_keys(lfile, fingerprint_hash_type) if not s_keys: err = ( 'No keys detected in {0}. Is file properly formatted?'.format( @@ -514,7 +539,8 @@ def rm_auth_key_from_file(user, rval += rm_auth_key( user, key, - config + config=config, + fingerprint_hash_type=fingerprint_hash_type ) # Due to the ability for a single file to have multiple keys, it's # possible for a single call to this function to have both "replace" @@ -528,7 +554,10 @@ def rm_auth_key_from_file(user, return 'Key not present' -def rm_auth_key(user, key, config='.ssh/authorized_keys'): +def rm_auth_key(user, + key, + config='.ssh/authorized_keys', + fingerprint_hash_type=None): ''' Remove an authorized key from the specified user's authorized key file @@ -538,7 +567,9 @@ def rm_auth_key(user, key, config='.ssh/authorized_keys'): salt '*' ssh.rm_auth_key ''' - current = auth_keys(user, config) + current = auth_keys(user, + config=config, + fingerprint_hash_type=fingerprint_hash_type) linere = re.compile(r'^(.*?)\s?((?:ssh\-|ecds)[\w-]+\s.+)$') if key in current: # Remove the key @@ -552,8 +583,14 @@ def rm_auth_key(user, key, config='.ssh/authorized_keys'): try: # Read every line in the file to find the right ssh key # and then write out the correct one. Open the file once - with salt.utils.fopen(full, 'r') as _fh: + with salt.utils.files.fopen(full, 'r') as _fh: for line in _fh: + # We don't need any whitespace-only containing lines or arbitrary doubled newlines + line = line.strip() + if line == '': + continue + line += '\n' + if line.startswith('#'): # Commented Line lines.append(line) @@ -583,7 +620,7 @@ def rm_auth_key(user, key, config='.ssh/authorized_keys'): # Let the context manager do the right thing here and then # re-open the file in write mode to save the changes out. - with salt.utils.fopen(full, 'w') as _fh: + with salt.utils.files.fopen(full, 'w') as _fh: _fh.writelines(lines) except (IOError, OSError) as exc: log.warning('Could not read/write key file: {0}'.format(str(exc))) @@ -596,7 +633,8 @@ def rm_auth_key(user, key, config='.ssh/authorized_keys'): def set_auth_key_from_file(user, source, config='.ssh/authorized_keys', - saltenv='base'): + saltenv='base', + fingerprint_hash_type=None): ''' Add a key to the authorized_keys file, using a file as the source. @@ -613,7 +651,7 @@ def set_auth_key_from_file(user, 'Failed to pull key file from salt file server' ) - s_keys = _validate_keys(lfile) + s_keys = _validate_keys(lfile, fingerprint_hash_type) if not s_keys: err = ( 'No keys detected in {0}. Is file properly formatted?'.format( @@ -629,11 +667,12 @@ def set_auth_key_from_file(user, rval += set_auth_key( user, key, - s_keys[key]['enc'], - s_keys[key]['comment'], - s_keys[key]['options'], - config, - list(s_keys.keys()) + enc=s_keys[key]['enc'], + comment=s_keys[key]['comment'], + options=s_keys[key]['options'], + config=config, + cache_keys=list(s_keys.keys()), + fingerprint_hash_type=fingerprint_hash_type ) # Due to the ability for a single file to have multiple keys, it's # possible for a single call to this function to have both "replace" @@ -656,7 +695,8 @@ def set_auth_key( comment='', options=None, config='.ssh/authorized_keys', - cache_keys=None): + cache_keys=None, + fingerprint_hash_type=None): ''' Add a key to the authorized_keys file. The "key" parameter must only be the string of text that is the encoded key. If the key begins with "ssh-rsa" @@ -683,11 +723,18 @@ def set_auth_key( # the same filtering done when reading the authorized_keys file. Apply # the same check to ensure we don't insert anything that will not # subsequently be read) - key_is_valid = _fingerprint(key) is not None + key_is_valid = _fingerprint(key, fingerprint_hash_type) is not None if not key_is_valid: return 'Invalid public key' - status = check_key(user, key, enc, comment, options, config, cache_keys) + status = check_key(user, + key, + enc, + comment, + options, + config=config, + cache_keys=cache_keys, + fingerprint_hash_type=fingerprint_hash_type) if status == 'update': _replace_auth_key(user, key, enc, comment, options or [], config) return 'replace' @@ -708,7 +755,7 @@ def set_auth_key( os.chown(dpath, uinfo['uid'], uinfo['gid']) os.chmod(dpath, 448) # If SELINUX is available run a restorecon on the file - rcon = salt.utils.which('restorecon') + rcon = salt.utils.path.which('restorecon') if rcon: cmd = [rcon, dpath] subprocess.call(cmd) @@ -719,13 +766,16 @@ def set_auth_key( new_file = False try: - with salt.utils.fopen(fconfig, 'ab+') as _fh: + with salt.utils.files.fopen(fconfig, 'ab+') as _fh: if new_file is False: # Let's make sure we have a new line at the end of the file - _fh.seek(1024, 2) - if not _fh.read(1024).rstrip(six.b(' ')).endswith(six.b('\n')): - _fh.seek(0, 2) - _fh.write(six.b('\n')) + _fh.seek(0, 2) + if _fh.tell() > 0: + # File isn't empty, check if last byte is a newline + # If not, add one + _fh.seek(-1, 2) + if _fh.read(1) != six.b('\n'): + _fh.write(six.b('\n')) if six.PY3: auth_line = auth_line.encode(__salt_system_encoding__) _fh.write(auth_line) @@ -738,7 +788,7 @@ def set_auth_key( os.chown(fconfig, uinfo['uid'], uinfo['gid']) os.chmod(fconfig, 384) # If SELINUX is available run a restorecon on the file - rcon = salt.utils.which('restorecon') + rcon = salt.utils.path.which('restorecon') if rcon: cmd = [rcon, fconfig] subprocess.call(cmd) @@ -751,6 +801,12 @@ def _parse_openssh_output(lines, fingerprint_hash_type=None): and yield dict with keys information, one by one. ''' for line in lines: + # We don't need any whitespace-only containing lines or arbitrary doubled newlines + line = line.strip() + if line == '': + continue + line += '\n' + if line.startswith('#'): continue try: @@ -765,7 +821,7 @@ def _parse_openssh_output(lines, fingerprint_hash_type=None): 'fingerprint': fingerprint} -@decorators.which('ssh-keygen') +@salt.utils.decorators.path.which('ssh-keygen') def get_known_host(user, hostname, config=None, @@ -798,7 +854,7 @@ def get_known_host(user, return known_hosts[0] if known_hosts else None -@decorators.which('ssh-keyscan') +@salt.utils.decorators.path.which('ssh-keyscan') def recv_known_host(hostname, enc=None, port=None, @@ -1116,7 +1172,7 @@ def set_known_host(user=None, # write line to known_hosts file try: - with salt.utils.fopen(full, 'a') as ofile: + with salt.utils.files.fopen(full, 'a') as ofile: ofile.write(line) except (IOError, OSError) as exception: raise CommandExecutionError( @@ -1204,7 +1260,7 @@ def user_keys(user=None, pubfile=None, prvfile=None): if os.path.exists(fn_): try: - with salt.utils.fopen(fn_, 'r') as _fh: + with salt.utils.files.fopen(fn_, 'r') as _fh: keys[u][keyname] = ''.join(_fh.readlines()).strip() except (IOError, OSError): pass @@ -1217,7 +1273,7 @@ def user_keys(user=None, pubfile=None, prvfile=None): return _keys -@decorators.which('ssh-keygen') +@salt.utils.decorators.path.which('ssh-keygen') def hash_known_hosts(user=None, config=None): ''' @@ -1282,7 +1338,7 @@ def key_is_encrypted(key): salt '*' ssh.key_is_encrypted /root/id_rsa ''' try: - with salt.utils.fopen(key, 'r') as fp_: + with salt.utils.files.fopen(key, 'r') as fp_: key_data = fp_.read() except (IOError, OSError) as exc: # Raise a CommandExecutionError diff --git a/salt/modules/ssh_package.py b/salt/modules/ssh_package.py index b675fd497be..d71d4791806 100644 --- a/salt/modules/ssh_package.py +++ b/salt/modules/ssh_package.py @@ -4,11 +4,11 @@ Service support for the REST example ''' from __future__ import absolute_import -# Import python libs +# Import Python libs import logging -# Import Salt's libs -import salt.utils +# Import Salts libs +import salt.utils.platform log = logging.getLogger(__name__) @@ -22,12 +22,21 @@ def __virtual__(): Only work on proxy ''' try: - if salt.utils.is_proxy() and __opts__['proxy']['proxytype'] == 'ssh_sample': + if salt.utils.platform.is_proxy() \ + and __opts__['proxy']['proxytype'] == 'ssh_sample': return __virtualname__ except KeyError: - return (False, 'The ssh_package execution module failed to load. Check the proxy key in pillar.') + return ( + False, + 'The ssh_package execution module failed to load. Check the ' + 'proxy key in pillar.' + ) - return (False, 'The ssh_package execution module failed to load: only works on an ssh_sample proxy minion.') + return ( + False, + 'The ssh_package execution module failed to load: only works on an ' + 'ssh_sample proxy minion.' + ) def list_pkgs(versions_as_list=False, **kwargs): diff --git a/salt/modules/ssh_service.py b/salt/modules/ssh_service.py index 04a1c92b963..731dd281f18 100644 --- a/salt/modules/ssh_service.py +++ b/salt/modules/ssh_service.py @@ -3,13 +3,15 @@ Provide the service module for the proxy-minion SSH sample .. versionadded:: 2015.8.2 ''' -# Import python libs +# Import Python libs from __future__ import absolute_import import logging -import salt.utils import fnmatch import re +# Import Salt libs +import salt.utils.platform + log = logging.getLogger(__name__) __func_alias__ = { @@ -25,12 +27,21 @@ def __virtual__(): Only work on systems that are a proxy minion ''' try: - if salt.utils.is_proxy() and __opts__['proxy']['proxytype'] == 'ssh_sample': + if salt.utils.platform.is_proxy() \ + and __opts__['proxy']['proxytype'] == 'ssh_sample': return __virtualname__ except KeyError: - return (False, 'The ssh_service execution module failed to load. Check the proxy key in pillar.') + return ( + False, + 'The ssh_service execution module failed to load. Check the ' + 'proxy key in pillar.' + ) - return (False, 'The ssh_service execution module failed to load: only works on an ssh_sample proxy minion.') + return ( + False, + 'The ssh_service execution module failed to load: only works on an ' + 'ssh_sample proxy minion.' + ) def get_all(): diff --git a/salt/modules/state.py b/salt/modules/state.py index a5ae950d2d6..bd2d90893f0 100644 --- a/salt/modules/state.py +++ b/salt/modules/state.py @@ -29,13 +29,17 @@ import salt.config import salt.payload import salt.state import salt.utils +import salt.utils.event +import salt.utils.files import salt.utils.jid +import salt.utils.platform import salt.utils.url +import salt.utils.versions from salt.exceptions import CommandExecutionError, SaltInvocationError from salt.runners.state import orchestrate as _orchestrate # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six __proxyenabled__ = ['*'] @@ -94,8 +98,7 @@ def _set_retcode(ret, highstate=None): if isinstance(ret, list): __context__['retcode'] = 1 return - if not salt.utils.check_state_result(ret, highstate=highstate): - + if not __utils__['state.check_result'](ret, highstate=highstate): __context__['retcode'] = 2 @@ -117,7 +120,7 @@ def _wait(jid): Wait for all previously started state jobs to finish running ''' if jid is None: - jid = salt.utils.jid.gen_jid() + jid = salt.utils.jid.gen_jid(__opts__) states = _prior_running_states(jid) while states: time.sleep(1) @@ -253,16 +256,38 @@ def _check_queue(queue, kwargs): return conflict -def _get_opts(localconfig=None): +def _get_opts(**kwargs): ''' Return a copy of the opts for use, optionally load a local config on top ''' opts = copy.deepcopy(__opts__) - if localconfig: - opts = salt.config.minion_config(localconfig, defaults=opts) + + if 'localconfig' in kwargs: + return salt.config.minion_config(kwargs['localconfig'], defaults=opts) + + if 'saltenv' in kwargs: + saltenv = kwargs['saltenv'] + if saltenv is not None and not isinstance(saltenv, six.string_types): + opts['environment'] = str(kwargs['saltenv']) + else: + opts['environment'] = kwargs['saltenv'] + + if 'pillarenv' in kwargs or opts.get('pillarenv_from_saltenv', False): + pillarenv = kwargs.get('pillarenv') or kwargs.get('saltenv') + if pillarenv is not None and not isinstance(pillarenv, six.string_types): + opts['pillarenv'] = str(pillarenv) + else: + opts['pillarenv'] = pillarenv + return opts +def _get_initial_pillar(opts): + return __pillar__ if __opts__['__cli'] == 'salt-call' \ + and opts['pillarenv'] == __opts__['pillarenv'] \ + else None + + def low(data, queue=False, **kwargs): ''' Execute a single low data call @@ -290,7 +315,7 @@ def low(data, queue=False, **kwargs): ret = st_.call(data) if isinstance(ret, list): __context__['retcode'] = 1 - if salt.utils.check_state_result(ret): + if __utils__['state.check_result'](ret): __context__['retcode'] = 2 return ret @@ -326,24 +351,31 @@ def high(data, test=None, queue=False, **kwargs): conflict = _check_queue(queue, kwargs) if conflict is not None: return conflict - opts = _get_opts(kwargs.get('localconfig')) + opts = _get_opts(**kwargs) opts['test'] = _get_test_value(test, **kwargs) - pillar = kwargs.get('pillar') + pillar_override = kwargs.get('pillar') pillar_enc = kwargs.get('pillar_enc') if pillar_enc is None \ - and pillar is not None \ - and not isinstance(pillar, dict): + and pillar_override is not None \ + and not isinstance(pillar_override, dict): raise SaltInvocationError( 'Pillar data must be formatted as a dictionary, unless pillar_enc ' 'is specified.' ) try: - st_ = salt.state.State(opts, pillar, pillar_enc=pillar_enc, proxy=__proxy__, - context=__context__) + st_ = salt.state.State(opts, + pillar_override, + pillar_enc=pillar_enc, + proxy=__proxy__, + context=__context__, + initial_pillar=_get_initial_pillar(opts)) except NameError: - st_ = salt.state.State(opts, pillar, pillar_enc=pillar_enc) + st_ = salt.state.State(opts, + pillar_override, + pillar_enc=pillar_enc, + initial_pillar=_get_initial_pillar(opts)) ret = st_.call_high(data) _set_retcode(ret, highstate=data) @@ -364,27 +396,23 @@ def template(tem, queue=False, **kwargs): salt '*' state.template '' ''' if 'env' in kwargs: - salt.utils.warn_until( - 'Oxygen', - 'Parameter \'env\' has been detected in the argument list. This ' - 'parameter is no longer used and has been replaced by \'saltenv\' ' - 'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.' - ) + # "env" is not supported; Use "saltenv". kwargs.pop('env') - if 'saltenv' in kwargs: - saltenv = kwargs['saltenv'] - else: - saltenv = '' - conflict = _check_queue(queue, kwargs) if conflict is not None: return conflict + + opts = _get_opts(**kwargs) try: - st_ = salt.state.HighState(__opts__, context=__context__, - proxy=__proxy__) + st_ = salt.state.HighState(opts, + context=__context__, + proxy=__proxy__, + initial_pillar=_get_initial_pillar(opts)) except NameError: - st_ = salt.state.HighState(__opts__, context=__context__) + st_ = salt.state.HighState(opts, + context=__context__, + initial_pillar=_get_initial_pillar(opts)) if not _check_pillar(kwargs, st_.opts['pillar']): __context__['retcode'] = 5 @@ -393,7 +421,11 @@ def template(tem, queue=False, **kwargs): if not tem.endswith('.sls'): tem = '{sls}.sls'.format(sls=tem) - high_state, errors = st_.render_state(tem, saltenv, '', None, local=True) + high_state, errors = st_.render_state(tem, + kwargs.get('saltenv', ''), + '', + None, + local=True) if errors: __context__['retcode'] = 1 return errors @@ -415,10 +447,15 @@ def template_str(tem, queue=False, **kwargs): conflict = _check_queue(queue, kwargs) if conflict is not None: return conflict + + opts = _get_opts(**kwargs) + try: - st_ = salt.state.State(__opts__, proxy=__proxy__) + st_ = salt.state.State(opts, + proxy=__proxy__, + initial_pillar=_get_initial_pillar(opts)) except NameError: - st_ = salt.state.State(__opts__) + st_ = salt.state.State(opts, initial_pillar=_get_initial_pillar(opts)) ret = st_.call_template_str(tem) _set_retcode(ret) return ret @@ -589,10 +626,10 @@ def request(mods=None, }) cumask = os.umask(0o77) try: - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): # Make sure cache file isn't read-only __salt__['cmd.run']('attrib -R "{0}"'.format(notify_path)) - with salt.utils.fopen(notify_path, 'w+b') as fp_: + with salt.utils.files.fopen(notify_path, 'w+b') as fp_: serial.dump(req, fp_) except (IOError, OSError): msg = 'Unable to write state request file {0}. Check permission.' @@ -616,7 +653,7 @@ def check_request(name=None): notify_path = os.path.join(__opts__['cachedir'], 'req_state.p') serial = salt.payload.Serial(__opts__) if os.path.isfile(notify_path): - with salt.utils.fopen(notify_path, 'rb') as fp_: + with salt.utils.files.fopen(notify_path, 'rb') as fp_: req = serial.load(fp_) if name: return req[name] @@ -653,10 +690,10 @@ def clear_request(name=None): return False cumask = os.umask(0o77) try: - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): # Make sure cache file isn't read-only __salt__['cmd.run']('attrib -R "{0}"'.format(notify_path)) - with salt.utils.fopen(notify_path, 'w+b') as fp_: + with salt.utils.files.fopen(notify_path, 'w+b') as fp_: serial.dump(req, fp_) except (IOError, OSError): msg = 'Unable to write state request file {0}. Check permission.' @@ -696,9 +733,7 @@ def run_request(name='default', **kwargs): return {} -def highstate(test=None, - queue=False, - **kwargs): +def highstate(test=None, queue=False, **kwargs): ''' Retrieve the state data from the salt master for this minion and execute it @@ -760,7 +795,7 @@ def highstate(test=None, states to be run with their own custom minion configuration, including different pillars, file_roots, etc. - mock: + mock The mock option allows for the state run to execute without actually calling any states. This then returns a mocked return which will show the requisite ordering as well as fully validate the state run. @@ -793,17 +828,12 @@ def highstate(test=None, return conflict orig_test = __opts__.get('test', None) - opts = _get_opts(kwargs.get('localconfig')) + opts = _get_opts(**kwargs) opts['test'] = _get_test_value(test, **kwargs) if 'env' in kwargs: - salt.utils.warn_until( - 'Oxygen', - 'Parameter \'env\' has been detected in the argument list. This ' - 'parameter is no longer used and has been replaced by \'saltenv\' ' - 'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.' - ) + # "env" is not supported; Use "saltenv". kwargs.pop('env') if 'saltenv' in kwargs: @@ -813,10 +843,11 @@ def highstate(test=None, opts['pillarenv'] = kwargs['pillarenv'] pillar = kwargs.get('pillar') + pillar_override = kwargs.get('pillar') pillar_enc = kwargs.get('pillar_enc') if pillar_enc is None \ - and pillar is not None \ - and not isinstance(pillar, dict): + and pillar_override is not None \ + and not isinstance(pillar_override, dict): raise SaltInvocationError( 'Pillar data must be formatted as a dictionary, unless pillar_enc ' 'is specified.' @@ -824,18 +855,20 @@ def highstate(test=None, try: st_ = salt.state.HighState(opts, - pillar, + pillar_override, kwargs.get('__pub_jid'), pillar_enc=pillar_enc, proxy=__proxy__, context=__context__, - mocked=kwargs.get('mock', False)) + mocked=kwargs.get('mock', False), + initial_pillar=_get_initial_pillar(opts)) except NameError: st_ = salt.state.HighState(opts, - pillar, + pillar_override, kwargs.get('__pub_jid'), pillar_enc=pillar_enc, - mocked=kwargs.get('mock', False)) + mocked=kwargs.get('mock', False), + initial_pillar=_get_initial_pillar(opts)) if not _check_pillar(kwargs, st_.opts['pillar']): __context__['retcode'] = 5 @@ -872,13 +905,7 @@ def highstate(test=None, return ret -def sls(mods, - saltenv=None, - test=None, - exclude=None, - queue=False, - pillarenv=None, - **kwargs): +def sls(mods, test=None, exclude=None, queue=False, **kwargs): ''' Execute the states in one or more SLS files @@ -968,24 +995,9 @@ def sls(mods, ''' concurrent = kwargs.get('concurrent', False) if 'env' in kwargs: - salt.utils.warn_until( - 'Oxygen', - 'Parameter \'env\' has been detected in the argument list. This ' - 'parameter is no longer used and has been replaced by \'saltenv\' ' - 'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.' - ) + # "env" is not supported; Use "saltenv". kwargs.pop('env') - if saltenv is None: - if __opts__.get('environment', None): - saltenv = __opts__['environment'] - else: - saltenv = 'base' - - if not pillarenv: - if __opts__.get('pillarenv', None): - pillarenv = __opts__['pillarenv'] - # Modification to __opts__ lost after this if-else if queue: _wait(kwargs.get('__pub_jid')) @@ -995,10 +1007,6 @@ def sls(mods, __context__['retcode'] = 1 return conflict - # Ensure desired environment - __opts__['environment'] = saltenv - __opts__['pillarenv'] = pillarenv - if isinstance(mods, list): disabled = _disabled(mods) else: @@ -1006,20 +1014,28 @@ def sls(mods, if disabled: for state in disabled: - log.debug('Salt state {0} run is disabled. To re-enable, run state.enable {0}'.format(state)) + log.debug( + 'Salt state %s is disabled. To re-enable, run ' + 'state.enable %s', state, state + ) __context__['retcode'] = 1 return disabled orig_test = __opts__.get('test', None) - opts = _get_opts(kwargs.get('localconfig')) + opts = _get_opts(**kwargs) opts['test'] = _get_test_value(test, **kwargs) - pillar = kwargs.get('pillar') + # Since this is running a specific SLS file (or files), fall back to the + # 'base' saltenv if none is configured and none was passed. + if opts['environment'] is None: + opts['environment'] = 'base' + + pillar_override = kwargs.get('pillar') pillar_enc = kwargs.get('pillar_enc') if pillar_enc is None \ - and pillar is not None \ - and not isinstance(pillar, dict): + and pillar_override is not None \ + and not isinstance(pillar_override, dict): raise SaltInvocationError( 'Pillar data must be formatted as a dictionary, unless pillar_enc ' 'is specified.' @@ -1030,20 +1046,23 @@ def sls(mods, __opts__['cachedir'], '{0}.cache.p'.format(kwargs.get('cache_name', 'highstate')) ) + try: st_ = salt.state.HighState(opts, - pillar, + pillar_override, kwargs.get('__pub_jid'), pillar_enc=pillar_enc, proxy=__proxy__, context=__context__, - mocked=kwargs.get('mock', False)) + mocked=kwargs.get('mock', False), + initial_pillar=_get_initial_pillar(opts)) except NameError: st_ = salt.state.HighState(opts, - pillar, + pillar_override, kwargs.get('__pub_jid'), pillar_enc=pillar_enc, - mocked=kwargs.get('mock', False)) + mocked=kwargs.get('mock', False), + initial_pillar=_get_initial_pillar(opts)) if not _check_pillar(kwargs, st_.opts['pillar']): __context__['retcode'] = 5 @@ -1055,7 +1074,7 @@ def sls(mods, umask = os.umask(0o77) if kwargs.get('cache'): if os.path.isfile(cfn): - with salt.utils.fopen(cfn, 'rb') as fp_: + with salt.utils.files.fopen(cfn, 'rb') as fp_: high_ = serial.load(fp_) return st_.state.call_high(high_, orchestration_jid) os.umask(umask) @@ -1066,14 +1085,14 @@ def sls(mods, st_.push_active() ret = {} try: - high_, errors = st_.render_highstate({saltenv: mods}) + high_, errors = st_.render_highstate({opts['environment']: mods}) if errors: __context__['retcode'] = 1 return errors if exclude: - if isinstance(exclude, str): + if isinstance(exclude, six.string_types): exclude = exclude.split(',') if '__exclude__' in high_: high_['__exclude__'].extend(exclude) @@ -1088,10 +1107,10 @@ def sls(mods, cache_file = os.path.join(__opts__['cachedir'], 'sls.p') cumask = os.umask(0o77) try: - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): # Make sure cache file isn't read-only __salt__['cmd.run'](['attrib', '-R', cache_file], python_shell=False) - with salt.utils.fopen(cache_file, 'w+b') as fp_: + with salt.utils.files.fopen(cache_file, 'w+b') as fp_: serial.dump(ret, fp_) except (IOError, OSError): msg = 'Unable to write to SLS cache file {0}. Check permission.' @@ -1102,7 +1121,7 @@ def sls(mods, __opts__['test'] = orig_test try: - with salt.utils.fopen(cfn, 'w+b') as fp_: + with salt.utils.files.fopen(cfn, 'w+b') as fp_: try: serial.dump(high_, fp_) except TypeError: @@ -1116,12 +1135,7 @@ def sls(mods, return ret -def top(topfn, - test=None, - queue=False, - saltenv=None, - pillarenv=None, - **kwargs): +def top(topfn, test=None, queue=False, **kwargs): ''' Execute a specific top file instead of the default. This is useful to apply configurations from a different environment (for example, dev or prod), without @@ -1158,31 +1172,31 @@ def top(topfn, if conflict is not None: return conflict orig_test = __opts__.get('test', None) - opts = _get_opts(kwargs.get('localconfig')) + opts = _get_opts(**kwargs) opts['test'] = _get_test_value(test, **kwargs) - if saltenv is not None: - opts['environment'] = saltenv - - if pillarenv is not None: - opts['pillarenv'] = pillarenv - - pillar = kwargs.get('pillar') + pillar_override = kwargs.get('pillar') pillar_enc = kwargs.get('pillar_enc') if pillar_enc is None \ - and pillar is not None \ - and not isinstance(pillar, dict): + and pillar_override is not None \ + and not isinstance(pillar_override, dict): raise SaltInvocationError( 'Pillar data must be formatted as a dictionary, unless pillar_enc ' 'is specified.' ) try: - st_ = salt.state.HighState(opts, pillar, pillar_enc=pillar_enc, - context=__context__, proxy=__proxy__) + st_ = salt.state.HighState(opts, + pillar_override, + pillar_enc=pillar_enc, + context=__context__, + proxy=__proxy__, + initial_pillar=_get_initial_pillar(opts)) except NameError: - st_ = salt.state.HighState(opts, pillar, pillar_enc=pillar_enc, - context=__context__) - + st_ = salt.state.HighState(opts, + pillar_override, + pillar_enc=pillar_enc, + context=__context__, + initial_pillar=_get_initial_pillar(opts)) if not _check_pillar(kwargs, st_.opts['pillar']): __context__['retcode'] = 5 err = ['Pillar failed to render with the following messages:'] @@ -1193,8 +1207,8 @@ def top(topfn, st_.opts['state_top'] = salt.utils.url.create(topfn) ret = {} orchestration_jid = kwargs.get('orchestration_jid') - if saltenv: - st_.opts['state_top_saltenv'] = saltenv + if 'saltenv' in kwargs: + st_.opts['state_top_saltenv'] = kwargs['saltenv'] try: snapper_pre = _snapper_pre(opts, kwargs.get('__pub_jid', 'called localy')) ret = st_.call_highstate( @@ -1228,21 +1242,28 @@ def show_highstate(queue=False, **kwargs): conflict = _check_queue(queue, kwargs) if conflict is not None: return conflict - pillar = kwargs.get('pillar') + pillar_override = kwargs.get('pillar') pillar_enc = kwargs.get('pillar_enc') if pillar_enc is None \ - and pillar is not None \ - and not isinstance(pillar, dict): + and pillar_override is not None \ + and not isinstance(pillar_override, dict): raise SaltInvocationError( 'Pillar data must be formatted as a dictionary, unless pillar_enc ' 'is specified.' ) + opts = _get_opts(**kwargs) try: - st_ = salt.state.HighState(__opts__, pillar, pillar_enc=pillar_enc, - proxy=__proxy__) + st_ = salt.state.HighState(opts, + pillar_override, + pillar_enc=pillar_enc, + proxy=__proxy__, + initial_pillar=_get_initial_pillar(opts)) except NameError: - st_ = salt.state.HighState(__opts__, pillar, pillar_enc=pillar_enc) + st_ = salt.state.HighState(opts, + pillar_override, + pillar_enc=pillar_enc, + initial_pillar=_get_initial_pillar(opts)) if not _check_pillar(kwargs, st_.opts['pillar']): __context__['retcode'] = 5 @@ -1272,10 +1293,15 @@ def show_lowstate(queue=False, **kwargs): if conflict is not None: assert False return conflict + + opts = _get_opts(**kwargs) try: - st_ = salt.state.HighState(__opts__, proxy=__proxy__) + st_ = salt.state.HighState(opts, + proxy=__proxy__, + initial_pillar=_get_initial_pillar(opts)) except NameError: - st_ = salt.state.HighState(__opts__) + st_ = salt.state.HighState(opts, + initial_pillar=_get_initial_pillar(opts)) if not _check_pillar(kwargs, st_.opts['pillar']): __context__['retcode'] = 5 @@ -1326,14 +1352,7 @@ def show_state_usage(queue=False, **kwargs): return ret -def sls_id( - id_, - mods, - saltenv='base', - pillarenv=None, - test=None, - queue=False, - **kwargs): +def sls_id(id_, mods, test=None, queue=False, **kwargs): ''' Call a single ID from the named module(s) and handle all requisites @@ -1369,26 +1388,21 @@ def sls_id( if conflict is not None: return conflict orig_test = __opts__.get('test', None) - opts = _get_opts(kwargs.get('localconfig')) + opts = _get_opts(**kwargs) opts['test'] = _get_test_value(test, **kwargs) - opts['environment'] = saltenv - if pillarenv is not None: - opts['pillarenv'] = pillarenv - pillar = kwargs.get('pillar') - pillar_enc = kwargs.get('pillar_enc') - if pillar_enc is None \ - and pillar is not None \ - and not isinstance(pillar, dict): - raise SaltInvocationError( - 'Pillar data must be formatted as a dictionary, unless pillar_enc ' - 'is specified.' - ) + # Since this is running a specific ID within a specific SLS file, fall back + # to the 'base' saltenv if none is configured and none was passed. + if opts['environment'] is None: + opts['environment'] = 'base' try: - st_ = salt.state.HighState(opts, pillar=pillar, pillar_enc=pillar_enc, proxy=__proxy__) + st_ = salt.state.HighState(opts, + proxy=__proxy__, + initial_pillar=_get_initial_pillar(opts)) except NameError: - st_ = salt.state.HighState(opts) + st_ = salt.state.HighState(opts, + initial_pillar=_get_initial_pillar(opts)) if not _check_pillar(kwargs, st_.opts['pillar']): __context__['retcode'] = 5 @@ -1400,7 +1414,7 @@ def sls_id( split_mods = mods.split(',') st_.push_active() try: - high_, errors = st_.render_highstate({saltenv: split_mods}) + high_, errors = st_.render_highstate({opts['environment']: split_mods}) finally: st_.pop_active() errors += st_.state.verify_high(high_) @@ -1420,17 +1434,12 @@ def sls_id( if not ret: raise SaltInvocationError( 'No matches for ID \'{0}\' found in SLS \'{1}\' within saltenv ' - '\'{2}\''.format(id_, mods, saltenv) + '\'{2}\''.format(id_, mods, opts['environment']) ) return ret -def show_low_sls(mods, - saltenv='base', - pillarenv=None, - test=None, - queue=False, - **kwargs): +def show_low_sls(mods, test=None, queue=False, **kwargs): ''' Display the low data from a specific sls. The default environment is ``base``, use ``saltenv`` to specify a different environment. @@ -1438,6 +1447,17 @@ def show_low_sls(mods, saltenv Specify a salt fileserver environment to be used when applying states + pillar + Custom Pillar values, passed as a dictionary of key-value pairs + + .. code-block:: bash + + salt '*' state.show_low_sls test pillar='{"foo": "bar"}' + + .. note:: + Values passed this way will override Pillar values set via + ``pillar_roots`` or an external Pillar source. + pillarenv Specify a Pillar environment to be used when applying states. This can also be set in the minion config file using the @@ -1450,29 +1470,43 @@ def show_low_sls(mods, .. code-block:: bash salt '*' state.show_low_sls foo + salt '*' state.show_low_sls foo saltenv=dev ''' if 'env' in kwargs: - salt.utils.warn_until( - 'Oxygen', - 'Parameter \'env\' has been detected in the argument list. This ' - 'parameter is no longer used and has been replaced by \'saltenv\' ' - 'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.' - ) + # "env" is not supported; Use "saltenv". kwargs.pop('env') conflict = _check_queue(queue, kwargs) if conflict is not None: return conflict orig_test = __opts__.get('test', None) - opts = _get_opts(kwargs.get('localconfig')) + opts = _get_opts(**kwargs) opts['test'] = _get_test_value(test, **kwargs) - opts['environment'] = saltenv - if pillarenv is not None: - opts['pillarenv'] = pillarenv + + # Since this is dealing with a specific SLS file (or files), fall back to + # the 'base' saltenv if none is configured and none was passed. + if opts['environment'] is None: + opts['environment'] = 'base' + + pillar_override = kwargs.get('pillar') + pillar_enc = kwargs.get('pillar_enc') + if pillar_enc is None \ + and pillar_override is not None \ + and not isinstance(pillar_override, dict): + raise SaltInvocationError( + 'Pillar data must be formatted as a dictionary, unless pillar_enc ' + 'is specified.' + ) + try: - st_ = salt.state.HighState(opts, proxy=__proxy__) + st_ = salt.state.HighState(opts, + pillar_override, + proxy=__proxy__, + initial_pillar=_get_initial_pillar(opts)) except NameError: - st_ = salt.state.HighState(opts) + st_ = salt.state.HighState(opts, + pillar_override, + initial_pillar=_get_initial_pillar(opts)) if not _check_pillar(kwargs, st_.opts['pillar']): __context__['retcode'] = 5 @@ -1483,7 +1517,7 @@ def show_low_sls(mods, mods = mods.split(',') st_.push_active() try: - high_, errors = st_.render_highstate({saltenv: mods}) + high_, errors = st_.render_highstate({opts['environment']: mods}) finally: st_.pop_active() errors += st_.state.verify_high(high_) @@ -1497,7 +1531,7 @@ def show_low_sls(mods, return ret -def show_sls(mods, saltenv='base', test=None, queue=False, **kwargs): +def show_sls(mods, test=None, queue=False, **kwargs): ''' Display the state data from a specific sls or list of sls files on the master. The default environment is ``base``, use ``saltenv`` to specify a @@ -1525,40 +1559,43 @@ def show_sls(mods, saltenv='base', test=None, queue=False, **kwargs): salt '*' state.show_sls core,edit.vim dev ''' if 'env' in kwargs: - salt.utils.warn_until( - 'Oxygen', - 'Parameter \'env\' has been detected in the argument list. This ' - 'parameter is no longer used and has been replaced by \'saltenv\' ' - 'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.' - ) + # "env" is not supported; Use "saltenv". kwargs.pop('env') conflict = _check_queue(queue, kwargs) if conflict is not None: return conflict orig_test = __opts__.get('test', None) - opts = _get_opts(kwargs.get('localconfig')) + opts = _get_opts(**kwargs) opts['test'] = _get_test_value(test, **kwargs) - pillar = kwargs.get('pillar') + # Since this is dealing with a specific SLS file (or files), fall back to + # the 'base' saltenv if none is configured and none was passed. + if opts['environment'] is None: + opts['environment'] = 'base' + + pillar_override = kwargs.get('pillar') pillar_enc = kwargs.get('pillar_enc') if pillar_enc is None \ - and pillar is not None \ - and not isinstance(pillar, dict): + and pillar_override is not None \ + and not isinstance(pillar_override, dict): raise SaltInvocationError( 'Pillar data must be formatted as a dictionary, unless pillar_enc ' 'is specified.' ) - if 'pillarenv' in kwargs: - opts['pillarenv'] = kwargs['pillarenv'] - try: - st_ = salt.state.HighState(opts, pillar, pillar_enc=pillar_enc, - proxy=__proxy__) + st_ = salt.state.HighState(opts, + pillar_override, + pillar_enc=pillar_enc, + proxy=__proxy__, + initial_pillar=_get_initial_pillar(opts)) except NameError: - st_ = salt.state.HighState(opts, pillar, pillar_enc=pillar_enc) + st_ = salt.state.HighState(opts, + pillar_override, + pillar_enc=pillar_enc, + initial_pillar=_get_initial_pillar(opts)) if not _check_pillar(kwargs, st_.opts['pillar']): __context__['retcode'] = 5 @@ -1569,7 +1606,7 @@ def show_sls(mods, saltenv='base', test=None, queue=False, **kwargs): mods = mods.split(',') st_.push_active() try: - high_, errors = st_.render_highstate({saltenv: mods}) + high_, errors = st_.render_highstate({opts['environment']: mods}) finally: st_.pop_active() errors += st_.state.verify_high(high_) @@ -1592,26 +1629,21 @@ def show_top(queue=False, **kwargs): salt '*' state.show_top ''' - opts = copy.deepcopy(__opts__) - if 'env' in kwargs: - salt.utils.warn_until( - 'Oxygen', - 'Parameter \'env\' has been detected in the argument list. This ' - 'parameter is no longer used and has been replaced by \'saltenv\' ' - 'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.' - ) + # "env" is not supported; Use "saltenv". kwargs.pop('env') - if 'saltenv' in kwargs: - opts['environment'] = kwargs['saltenv'] conflict = _check_queue(queue, kwargs) if conflict is not None: return conflict + + opts = _get_opts(**kwargs) try: - st_ = salt.state.HighState(opts, proxy=__proxy__) + st_ = salt.state.HighState(opts, + proxy=__proxy__, + initial_pillar=_get_initial_pillar(opts)) except NameError: - st_ = salt.state.HighState(opts) + st_ = salt.state.HighState(opts, initial_pillar=_get_initial_pillar(opts)) if not _check_pillar(kwargs, st_.opts['pillar']): __context__['retcode'] = 5 @@ -1657,23 +1689,30 @@ def single(fun, name, test=None, queue=False, **kwargs): '__id__': name, 'name': name}) orig_test = __opts__.get('test', None) - opts = _get_opts(kwargs.get('localconfig')) + opts = _get_opts(**kwargs) opts['test'] = _get_test_value(test, **kwargs) - pillar = kwargs.get('pillar') + pillar_override = kwargs.get('pillar') pillar_enc = kwargs.get('pillar_enc') if pillar_enc is None \ - and pillar is not None \ - and not isinstance(pillar, dict): + and pillar_override is not None \ + and not isinstance(pillar_override, dict): raise SaltInvocationError( 'Pillar data must be formatted as a dictionary, unless pillar_enc ' 'is specified.' ) try: - st_ = salt.state.State(opts, pillar, pillar_enc=pillar_enc, proxy=__proxy__) + st_ = salt.state.State(opts, + pillar_override, + pillar_enc=pillar_enc, + proxy=__proxy__, + initial_pillar=_get_initial_pillar(opts)) except NameError: - st_ = salt.state.State(opts, pillar, pillar_enc=pillar_enc) + st_ = salt.state.State(opts, + pillar_override, + pillar_enc=pillar_enc, + initial_pillar=_get_initial_pillar(opts)) err = st_.verify_data(kwargs) if err: __context__['retcode'] = 1 @@ -1681,7 +1720,7 @@ def single(fun, name, test=None, queue=False, **kwargs): st_._mod_init(kwargs) snapper_pre = _snapper_pre(opts, kwargs.get('__pub_jid', 'called localy')) - ret = {'{0[state]}_|-{0[__id__]}_|-{0[name]}_|-{0[fun]}'.format(kwargs): + ret = {u'{0[state]}_|-{0[__id__]}_|-{0[name]}_|-{0[fun]}'.format(kwargs): st_.call(kwargs)} _set_retcode(ret) # Work around Windows multiprocessing bug, set __opts__['test'] back to @@ -1716,7 +1755,11 @@ def clear_cache(): return ret -def pkg(pkg_path, pkg_sum, hash_type, test=None, **kwargs): +def pkg(pkg_path, + pkg_sum, + hash_type, + test=None, + **kwargs): ''' Execute a packaged state run, the packaged state run will exist in a tarball available locally. This packaged state @@ -1745,7 +1788,7 @@ def pkg(pkg_path, pkg_sum, hash_type, test=None, **kwargs): s_pkg.extractall(root) s_pkg.close() lowstate_json = os.path.join(root, 'lowstate.json') - with salt.utils.fopen(lowstate_json, 'r') as fp_: + with salt.utils.files.fopen(lowstate_json, 'r') as fp_: lowstate = json.load(fp_, object_hook=salt.utils.decode_dict) # Check for errors in the lowstate for chunk in lowstate: @@ -1753,16 +1796,17 @@ def pkg(pkg_path, pkg_sum, hash_type, test=None, **kwargs): return lowstate pillar_json = os.path.join(root, 'pillar.json') if os.path.isfile(pillar_json): - with salt.utils.fopen(pillar_json, 'r') as fp_: - pillar = json.load(fp_) + with salt.utils.files.fopen(pillar_json, 'r') as fp_: + pillar_override = json.load(fp_) else: - pillar = None + pillar_override = None + roster_grains_json = os.path.join(root, 'roster_grains.json') if os.path.isfile(roster_grains_json): - with salt.utils.fopen(roster_grains_json, 'r') as fp_: + with salt.utils.files.fopen(roster_grains_json, 'r') as fp_: roster_grains = json.load(fp_, object_hook=salt.utils.decode_dict) - popts = _get_opts(kwargs.get('localconfig')) + popts = _get_opts(**kwargs) if os.path.isfile(roster_grains_json): popts['grains'] = roster_grains popts['fileclient'] = 'local' @@ -1774,7 +1818,7 @@ def pkg(pkg_path, pkg_sum, hash_type, test=None, **kwargs): if not os.path.isdir(full): continue popts['file_roots'][fn_] = [full] - st_ = salt.state.State(popts, pillar=pillar) + st_ = salt.state.State(popts, pillar_override=pillar_override) snapper_pre = _snapper_pre(popts, kwargs.get('__pub_jid', 'called localy')) ret = st_.call_chunks(lowstate) ret = st_.call_listen(lowstate, ret) diff --git a/salt/modules/status.py b/salt/modules/status.py index f2eeaa42fae..edb268267ff 100644 --- a/salt/modules/status.py +++ b/salt/modules/status.py @@ -17,16 +17,17 @@ import time import logging # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import range # pylint: disable=import-error,no-name-in-module,redefined-builtin # Import salt libs import salt.config import salt.minion -import salt.utils import salt.utils.event -from salt.utils.network import host_to_ips as _host_to_ips -from salt.utils.network import remote_port_tcp as _remote_port_tcp +import salt.utils.files +import salt.utils.network +import salt.utils.path +import salt.utils.platform from salt.ext.six.moves import zip from salt.exceptions import CommandExecutionError @@ -48,7 +49,7 @@ def __virtual__(): ''' Not all functions supported by Windows ''' - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): return False, 'Windows platform is not supported by this module' return __virtualname__ @@ -211,25 +212,25 @@ def uptime(): curr_seconds = time.time() # Get uptime in seconds - if salt.utils.is_linux(): + if salt.utils.platform.is_linux(): ut_path = "/proc/uptime" if not os.path.exists(ut_path): raise CommandExecutionError("File {ut_path} was not found.".format(ut_path=ut_path)) - with salt.utils.fopen(ut_path) as rfh: + with salt.utils.files.fopen(ut_path) as rfh: seconds = int(float(rfh.read().split()[0])) - elif salt.utils.is_sunos(): - # note: some flavors/vesions report the host uptime inside a zone + elif salt.utils.platform.is_sunos(): + # note: some flavors/versions report the host uptime inside a zone # https://support.oracle.com/epmos/faces/BugDisplay?id=15611584 res = __salt__['cmd.run_all']('kstat -p unix:0:system_misc:boot_time') if res['retcode'] > 0: raise CommandExecutionError('The boot_time kstat was not found.') seconds = int(curr_seconds - int(res['stdout'].split()[-1])) - elif salt.utils.is_openbsd() or salt.utils.is_netbsd(): + elif salt.utils.platform.is_openbsd() or salt.utils.platform.is_netbsd(): bt_data = __salt__['sysctl.get']('kern.boottime') if not bt_data: raise CommandExecutionError('Cannot find kern.boottime system parameter') seconds = int(curr_seconds - int(bt_data)) - elif salt.utils.is_freebsd() or salt.utils.is_darwin(): + elif salt.utils.platform.is_freebsd() or salt.utils.platform.is_darwin(): # format: { sec = 1477761334, usec = 664698 } Sat Oct 29 17:15:34 2016 bt_data = __salt__['sysctl.get']('kern.boottime') if not bt_data: @@ -237,7 +238,7 @@ def uptime(): data = bt_data.split("{")[-1].split("}")[0].strip().replace(' ', '') uptime = dict([(k, int(v,)) for k, v in [p.strip().split('=') for p in data.split(',')]]) seconds = int(curr_seconds - uptime['sec']) - elif salt.utils.is_aix(): + elif salt.utils.platform.is_aix(): seconds = _get_boot_time_aix() else: return __salt__['cmd.run']('uptime') @@ -256,8 +257,8 @@ def uptime(): 'time': '{0}:{1}'.format(up_time.seconds // 3600, up_time.seconds % 3600 // 60), } - if salt.utils.which('who'): - who_cmd = 'who' if salt.utils.is_openbsd() else 'who -s' # OpenBSD does not support -s + if salt.utils.path.which('who'): + who_cmd = 'who' if salt.utils.platform.is_openbsd() else 'who -s' # OpenBSD does not support -s ut_ret['users'] = len(__salt__['cmd.run'](who_cmd).split(os.linesep)) return ut_ret @@ -310,7 +311,7 @@ def cpustats(): ''' ret = {} try: - with salt.utils.fopen('/proc/stat', 'r') as fp_: + with salt.utils.files.fopen('/proc/stat', 'r') as fp_: stats = fp_.read() except IOError: pass @@ -437,7 +438,7 @@ def meminfo(): ''' ret = {} try: - with salt.utils.fopen('/proc/meminfo', 'r') as fp_: + with salt.utils.files.fopen('/proc/meminfo', 'r') as fp_: stats = fp_.read() except IOError: pass @@ -583,7 +584,7 @@ def cpuinfo(): ''' ret = {} try: - with salt.utils.fopen('/proc/cpuinfo', 'r') as fp_: + with salt.utils.files.fopen('/proc/cpuinfo', 'r') as fp_: stats = fp_.read() except IOError: pass @@ -782,7 +783,7 @@ def diskstats(): ''' ret = {} try: - with salt.utils.fopen('/proc/diskstats', 'r') as fp_: + with salt.utils.files.fopen('/proc/diskstats', 'r') as fp_: stats = fp_.read() except IOError: pass @@ -932,7 +933,7 @@ def diskusage(*args): # ifile source of data varies with OS, otherwise all the same if __grains__['kernel'] == 'Linux': try: - with salt.utils.fopen('/proc/mounts', 'r') as fp_: + with salt.utils.files.fopen('/proc/mounts', 'r') as fp_: ifile = fp_.read().splitlines() except OSError: return {} @@ -987,7 +988,7 @@ def vmstats(): ''' ret = {} try: - with salt.utils.fopen('/proc/vmstat', 'r') as fp_: + with salt.utils.files.fopen('/proc/vmstat', 'r') as fp_: stats = fp_.read() except IOError: pass @@ -1065,7 +1066,7 @@ def netstats(): ''' ret = {} try: - with salt.utils.fopen('/proc/net/netstat', 'r') as fp_: + with salt.utils.files.fopen('/proc/net/netstat', 'r') as fp_: stats = fp_.read() except IOError: pass @@ -1187,7 +1188,7 @@ def netdev(): ''' ret = {} try: - with salt.utils.fopen('/proc/net/dev', 'r') as fp_: + with salt.utils.files.fopen('/proc/net/dev', 'r') as fp_: stats = fp_.read() except IOError: pass @@ -1448,7 +1449,7 @@ def version(): linux specific implementation of version ''' try: - with salt.utils.fopen('/proc/version', 'r') as fp_: + with salt.utils.files.fopen('/proc/version', 'r') as fp_: return fp_.read().strip() except IOError: return {} @@ -1485,14 +1486,14 @@ def master(master=None, connected=True): master_ips = None if master: - master_ips = _host_to_ips(master) + master_ips = salt.utils.network.host_to_ips(master) if not master_ips: return master_connection_status = False port = __salt__['config.get']('publish_port', default=4505) - connected_ips = _remote_port_tcp(port) + connected_ips = salt.utils.network.remote_port_tcp(port) # Get connection status for master for master_ip in master_ips: diff --git a/salt/modules/stormpath.py b/salt/modules/stormpath.py deleted file mode 100644 index 020d70b44ef..00000000000 --- a/salt/modules/stormpath.py +++ /dev/null @@ -1,242 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Support for Stormpath - -.. versionadded:: 2015.8.0 -''' - -# Import python libs -from __future__ import absolute_import, print_function -import json -import logging - -# Import salt libs -import salt.utils.http - -log = logging.getLogger(__name__) - - -def __virtual__(): - ''' - Only load the module if apache is installed - ''' - if not __opts__.get('stormpath', {}).get('apiid', None): - return (False, 'The stormpath execution module failed to load: requires the stormpath:apiid config option to be set.') - if not __opts__.get('stormpath', {}).get('apikey', None): - return (False, 'The stormpath execution module failed to load: requires the stormpath:apikey config option to be set.') - return True - - -def create_account(directory_id, email, password, givenName, surname, **kwargs): - ''' - Create an account - - CLI Examples: - - salt myminion stormpath.create_account shemp@example.com letmein Shemp Howard - - ''' - items = { - 'email': email, - 'password': password, - 'givenName': givenName, - 'surname': surname, - } - items.update(**kwargs) - - status, result = _query( - action='directories', - command='{0}/accounts'.format(directory_id), - data=json.dumps(items), - header_dict={'Content-Type': 'application/json;charset=UTF-8'}, - method='POST', - ) - - comps = result['href'].split('/') - return show_account(comps[-1]) - - -def list_accounts(): - ''' - Show all accounts. - - CLI Example: - - salt myminion stormpath.list_accounts - - ''' - status, result = _query(action='accounts', command='current') - return result - - -def show_account(account_id=None, - email=None, - directory_id=None, - application_id=None, - group_id=None, - **kwargs): - ''' - Show a specific account. - - CLI Example: - - salt myminion stormpath.show_account - - ''' - if account_id: - status, result = _query( - action='accounts', - command=account_id, - ) - return result - - if email: - if not directory_id and not application_id and not group_id: - return {'Error': 'Either a directory_id, application_id, or ' - 'group_id must be specified with an email address'} - if directory_id: - status, result = _query( - action='directories', - command='{0}/accounts'.format(directory_id), - args={'email': email} - ) - elif application_id: - status, result = _query( - action='applications', - command='{0}/accounts'.format(application_id), - args={'email': email} - ) - elif group_id: - status, result = _query( - action='groups', - command='{0}/accounts'.format(group_id), - args={'email': email} - ) - return result - - -def update_account(account_id, key=None, value=None, items=None): - ''' - Update one or more items for this account. Specifying an empty value will - clear it for that account. - - CLI Examples: - - salt myminion stormpath.update_account givenName shemp - salt myminion stormpath.update_account middleName '' - salt myminion stormpath.update_account items='{"givenName": "Shemp"} - salt myminion stormpath.update_account items='{"middlename": ""} - - ''' - if items is None: - if key is None or value is None: - return {'Error': 'At least one key/value pair is required'} - items = {key: value} - - status, result = _query( - action='accounts', - command=account_id, - data=json.dumps(items), - header_dict={'Content-Type': 'application/json;charset=UTF-8'}, - method='POST', - ) - - return show_account(account_id) - - -def delete_account(account_id): - ''' - Delete an account. - - CLI Examples: - - salt myminion stormpath.delete_account - ''' - _query( - action='accounts', - command=account_id, - method='DELETE', - ) - - return True - - -def list_directories(): - ''' - Show all directories. - - CLI Example: - - salt myminion stormpath.list_directories - - ''' - tenant = show_tenant() - tenant_id = tenant.get('href', '').split('/')[-1] - status, result = _query(action='tenants', command='{0}/directories'.format(tenant_id)) - return result - - -def show_tenant(): - ''' - Get the tenant for the login being used. - ''' - status, result = _query(action='tenants', command='current') - return result - - -def _query(action=None, - command=None, - args=None, - method='GET', - header_dict=None, - data=None): - ''' - Make a web call to Stormpath. - ''' - apiid = __opts__.get('stormpath', {}).get('apiid', None) - apikey = __opts__.get('stormpath', {}).get('apikey', None) - path = 'https://api.stormpath.com/v1/' - - if action: - path += action - - if command: - path += '/{0}'.format(command) - - log.debug('Stormpath URL: {0}'.format(path)) - - if not isinstance(args, dict): - args = {} - - if header_dict is None: - header_dict = {} - - if method != 'POST': - header_dict['Accept'] = 'application/json' - - decode = True - if method == 'DELETE': - decode = False - - return_content = None - result = salt.utils.http.query( - path, - method, - username=apiid, - password=apikey, - params=args, - data=data, - header_dict=header_dict, - decode=decode, - decode_type='json', - text=True, - status=True, - opts=__opts__, - ) - log.debug( - 'Stormpath Response Status Code: {0}'.format( - result['status'] - ) - ) - - return [result['status'], result.get('dict', {})] diff --git a/salt/modules/supervisord.py b/salt/modules/supervisord.py index dd22987cff7..648a65e2b9d 100644 --- a/salt/modules/supervisord.py +++ b/salt/modules/supervisord.py @@ -15,6 +15,7 @@ from salt.ext.six.moves import configparser # pylint: disable=import-error # Import salt libs import salt.utils +import salt.utils.stringutils from salt.exceptions import CommandExecutionError, CommandNotFoundError @@ -408,7 +409,7 @@ def options(name, conf_file=None): raise CommandExecutionError('Process \'{0}\' not found'.format(name)) ret = {} for key, val in config.items(section_name): - val = salt.utils.str_to_num(val.split(';')[0].strip()) + val = salt.utils.stringutils.to_num(val.split(';')[0].strip()) # pylint: disable=maybe-no-member if isinstance(val, string_types): if val.lower() == 'true': diff --git a/salt/modules/suse_apache.py b/salt/modules/suse_apache.py index 00c2930ad3f..ab4245abfd2 100644 --- a/salt/modules/suse_apache.py +++ b/salt/modules/suse_apache.py @@ -12,7 +12,7 @@ from __future__ import absolute_import import logging # Import salt libs -import salt.utils +import salt.utils.path log = logging.getLogger(__name__) @@ -23,7 +23,7 @@ def __virtual__(): ''' Only load the module if apache is installed. ''' - if salt.utils.which('apache2ctl') and __grains__['os_family'] == 'SUSE': + if salt.utils.path.which('apache2ctl') and __grains__['os_family'] == 'SUSE': return __virtualname__ return (False, 'apache execution module not loaded: apache not installed.') diff --git a/salt/modules/svn.py b/salt/modules/svn.py index 41262144511..cdc76320ea1 100644 --- a/salt/modules/svn.py +++ b/salt/modules/svn.py @@ -8,8 +8,9 @@ from __future__ import absolute_import import re # Import salt libs -import salt.utils -from salt import utils, exceptions +import salt.utils.args +import salt.utils.path +from salt.exceptions import CommandExecutionError _INI_RE = re.compile(r"^([^:]+):\s+(\S.*)$", re.M) @@ -18,7 +19,7 @@ def __virtual__(): ''' Only load if svn is installed ''' - if utils.which('svn') is None: + if salt.utils.path.which('svn') is None: return (False, 'The svn execution module cannot be loaded: svn unavailable.') else: @@ -68,7 +69,7 @@ def _run_svn(cmd, cwd, user, username, password, opts, **kwargs): if retcode == 0: return result['stdout'] - raise exceptions.CommandExecutionError(result['stderr'] + '\n\n' + ' '.join(cmd)) + raise CommandExecutionError(result['stderr'] + '\n\n' + ' '.join(cmd)) def info(cwd, @@ -112,7 +113,7 @@ def info(cwd, if fmt == 'xml': opts.append('--xml') if targets: - opts += salt.utils.shlex_split(targets) + opts += salt.utils.args.shlex_split(targets) infos = _run_svn('info', cwd, user, username, password, opts) if fmt in ('str', 'xml'): @@ -241,7 +242,7 @@ def update(cwd, targets=None, user=None, username=None, password=None, *opts): salt '*' svn.update /path/to/repo ''' if targets: - opts += tuple(salt.utils.shlex_split(targets)) + opts += tuple(salt.utils.args.shlex_split(targets)) return _run_svn('update', cwd, user, username, password, opts) @@ -275,7 +276,7 @@ def diff(cwd, targets=None, user=None, username=None, password=None, *opts): salt '*' svn.diff /path/to/repo ''' if targets: - opts += tuple(salt.utils.shlex_split(targets)) + opts += tuple(salt.utils.args.shlex_split(targets)) return _run_svn('diff', cwd, user, username, password, opts) @@ -320,7 +321,7 @@ def commit(cwd, if msg: opts += ('-m', msg) if targets: - opts += tuple(salt.utils.shlex_split(targets)) + opts += tuple(salt.utils.args.shlex_split(targets)) return _run_svn('commit', cwd, user, username, password, opts) @@ -352,7 +353,7 @@ def add(cwd, targets, user=None, username=None, password=None, *opts): salt '*' svn.add /path/to/repo /path/to/new/file ''' if targets: - opts += tuple(salt.utils.shlex_split(targets)) + opts += tuple(salt.utils.args.shlex_split(targets)) return _run_svn('add', cwd, user, username, password, opts) @@ -395,7 +396,7 @@ def remove(cwd, if msg: opts += ('-m', msg) if targets: - opts += tuple(salt.utils.shlex_split(targets)) + opts += tuple(salt.utils.args.shlex_split(targets)) return _run_svn('remove', cwd, user, username, password, opts) @@ -429,7 +430,7 @@ def status(cwd, targets=None, user=None, username=None, password=None, *opts): salt '*' svn.status /path/to/repo ''' if targets: - opts += tuple(salt.utils.shlex_split(targets)) + opts += tuple(salt.utils.args.shlex_split(targets)) return _run_svn('status', cwd, user, username, password, opts) diff --git a/salt/modules/sysbench.py b/salt/modules/sysbench.py index 439902d1827..1ee86e02ca4 100644 --- a/salt/modules/sysbench.py +++ b/salt/modules/sysbench.py @@ -8,7 +8,7 @@ CPU, Memory, File I/O, Threads and Mutex. from __future__ import absolute_import import re -import salt.utils +import salt.utils.path from salt.ext.six.moves import zip @@ -17,7 +17,7 @@ def __virtual__(): loads the module, if only sysbench is installed ''' # finding the path of the binary - if salt.utils.which('sysbench'): + if salt.utils.path.which('sysbench'): return 'sysbench' return (False, 'The sysbench execution module failed to load: the sysbench binary is not in the path.') diff --git a/salt/modules/sysfs.py b/salt/modules/sysfs.py index 3afa54b66f9..69f5325c87b 100644 --- a/salt/modules/sysfs.py +++ b/salt/modules/sysfs.py @@ -5,17 +5,18 @@ Module for interfacing with SysFS .. seealso:: https://www.kernel.org/doc/Documentation/filesystems/sysfs.txt .. versionadded:: 2016.3.0 ''' -# Import python libs +# Import Python libs from __future__ import absolute_import import logging import os import stat -# Import external libs -import salt.ext.six as six +# Import Salt libs +import salt.utils.files +import salt.utils.platform -# Import salt libs -import salt.utils +# Import 3rd-party libs +from salt.ext import six log = logging.getLogger(__name__) @@ -24,7 +25,7 @@ def __virtual__(): ''' Only work on Linux ''' - return salt.utils.is_linux() + return salt.utils.platform.is_linux() def attr(key, value=None): @@ -63,7 +64,7 @@ def write(key, value): try: key = target(key) log.trace('Writing {0} to {1}'.format(value, key)) - with salt.utils.fopen(key, 'w') as twriter: + with salt.utils.files.fopen(key, 'w') as twriter: twriter.write('{0}\n'.format(value)) return True except: # pylint: disable=bare-except diff --git a/salt/modules/syslog_ng.py b/salt/modules/syslog_ng.py index 4967b65c0a8..8b03b2229fb 100644 --- a/salt/modules/syslog_ng.py +++ b/salt/modules/syslog_ng.py @@ -34,11 +34,12 @@ import os import os.path # Import Salt libs -import salt.utils +import salt.utils.files +import salt.utils.path from salt.exceptions import CommandExecutionError # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import range # pylint: disable=import-error,no-name-in-module,redefined-builtin @@ -425,7 +426,7 @@ def _is_simple_type(value): Returns True, if the given parameter value is an instance of either int, str, float or bool. ''' - return isinstance(value, str) or isinstance(value, int) or isinstance(value, float) or isinstance(value, bool) + return isinstance(value, six.string_types) or isinstance(value, int) or isinstance(value, float) or isinstance(value, bool) def _get_type_id_options(name, configuration): @@ -814,7 +815,7 @@ def _run_command_in_extended_path(syslog_ng_sbin_dir, command, params): ''' orig_path = _add_to_path_envvar(syslog_ng_sbin_dir) - if not salt.utils.which(command): + if not salt.utils.path.which(command): error_message = ( 'Unable to execute the command \'{0}\'. It is not in the PATH.' .format(command) @@ -1174,7 +1175,7 @@ def _write_config(config, newlines=2): text = config[key] try: - with salt.utils.fopen(__SYSLOG_NG_CONFIG_FILE, 'a') as fha: + with salt.utils.files.fopen(__SYSLOG_NG_CONFIG_FILE, 'a') as fha: fha.write(text) for _ in range(0, newlines): diff --git a/salt/modules/sysmod.py b/salt/modules/sysmod.py index a7c731e93eb..31de9fa67b6 100644 --- a/salt/modules/sysmod.py +++ b/salt/modules/sysmod.py @@ -12,13 +12,13 @@ import logging import salt.loader import salt.runner import salt.state -import salt.utils -import salt.utils.schema as S +import salt.utils.args +import salt.utils.schema from salt.utils.doc import strip_rst as _strip_rst from salt.ext.six.moves import zip # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six log = logging.getLogger(__name__) @@ -450,7 +450,7 @@ def argspec(module=''): salt '*' sys.argspec 'pkg.*' ''' - return salt.utils.argspec_report(__salt__, module) + return salt.utils.args.argspec_report(__salt__, module) def state_argspec(module=''): @@ -476,7 +476,7 @@ def state_argspec(module=''): ''' st_ = salt.state.State(__opts__) - return salt.utils.argspec_report(st_.states, module) + return salt.utils.args.argspec_report(st_.states, module) def returner_argspec(module=''): @@ -502,7 +502,7 @@ def returner_argspec(module=''): ''' returners_ = salt.loader.returners(__opts__, []) - return salt.utils.argspec_report(returners_, module) + return salt.utils.args.argspec_report(returners_, module) def runner_argspec(module=''): @@ -527,7 +527,7 @@ def runner_argspec(module=''): salt '*' sys.runner_argspec 'winrepo.*' ''' run_ = salt.runner.Runner(__opts__) - return salt.utils.argspec_report(run_.functions, module) + return salt.utils.args.argspec_report(run_.functions, module) def list_state_functions(*args, **kwargs): # pylint: disable=unused-argument @@ -844,28 +844,28 @@ def _argspec_to_schema(mod, spec): } for i in args_req: - types[i] = S.OneOfItem(items=( - S.BooleanItem(title=i, description=i, required=True), - S.IntegerItem(title=i, description=i, required=True), - S.NumberItem(title=i, description=i, required=True), - S.StringItem(title=i, description=i, required=True), + types[i] = salt.utils.schema.OneOfItem(items=( + salt.utils.schema.BooleanItem(title=i, description=i, required=True), + salt.utils.schema.IntegerItem(title=i, description=i, required=True), + salt.utils.schema.NumberItem(title=i, description=i, required=True), + salt.utils.schema.StringItem(title=i, description=i, required=True), # S.ArrayItem(title=i, description=i, required=True), # S.DictItem(title=i, description=i, required=True), )) for i, j in args_defaults: - types[i] = S.OneOfItem(items=( - S.BooleanItem(title=i, description=i, default=j), - S.IntegerItem(title=i, description=i, default=j), - S.NumberItem(title=i, description=i, default=j), - S.StringItem(title=i, description=i, default=j), + types[i] = salt.utils.schema.OneOfItem(items=( + salt.utils.schema.BooleanItem(title=i, description=i, default=j), + salt.utils.schema.IntegerItem(title=i, description=i, default=j), + salt.utils.schema.NumberItem(title=i, description=i, default=j), + salt.utils.schema.StringItem(title=i, description=i, default=j), # S.ArrayItem(title=i, description=i, default=j), # S.DictItem(title=i, description=i, default=j), )) - return type(mod, (S.Schema,), types).serialize() + return type(mod, (salt.utils.schema.Schema,), types).serialize() def state_schema(module=''): diff --git a/salt/modules/sysrc.py b/salt/modules/sysrc.py index c2c989b3663..be0cec44169 100644 --- a/salt/modules/sysrc.py +++ b/salt/modules/sysrc.py @@ -7,7 +7,7 @@ sysrc module for FreeBSD from __future__ import absolute_import # Import Salt libs -import salt.utils +import salt.utils.path from salt.exceptions import CommandExecutionError @@ -22,7 +22,7 @@ def __virtual__(): ''' Only runs if sysrc exists ''' - if salt.utils.which('sysrc') is not None: + if salt.utils.path.which('sysrc') is not None: return True return (False, 'The sysrc execution module failed to load: the sysrc binary is not in the path.') @@ -115,8 +115,8 @@ def set_(name, value, **kwargs): for sysrc in sysrcs.split("\n"): rcfile = sysrc.split(': ')[0] var = sysrc.split(': ')[1] - oldval = sysrc.split(': ')[2].split(" -> ")[0] - newval = sysrc.split(': ')[2].split(" -> ")[1] + oldval = sysrc.split(': ')[2].strip().split("->")[0] + newval = sysrc.split(': ')[2].strip().split("->")[1] if rcfile not in ret: ret[rcfile] = {} ret[rcfile][var] = newval diff --git a/salt/modules/system.py b/salt/modules/system.py index c397ad2f272..87673a372e6 100644 --- a/salt/modules/system.py +++ b/salt/modules/system.py @@ -1,19 +1,32 @@ # -*- coding: utf-8 -*- ''' -Support for reboot, shutdown, etc +Support for reboot, shutdown, etc on POSIX-like systems. + +.. note:: + + If you have configured a wrapper such as ``molly-guard`` to + intercept *interactive* shutdown commands, be aware that calling + ``system.halt``, ``system.poweroff``, ``system.reboot``, and + ``system.shutdown`` with ``salt-call`` will hang indefinitely + while the wrapper script waits for user input. Calling them with + ``salt`` will work as expected. + ''' from __future__ import absolute_import -# Import python libs +# Import Python libs from datetime import datetime, timedelta, tzinfo import re import os.path -# Import salt libs -import salt.utils -import salt.ext.six as six +# Import Salt libs +import salt.utils.files +import salt.utils.path +import salt.utils.platform from salt.exceptions import CommandExecutionError, SaltInvocationError +# Import 3rd-party libs +from salt.ext import six __virtualname__ = 'system' @@ -23,13 +36,13 @@ def __virtual__(): Only supported on POSIX-like systems Windows, Solaris, and Mac have their own modules ''' - if salt.utils.is_windows(): - return (False, 'This module is not available on windows') + if salt.utils.platform.is_windows(): + return (False, 'This module is not available on Windows') - if salt.utils.is_darwin(): + if salt.utils.platform.is_darwin(): return (False, 'This module is not available on Mac OS') - if salt.utils.is_sunos(): + if salt.utils.platform.is_sunos(): return (False, 'This module is not available on SunOS') return __virtualname__ @@ -167,8 +180,11 @@ def has_settable_hwclock(): salt '*' system.has_settable_hwclock ''' - if salt.utils.which_bin(['hwclock']) is not None: - res = __salt__['cmd.run_all'](['hwclock', '--test', '--systohc'], python_shell=False) + if salt.utils.path.which_bin(['hwclock']) is not None: + res = __salt__['cmd.run_all']( + ['hwclock', '--test', '--systohc'], python_shell=False, + output_loglevel='quiet', ignore_retcode=True + ) return res['retcode'] == 0 return False @@ -415,7 +431,7 @@ def get_system_date(utc_offset=None): def set_system_date(newdate, utc_offset=None): ''' - Set the Windows system date. Use format for the date. + Set the system date. Use format for the date. :param str newdate: The date to set. Can be any of the following formats @@ -494,7 +510,7 @@ def get_computer_desc(): salt '*' system.get_computer_desc ''' desc = None - hostname_cmd = salt.utils.which('hostnamectl') + hostname_cmd = salt.utils.path.which('hostnamectl') if hostname_cmd: desc = __salt__['cmd.run']( [hostname_cmd, 'status', '--pretty'], @@ -503,7 +519,7 @@ def get_computer_desc(): else: pattern = re.compile(r'^\s*PRETTY_HOSTNAME=(.*)$') try: - with salt.utils.fopen('/etc/machine-info', 'r') as mach_info: + with salt.utils.files.fopen('/etc/machine-info', 'r') as mach_info: for line in mach_info.readlines(): match = pattern.match(line) if match: @@ -538,7 +554,7 @@ def set_computer_desc(desc): desc = desc.replace('"', '\\"') else: desc = desc.encode('string_escape').replace('"', '\\"') - hostname_cmd = salt.utils.which('hostnamectl') + hostname_cmd = salt.utils.path.which('hostnamectl') if hostname_cmd: result = __salt__['cmd.retcode']( [hostname_cmd, 'set-hostname', '--pretty', desc], @@ -547,14 +563,14 @@ def set_computer_desc(desc): return True if result == 0 else False if not os.path.isfile('/etc/machine-info'): - with salt.utils.fopen('/etc/machine-info', 'w'): + with salt.utils.files.fopen('/etc/machine-info', 'w'): pass is_pretty_hostname_found = False pattern = re.compile(r'^\s*PRETTY_HOSTNAME=(.*)$') new_line = 'PRETTY_HOSTNAME="{0}"'.format(desc) try: - with salt.utils.fopen('/etc/machine-info', 'r+') as mach_info: + with salt.utils.files.fopen('/etc/machine-info', 'r+') as mach_info: lines = mach_info.readlines() for i, line in enumerate(lines): if pattern.match(line): diff --git a/salt/modules/system_profiler.py b/salt/modules/system_profiler.py index 31e0b8efe36..230a5f9548d 100644 --- a/salt/modules/system_profiler.py +++ b/salt/modules/system_profiler.py @@ -13,7 +13,7 @@ from __future__ import absolute_import import plistlib import subprocess -import salt.utils +import salt.utils.path from salt.ext import six PROFILER_BINARY = '/usr/sbin/system_profiler' @@ -23,7 +23,7 @@ def __virtual__(): ''' Check to see if the system_profiler binary is available ''' - PROFILER_BINARY = salt.utils.which('system_profiler') + PROFILER_BINARY = salt.utils.path.which('system_profiler') if PROFILER_BINARY: return True diff --git a/salt/modules/systemd.py b/salt/modules/systemd.py index 2370563b5b6..f2e6a6f1a33 100644 --- a/salt/modules/systemd.py +++ b/salt/modules/systemd.py @@ -10,7 +10,7 @@ Provides the service module for systemd *'service.start' is not available*), see :ref:`here `. ''' -# Import python libs +# Import Python libs from __future__ import absolute_import import errno import glob @@ -20,10 +20,14 @@ import fnmatch import re import shlex -# Import 3rd-party libs +# Import Salt libs +import salt.utils.files import salt.utils.itertools +import salt.utils.path import salt.utils.systemd from salt.exceptions import CommandExecutionError + +# Import 3rd-party libs from salt.ext import six log = logging.getLogger(__name__) @@ -152,7 +156,7 @@ def _default_runlevel(): # Try to get the "main" default. If this fails, throw up our # hands and just guess "2", because things are horribly broken try: - with salt.utils.fopen('/etc/init/rc-sysinit.conf') as fp_: + with salt.utils.files.fopen('/etc/init/rc-sysinit.conf') as fp_: for line in fp_: if line.startswith('env DEFAULT_RUNLEVEL'): runlevel = line.split('=')[-1].strip() @@ -161,7 +165,7 @@ def _default_runlevel(): # Look for an optional "legacy" override in /etc/inittab try: - with salt.utils.fopen('/etc/inittab') as fp_: + with salt.utils.files.fopen('/etc/inittab') as fp_: for line in fp_: if not line.startswith('#') and 'initdefault' in line: runlevel = line.split(':')[1] @@ -173,7 +177,7 @@ def _default_runlevel(): try: valid_strings = set( ('0', '1', '2', '3', '4', '5', '6', 's', 'S', '-s', 'single')) - with salt.utils.fopen('/proc/cmdline') as fp_: + with salt.utils.files.fopen('/proc/cmdline') as fp_: for line in fp_: for arg in line.strip().split(): if arg in valid_strings: @@ -253,7 +257,7 @@ def _get_service_exec(): if contextkey not in __context__: executables = ('update-rc.d', 'chkconfig') for executable in executables: - service_exec = salt.utils.which(executable) + service_exec = salt.utils.path.which(executable) if service_exec is not None: break else: diff --git a/salt/modules/test.py b/salt/modules/test.py index b1f13d61600..86d434298d7 100644 --- a/salt/modules/test.py +++ b/salt/modules/test.py @@ -15,9 +15,12 @@ import random # Import Salt libs import salt import salt.utils +import salt.utils.args +import salt.utils.hashutils +import salt.utils.platform import salt.version import salt.loader -import salt.ext.six as six +from salt.ext import six from salt.utils.decorators import depends __proxyenabled__ = ['*'] @@ -25,7 +28,8 @@ __proxyenabled__ = ['*'] # Don't shadow built-in's. __func_alias__ = { 'true_': 'true', - 'false_': 'false' + 'false_': 'false', + 'try_': 'try', } log = logging.getLogger(__name__) @@ -114,7 +118,7 @@ def ping(): salt '*' test.ping ''' - if not salt.utils.is_proxy(): + if not salt.utils.platform.is_proxy(): log.debug('test.ping received for minion \'%s\'', __opts__.get('id')) return True else: @@ -311,6 +315,18 @@ def arg_repr(*args, **kwargs): return {"args": repr(args), "kwargs": repr(kwargs)} +def arg_clean(*args, **kwargs): + ''' + Like test.arg but cleans kwargs of the __pub* items + CLI Example: + + .. code-block:: bash + + salt '*' test.arg_clean 1 "two" 3.1 txt="hello" wow='{a: 1, b: "hello"}' + ''' + return dict(args=args, kwargs=salt.utils.args.clean_kwargs(**kwargs)) + + def fib(num): ''' Return the num-th Fibonacci number, and the time it took to compute in @@ -479,25 +495,34 @@ def opts_pkg(): def rand_str(size=9999999999, hash_type=None): + salt.utils.warn_until( + 'Neon', + 'test.rand_str has been renamed to test.random_hash' + ) + return random_hash(size=size, hash_type=hash_type) + + +def random_hash(size=9999999999, hash_type=None): ''' - Return a random string + .. versionadded:: 2015.5.2 + .. versionchanged:: Oxygen + Function has been renamed from ``test.rand_str`` to + ``test.random_hash`` - size - size of the string to generate - hash_type - hash type to use - - .. versionadded:: 2015.5.2 + Generates a random number between 1 and ``size``, then returns a hash of + that number. If no ``hash_type`` is passed, the hash_type specified by the + minion's :conf_minion:`hash_type` config option is used. CLI Example: .. code-block:: bash - salt '*' test.rand_str + salt '*' test.random_hash + salt '*' test.random_hash hash_type=sha512 ''' if not hash_type: hash_type = __opts__.get('hash_type', 'md5') - return salt.utils.rand_str(hash_type=hash_type, size=size) + return salt.utils.hashutils.random_hash(size=size, hash_type=hash_type) def exception(message='Test Exception'): diff --git a/salt/modules/testinframod.py b/salt/modules/testinframod.py index c4ad1df0ef2..77b1d5f3bcb 100644 --- a/salt/modules/testinframod.py +++ b/salt/modules/testinframod.py @@ -242,6 +242,8 @@ def _copy_function(module_name, name=None): elif hasattr(mod, '__call__'): mod_sig = inspect.getargspec(mod.__call__) parameters = mod_sig.args + log.debug('Parameters accepted by module {0}: {1}'.format(module_name, + parameters)) additional_args = {} for arg in set(parameters).intersection(set(methods)): additional_args[arg] = methods.pop(arg) @@ -251,12 +253,15 @@ def _copy_function(module_name, name=None): else: modinstance = mod() except TypeError: - modinstance = None - methods = {} + log.exception('Module failed to instantiate') + raise + valid_methods = {} + log.debug('Called methods are: {0}'.format(methods)) for meth_name in methods: if not meth_name.startswith('_'): - methods[meth_name] = methods[meth_name] - for meth, arg in methods.items(): + valid_methods[meth_name] = methods[meth_name] + log.debug('Valid methods are: {0}'.format(valid_methods)) + for meth, arg in valid_methods.items(): result = _get_method_result(mod, modinstance, meth, arg) assertion_result = _apply_assertion(arg, result) if not assertion_result: @@ -285,13 +290,16 @@ def _register_functions(): functions, and then register them in the module namespace so that they can be called via salt. """ - for module_ in modules.__all__: - mod_name = _to_snake_case(module_) + try: + modules_ = [_to_snake_case(module_) for module_ in modules.__all__] + except AttributeError: + modules_ = [module_ for module_ in modules.modules] + + for mod_name in modules_: mod_func = _copy_function(mod_name, str(mod_name)) - mod_func.__doc__ = _build_doc(module_) + mod_func.__doc__ = _build_doc(mod_name) __all__.append(mod_name) globals()[mod_name] = mod_func - if TESTINFRA_PRESENT: _register_functions() diff --git a/salt/modules/textfsm_mod.py b/salt/modules/textfsm_mod.py new file mode 100644 index 00000000000..760f27d3a01 --- /dev/null +++ b/salt/modules/textfsm_mod.py @@ -0,0 +1,459 @@ +# -*- coding: utf-8 -*- +''' +TextFSM +======= + +.. versionadded:: Oxygen + +Execution module that processes plain text and extracts data +using TextFSM templates. The output is presented in JSON serializable +data, and can be easily re-used in other modules, or directly +inside the renderer (Jinja, Mako, Genshi, etc.). + +:depends: - textfsm Python library + +.. note:: + + For Python 2/3 compatibility, it is more recommended to + install the ``jtextfsm`` library: ``pip install jtextfsm``. +''' +from __future__ import absolute_import + +# Import python libs +import os +import logging + +# Import third party modules +try: + import textfsm + HAS_TEXTFSM = True +except ImportError: + HAS_TEXTFSM = False + +try: + import clitable + HAS_CLITABLE = True +except ImportError: + HAS_CLITABLE = False + +try: + from salt.utils.files import fopen +except ImportError: + from salt.utils import fopen + +log = logging.getLogger(__name__) + +__virtualname__ = 'textfsm' +__proxyenabled__ = ['*'] + + +def __virtual__(): + ''' + Only load this execution module if TextFSM is installed. + ''' + if HAS_TEXTFSM: + return __virtualname__ + return (False, 'The textfsm execution module failed to load: requires the textfsm library.') + + +def _clitable_to_dict(objects, fsm_handler): + ''' + Converts TextFSM cli_table object to list of dictionaries. + ''' + objs = [] + log.debug('Cli Table:') + log.debug(objects) + log.debug('FSM handler:') + log.debug(fsm_handler) + for row in objects: + temp_dict = {} + for index, element in enumerate(row): + temp_dict[fsm_handler.header[index].lower()] = element + objs.append(temp_dict) + log.debug('Extraction result:') + log.debug(objs) + return objs + + +def extract(template_path, raw_text=None, raw_text_file=None, saltenv='base'): + r''' + Extracts the data entities from the unstructured + raw text sent as input and returns the data + mapping, processing using the TextFSM template. + + template_path + The path to the TextFSM template. + This can be specified using the absolute path + to the file, or using one of the following URL schemes: + + - ``salt://``, to fetch the template from the Salt fileserver. + - ``http://`` or ``https://`` + - ``ftp://`` + - ``s3://`` + - ``swift://`` + + raw_text: ``None`` + The unstructured text to be parsed. + + raw_text_file: ``None`` + Text file to read, having the raw text to be parsed using the TextFSM template. + Supports the same URL schemes as the ``template_path`` argument. + + saltenv: ``base`` + Salt fileserver envrionment from which to retrieve the file. + Ignored if ``template_path`` is not a ``salt://`` URL. + + CLI Example: + + .. code-block:: bash + + salt '*' textfsm.extract salt://textfsm/juniper_version_template raw_text_file=s3://junos_ver.txt + salt '*' textfsm.extract http://some-server/textfsm/juniper_version_template raw_text='Hostname: router.abc ... snip ...' + + Jinja template example: + + .. code-block:: jinja + + {%- set raw_text = 'Hostname: router.abc ... snip ...' -%} + {%- set textfsm_extract = salt.textfsm.extract('https://some-server/textfsm/juniper_version_template', raw_text) -%} + + Raw text example: + + .. code-block:: text + + Hostname: router.abc + Model: mx960 + JUNOS Base OS boot [9.1S3.5] + JUNOS Base OS Software Suite [9.1S3.5] + JUNOS Kernel Software Suite [9.1S3.5] + JUNOS Crypto Software Suite [9.1S3.5] + JUNOS Packet Forwarding Engine Support (M/T Common) [9.1S3.5] + JUNOS Packet Forwarding Engine Support (MX Common) [9.1S3.5] + JUNOS Online Documentation [9.1S3.5] + JUNOS Routing Software Suite [9.1S3.5] + + TextFSM Example: + + .. code-block:: text + + Value Chassis (\S+) + Value Required Model (\S+) + Value Boot (.*) + Value Base (.*) + Value Kernel (.*) + Value Crypto (.*) + Value Documentation (.*) + Value Routing (.*) + + Start + # Support multiple chassis systems. + ^\S+:$$ -> Continue.Record + ^${Chassis}:$$ + ^Model: ${Model} + ^JUNOS Base OS boot \[${Boot}\] + ^JUNOS Software Release \[${Base}\] + ^JUNOS Base OS Software Suite \[${Base}\] + ^JUNOS Kernel Software Suite \[${Kernel}\] + ^JUNOS Crypto Software Suite \[${Crypto}\] + ^JUNOS Online Documentation \[${Documentation}\] + ^JUNOS Routing Software Suite \[${Routing}\] + + Output example: + + .. code-block:: json + + { + "comment": "", + "result": true, + "out": [ + { + "kernel": "9.1S3.5", + "documentation": "9.1S3.5", + "boot": "9.1S3.5", + "crypto": "9.1S3.5", + "chassis": "", + "routing": "9.1S3.5", + "base": "9.1S3.5", + "model": "mx960" + } + ] + } + ''' + ret = { + 'result': False, + 'comment': '', + 'out': None + } + log.debug('Using the saltenv: {}'.format(saltenv)) + log.debug('Caching {} using the Salt fileserver'.format(template_path)) + tpl_cached_path = __salt__['cp.cache_file'](template_path, saltenv=saltenv) + if tpl_cached_path is False: + ret['comment'] = 'Unable to read the TextFSM template from {}'.format(template_path) + log.error(ret['comment']) + return ret + try: + log.debug('Reading TextFSM template from cache path: {}'.format(tpl_cached_path)) + # Disabling pylint W8470 to nto complain about fopen. + # Unfortunately textFSM needs the file handle rather than the content... + # pylint: disable=W8470 + tpl_file_handle = fopen(tpl_cached_path, 'r') + # pylint: disable=W8470 + log.debug(tpl_file_handle.read()) + tpl_file_handle.seek(0) # move the object position back at the top of the file + fsm_handler = textfsm.TextFSM(tpl_file_handle) + except textfsm.TextFSMTemplateError as tfte: + log.error('Unable to parse the TextFSM template', exc_info=True) + ret['comment'] = 'Unable to parse the TextFSM template from {}: {}. Please check the logs.'.format( + template_path, tfte) + return ret + if not raw_text and raw_text_file: + log.debug('Trying to read the raw input from {}'.format(raw_text_file)) + raw_text = __salt__['cp.get_file_str'](raw_text_file, saltenv=saltenv) + if raw_text is False: + ret['comment'] = 'Unable to read from {}. Please specify a valid input file or text.'.format(raw_text_file) + log.error(ret['comment']) + return ret + if not raw_text: + ret['comment'] = 'Please specify a valid input file or text.' + log.error(ret['comment']) + return ret + log.debug('Processing the raw text:') + log.debug(raw_text) + objects = fsm_handler.ParseText(raw_text) + ret['out'] = _clitable_to_dict(objects, fsm_handler) + ret['result'] = True + return ret + + +def index(command, + platform=None, + platform_grain_name=None, + platform_column_name=None, + output=None, + output_file=None, + textfsm_path=None, + index_file=None, + saltenv='base', + include_empty=False, + include_pat=None, + exclude_pat=None): + ''' + Dynamically identify the template required to extract the + information from the unstructured raw text. + + The output has the same structure as the ``extract`` execution + function, the difference being that ``index`` is capable + to identify what template to use, based on the platform + details and the ``command``. + + command + The command executed on the device, to get the output. + + platform + The platform name, as defined in the TextFSM index file. + + .. note:: + For ease of use, it is recommended to define the TextFSM + indexfile with values that can be matches using the grains. + + platform_grain_name + The name of the grain used to identify the platform name + in the TextFSM index file. + + .. note:: + This option can be also specified in the minion configuration + file or pillar as ``textfsm_platform_grain``. + + .. note:: + This option is ignored when ``platform`` is specified. + + platform_column_name: ``Platform`` + The column name used to identify the platform, + exactly as specified in the TextFSM index file. + Default: ``Platform``. + + .. note:: + This is field is case sensitive, make sure + to assign the correct value to this option, + exactly as defined in the index file. + + .. note:: + This option can be also specified in the minion configuration + file or pillar as ``textfsm_platform_column_name``. + + output + The raw output from the device, to be parsed + and extract the structured data. + + output_file + The path to a file that contains the raw output from the device, + used to extract the structured data. + This option supports the usual Salt-specific schemes: ``file://``, + ``salt://``, ``http://``, ``https://``, ``ftp://``, ``s3://``, ``swift://``. + + textfsm_path + The path where the TextFSM templates can be found. This can be either + absolute path on the server, either specified using the following URL + schemes: ``file://``, ``salt://``, ``http://``, ``https://``, ``ftp://``, + ``s3://``, ``swift://``. + + .. note:: + This needs to be a directory with a flat structure, having an + index file (whose name can be specified using the ``index_file`` option) + and a number of TextFSM templates. + + .. note:: + This option can be also specified in the minion configuration + file or pillar as ``textfsm_path``. + + index_file: ``index`` + The name of the TextFSM index file, under the ``textfsm_path``. Default: ``index``. + + .. note:: + This option can be also specified in the minion configuration + file or pillar as ``textfsm_index_file``. + + saltenv: ``base`` + Salt fileserver envrionment from which to retrieve the file. + Ignored if ``textfsm_path`` is not a ``salt://`` URL. + + include_empty: ``False`` + Include empty files under the ``textfsm_path``. + + include_pat + Glob or regex to narrow down the files cached from the given path. + If matching with a regex, the regex must be prefixed with ``E@``, + otherwise the expression will be interpreted as a glob. + + exclude_pat + Glob or regex to exclude certain files from being cached from the given path. + If matching with a regex, the regex must be prefixed with ``E@``, + otherwise the expression will be interpreted as a glob. + + .. note:: + If used with ``include_pat``, files matching this pattern will be + excluded from the subset of files defined by ``include_pat``. + + CLI Example: + + .. code-block:: bash + + salt '*' textfsm.index 'sh ver' platform=Juniper output_file=salt://textfsm/juniper_version_example textfsm_path=salt://textfsm/ + salt '*' textfsm.index 'sh ver' output_file=salt://textfsm/juniper_version_example textfsm_path=ftp://textfsm/ platform_column_name=Vendor + salt '*' textfsm.index 'sh ver' output_file=salt://textfsm/juniper_version_example textfsm_path=https://some-server/textfsm/ platform_column_name=Vendor platform_grain_name=vendor + + TextFSM index file example: + + ``salt://textfsm/index`` + + .. code-block:: text + + Template, Hostname, Vendor, Command + juniper_version_template, .*, Juniper, sh[[ow]] ve[[rsion]] + + The usage can be simplified, + by defining (some of) the following options: ``textfsm_platform_grain``, + ``textfsm_path``, ``textfsm_platform_column_name``, or ``textfsm_index_file``, + in the (proxy) minion configuration file or pillar. + + Configuration example: + + .. code-block:: yaml + + textfsm_platform_grain: vendor + textfsm_path: salt://textfsm/ + textfsm_platform_column_name: Vendor + + And the CLI usage becomes as simple as: + + .. code-block:: bash + + salt '*' textfsm.index 'sh ver' output_file=salt://textfsm/juniper_version_example + + Usgae inside a Jinja template: + + .. code-block:: jinja + + {%- set command = 'sh ver' -%} + {%- set output = salt.net.cli(command) -%} + {%- set textfsm_extract = salt.textfsm.index(command, output=output) -%} + ''' + ret = { + 'out': None, + 'result': False, + 'comment': '' + } + if not HAS_CLITABLE: + ret['comment'] = 'TextFSM doesnt seem that has clitable embedded.' + log.error(ret['comment']) + return ret + if not platform: + platform_grain_name = __opts__.get('textfsm_platform_grain') or\ + __pillar__.get('textfsm_platform_grain', platform_grain_name) + if platform_grain_name: + log.debug('Using the {} grain to identify the platform name'.format(platform_grain_name)) + platform = __grains__.get(platform_grain_name) + if not platform: + ret['comment'] = 'Unable to identify the platform name using the {} grain.'.format(platform_grain_name) + return ret + log.info('Using platform: {}'.format(platform)) + else: + ret['comment'] = 'No platform specified, no platform grain identifier configured.' + log.error(ret['comment']) + return ret + if not textfsm_path: + log.debug('No TextFSM templates path specified, trying to look into the opts and pillar') + textfsm_path = __opts__.get('textfsm_path') or __pillar__.get('textfsm_path') + if not textfsm_path: + ret['comment'] = 'No TextFSM templates path specified. Please configure in opts/pillar/function args.' + log.error(ret['comment']) + return ret + log.debug('Using the saltenv: {}'.format(saltenv)) + log.debug('Caching {} using the Salt fileserver'.format(textfsm_path)) + textfsm_cachedir_ret = __salt__['cp.cache_dir'](textfsm_path, + saltenv=saltenv, + include_empty=include_empty, + include_pat=include_pat, + exclude_pat=exclude_pat) + log.debug('Cache fun return:') + log.debug(textfsm_cachedir_ret) + if not textfsm_cachedir_ret: + ret['comment'] = 'Unable to fetch from {}. Is the TextFSM path correctly specified?'.format(textfsm_path) + log.error(ret['comment']) + return ret + textfsm_cachedir = os.path.dirname(textfsm_cachedir_ret[0]) # first item + index_file = __opts__.get('textfsm_index_file') or __pillar__.get('textfsm_index_file', 'index') + index_file_path = os.path.join(textfsm_cachedir, index_file) + log.debug('Using the cached index file: {}'.format(index_file_path)) + log.debug('TextFSM templates cached under: {}'.format(textfsm_cachedir)) + textfsm_obj = clitable.CliTable(index_file_path, textfsm_cachedir) + attrs = { + 'Command': command + } + platform_column_name = __opts__.get('textfsm_platform_column_name') or\ + __pillar__.get('textfsm_platform_column_name', 'Platform') + log.info('Using the TextFSM platform idenfiticator: {}'.format(platform_column_name)) + attrs[platform_column_name] = platform + log.debug('Processing the TextFSM index file using the attributes: {}'.format(attrs)) + if not output and output_file: + log.debug('Processing the output from {}'.format(output_file)) + output = __salt__['cp.get_file_str'](output_file, saltenv=saltenv) + if output is False: + ret['comment'] = 'Unable to read from {}. Please specify a valid file or text.'.format(output_file) + log.error(ret['comment']) + return ret + if not output: + ret['comment'] = 'Please specify a valid output text or file' + log.error(ret['comment']) + return ret + log.debug('Processing the raw text:') + log.debug(output) + try: + # Parse output through template + textfsm_obj.ParseCmd(output, attrs) + ret['out'] = _clitable_to_dict(textfsm_obj, textfsm_obj) + ret['result'] = True + except clitable.CliTableError as cterr: + log.error('Unable to proces the CliTable', exc_info=True) + ret['comment'] = 'Unable to process the output: {}'.format(cterr) + return ret diff --git a/salt/modules/timezone.py b/salt/modules/timezone.py index d12f97bb56d..d3f4b645b4c 100644 --- a/salt/modules/timezone.py +++ b/salt/modules/timezone.py @@ -14,7 +14,10 @@ import string # Import salt libs import salt.utils +import salt.utils.files import salt.utils.itertools +import salt.utils.path +import salt.utils.platform from salt.exceptions import SaltInvocationError, CommandExecutionError log = logging.getLogger(__name__) @@ -26,12 +29,12 @@ def __virtual__(): ''' Only work on POSIX-like systems ''' - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): return (False, 'The timezone execution module failed to load: ' 'win_timezone.py should replace this module on Windows.' 'There was a problem loading win_timezone.py.') - if salt.utils.is_darwin(): + if salt.utils.platform.is_darwin(): return (False, 'The timezone execution module failed to load: ' 'mac_timezone.py should replace this module on macOS.' 'There was a problem loading mac_timezone.py.') @@ -54,7 +57,7 @@ def _timedatectl(): def _get_zone_solaris(): tzfile = '/etc/TIMEZONE' - with salt.utils.fopen(tzfile, 'r') as fp_: + with salt.utils.files.fopen(tzfile, 'r') as fp_: for line in fp_: if 'TZ=' in line: zonepart = line.rstrip('\n').split('=')[-1] @@ -81,7 +84,7 @@ def _get_adjtime_timezone(): def _get_zone_sysconfig(): tzfile = '/etc/sysconfig/clock' - with salt.utils.fopen(tzfile, 'r') as fp_: + with salt.utils.files.fopen(tzfile, 'r') as fp_: for line in fp_: if re.match(r'^\s*#', line): continue @@ -132,7 +135,7 @@ def _get_zone_etc_localtime(): def _get_zone_etc_timezone(): tzfile = '/etc/timezone' try: - with salt.utils.fopen(tzfile, 'r') as fp_: + with salt.utils.files.fopen(tzfile, 'r') as fp_: return fp_.read().strip() except IOError as exc: raise CommandExecutionError( @@ -143,7 +146,7 @@ def _get_zone_etc_timezone(): def _get_zone_aix(): tzfile = '/etc/environment' - with salt.utils.fopen(tzfile, 'r') as fp_: + with salt.utils.files.fopen(tzfile, 'r') as fp_: for line in fp_: if 'TZ=' in line: zonepart = line.rstrip('\n').split('=')[-1] @@ -168,7 +171,7 @@ def get_zone(): salt '*' timezone.get_zone ''' - if salt.utils.which('timedatectl'): + if salt.utils.path.which('timedatectl'): ret = _timedatectl() for line in (x.strip() for x in salt.utils.itertools.split(ret['stdout'], '\n')): @@ -260,7 +263,7 @@ def set_zone(timezone): salt '*' timezone.set_zone 'CST6CDT,M3.2.0/2:00:00,M11.1.0/2:00:00' ''' - if salt.utils.which('timedatectl'): + if salt.utils.path.which('timedatectl'): try: __salt__['cmd.run']('timedatectl set-timezone {0}'.format(timezone)) except CommandExecutionError: @@ -302,7 +305,7 @@ def set_zone(timezone): __salt__['file.sed']( '/etc/sysconfig/clock', '^TIMEZONE=.*', 'TIMEZONE="{0}"'.format(timezone)) elif 'Debian' in __grains__['os_family'] or 'Gentoo' in __grains__['os_family']: - with salt.utils.fopen('/etc/timezone', 'w') as ofh: + with salt.utils.files.fopen('/etc/timezone', 'w') as ofh: ofh.write(timezone.strip()) ofh.write('\n') @@ -371,7 +374,7 @@ def get_hwclock(): salt '*' timezone.get_hwclock ''' - if salt.utils.which('timedatectl'): + if salt.utils.path.which('timedatectl'): ret = _timedatectl() for line in (x.strip() for x in ret['stdout'].splitlines()): if 'rtc in local tz' in line.lower(): @@ -396,7 +399,7 @@ def get_hwclock(): if 'Debian' in __grains__['os_family']: # Original way to look up hwclock on Debian-based systems try: - with salt.utils.fopen('/etc/default/rcS', 'r') as fp_: + with salt.utils.files.fopen('/etc/default/rcS', 'r') as fp_: for line in fp_: if re.match(r'^\s*#', line): continue @@ -415,7 +418,7 @@ def get_hwclock(): if not os.path.exists('/etc/adjtime'): offset_file = '/etc/conf.d/hwclock' try: - with salt.utils.fopen(offset_file, 'r') as fp_: + with salt.utils.files.fopen(offset_file, 'r') as fp_: for line in fp_: if line.startswith('clock='): line = line.rstrip('\n') @@ -438,7 +441,7 @@ def get_hwclock(): if 'Solaris' in __grains__['os_family']: offset_file = '/etc/rtc_config' try: - with salt.utils.fopen(offset_file, 'r') as fp_: + with salt.utils.files.fopen(offset_file, 'r') as fp_: for line in fp_: if line.startswith('zone_info=GMT'): return 'UTC' @@ -455,7 +458,7 @@ def get_hwclock(): if 'AIX' in __grains__['os_family']: offset_file = '/etc/environment' try: - with salt.utils.fopen(offset_file, 'r') as fp_: + with salt.utils.files.fopen(offset_file, 'r') as fp_: for line in fp_: if line.startswith('TZ=UTC'): return 'UTC' diff --git a/salt/modules/tls.py b/salt/modules/tls.py index 0ad6fd8fbd6..d065d4a95d2 100644 --- a/salt/modules/tls.py +++ b/salt/modules/tls.py @@ -101,7 +101,7 @@ Create a server req + cert with non-CN filename for the cert from __future__ import absolute_import # pylint: disable=C0103 -# Import python libs +# Import Python libs import os import re import time @@ -109,14 +109,15 @@ import calendar import logging import math import binascii -import salt.utils from datetime import datetime -# Import salt libs +# Import Salt libs +import salt.utils.files +import salt.utils.stringutils from salt.utils.versions import LooseVersion as _LooseVersion # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import range as _range HAS_SSL = False @@ -242,7 +243,7 @@ def _new_serial(ca_name): mode = 'w' else: mode = 'a+' - with salt.utils.fopen(serial_file, mode) as ofile: + with salt.utils.files.fopen(serial_file, mode) as ofile: ofile.write(str(hashnum)) return hashnum @@ -263,7 +264,7 @@ def _get_basic_info(ca_name, cert, ca_dir=None): expire_date = _four_digit_year_to_two_digit( datetime.strptime( - salt.utils.to_str(cert.get_notAfter()), + salt.utils.stringutils.to_str(cert.get_notAfter()), four_digit_year_fmt) ) serial_number = format(cert.get_serial_number(), 'X') @@ -306,7 +307,7 @@ def _write_cert_to_database(ca_name, cert, cacert_path=None, status='V'): subject ) - with salt.utils.fopen(index_file, 'a+') as ofile: + with salt.utils.files.fopen(index_file, 'a+') as ofile: ofile.write(index_data) @@ -343,14 +344,14 @@ def maybe_fix_ssl_version(ca_name, cacert_path=None, ca_filename=None): cert_base_path(), ca_name, ca_filename) - with salt.utils.fopen(certp) as fic: + with salt.utils.files.fopen(certp) as fic: cert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, fic.read()) if cert.get_version() == 3: log.info( 'Regenerating wrong x509 version ' 'for certificate {0}'.format(certp)) - with salt.utils.fopen(ca_keyp) as fic2: + with salt.utils.files.fopen(ca_keyp) as fic2: try: # try to determine the key bits key = OpenSSL.crypto.load_privatekey( @@ -445,7 +446,7 @@ def get_ca(ca_name, as_text=False, cacert_path=None): raise ValueError('Certificate does not exist for {0}'.format(ca_name)) else: if as_text: - with salt.utils.fopen(certp) as fic: + with salt.utils.files.fopen(certp) as fic: certp = fic.read() return certp @@ -490,7 +491,7 @@ def get_ca_signed_cert(ca_name, raise ValueError('Certificate does not exists for {0}'.format(CN)) else: if as_text: - with salt.utils.fopen(certp) as fic: + with salt.utils.files.fopen(certp) as fic: certp = fic.read() return certp @@ -539,7 +540,7 @@ def get_ca_signed_key(ca_name, raise ValueError('Certificate does not exists for {0}'.format(CN)) else: if as_text: - with salt.utils.fopen(keyp) as fic: + with salt.utils.files.fopen(keyp) as fic: keyp = fic.read() return keyp @@ -575,7 +576,7 @@ def create_ca(ca_name, L='Salt Lake City', O='SaltStack', OU=None, - emailAddress='xyz@pdq.net', + emailAddress=None, fixmode=False, cacert_path=None, ca_filename=None, @@ -605,7 +606,7 @@ def create_ca(ca_name, OU organizational unit, default is None emailAddress - email address for the CA owner, default is 'xyz@pdq.net' + email address for the CA owner, default is None cacert_path absolute path to ca certificates root directory ca_filename @@ -671,7 +672,7 @@ def create_ca(ca_name, # try to reuse existing ssl key key = None if os.path.exists(ca_keyp): - with salt.utils.fopen(ca_keyp) as fic2: + with salt.utils.files.fopen(ca_keyp) as fic2: # try to determine the key bits try: key = OpenSSL.crypto.load_privatekey( @@ -698,7 +699,8 @@ def create_ca(ca_name, if OU: ca.get_subject().OU = OU ca.get_subject().CN = CN - ca.get_subject().emailAddress = emailAddress + if emailAddress: + ca.get_subject().emailAddress = emailAddress ca.gmtime_adj_notBefore(0) ca.gmtime_adj_notAfter(int(days) * 24 * 60 * 60) @@ -729,20 +731,20 @@ def create_ca(ca_name, if os.path.exists(ca_keyp): bck = "{0}.{1}".format(ca_keyp, datetime.utcnow().strftime( "%Y%m%d%H%M%S")) - with salt.utils.fopen(ca_keyp) as fic: + with salt.utils.files.fopen(ca_keyp) as fic: old_key = fic.read().strip() if old_key.strip() == keycontent.strip(): write_key = False else: log.info('Saving old CA ssl key in {0}'.format(bck)) - with salt.utils.fopen(bck, 'w') as bckf: + with salt.utils.files.fopen(bck, 'w') as bckf: bckf.write(old_key) os.chmod(bck, 0o600) if write_key: - with salt.utils.fopen(ca_keyp, 'wb') as ca_key: + with salt.utils.files.fopen(ca_keyp, 'wb') as ca_key: ca_key.write(keycontent) - with salt.utils.fopen(certp, 'wb') as ca_crt: + with salt.utils.files.fopen(certp, 'wb') as ca_crt: ca_crt.write( OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, ca)) @@ -854,7 +856,7 @@ def create_csr(ca_name, L='Salt Lake City', O='SaltStack', OU=None, - emailAddress='xyz@pdq.net', + emailAddress=None, subjectAltName=None, cacert_path=None, ca_filename=None, @@ -886,7 +888,7 @@ def create_csr(ca_name, OU organizational unit, default is None emailAddress - email address for the request, default is 'xyz@pdq.net' + email address for the request, default is None subjectAltName valid subjectAltNames in full form, e.g. to add DNS entry you would call this function with this value: @@ -998,7 +1000,8 @@ def create_csr(ca_name, if OU: req.get_subject().OU = OU req.get_subject().CN = CN - req.get_subject().emailAddress = emailAddress + if emailAddress: + req.get_subject().emailAddress = emailAddress try: extensions = get_extensions(cert_type)['csr'] @@ -1007,9 +1010,9 @@ def create_csr(ca_name, for ext, value in extensions.items(): if six.PY3: - ext = salt.utils.to_bytes(ext) + ext = salt.utils.stringutils.to_bytes(ext) if isinstance(value, six.string_types): - value = salt.utils.to_bytes(value) + value = salt.utils.stringutils.to_bytes(value) extension_adds.append( OpenSSL.crypto.X509Extension( ext, False, value)) @@ -1019,7 +1022,7 @@ def create_csr(ca_name, if subjectAltName: if X509_EXT_ENABLED: - if isinstance(subjectAltName, str): + if isinstance(subjectAltName, six.string_types): subjectAltName = [subjectAltName] extension_adds.append( @@ -1038,13 +1041,13 @@ def create_csr(ca_name, req.sign(key, digest) # Write private key and request - with salt.utils.fopen('{0}/{1}.key'.format(csr_path, + with salt.utils.files.fopen('{0}/{1}.key'.format(csr_path, csr_filename), 'wb+') as priv_key: priv_key.write( OpenSSL.crypto.dump_privatekey(OpenSSL.crypto.FILETYPE_PEM, key) ) - with salt.utils.fopen(csr_f, 'wb+') as csr: + with salt.utils.files.fopen(csr_f, 'wb+') as csr: csr.write( OpenSSL.crypto.dump_certificate_request( OpenSSL.crypto.FILETYPE_PEM, @@ -1074,7 +1077,7 @@ def create_self_signed_cert(tls_dir='tls', L='Salt Lake City', O='SaltStack', OU=None, - emailAddress='xyz@pdq.net', + emailAddress=None, cacert_path=None, cert_filename=None, digest='sha256', @@ -1100,7 +1103,7 @@ def create_self_signed_cert(tls_dir='tls', OU organizational unit, default is None emailAddress - email address for the request, default is 'xyz@pdq.net' + email address for the request, default is None cacert_path absolute path to ca certificates root directory digest @@ -1171,7 +1174,8 @@ def create_self_signed_cert(tls_dir='tls', if OU: cert.get_subject().OU = OU cert.get_subject().CN = CN - cert.get_subject().emailAddress = emailAddress + if emailAddress: + cert.get_subject().emailAddress = emailAddress cert.set_serial_number(_new_serial(tls_dir)) cert.set_issuer(cert.get_subject()) @@ -1182,7 +1186,7 @@ def create_self_signed_cert(tls_dir='tls', priv_key_path = '{0}/{1}/certs/{2}.key'.format(cert_base_path(), tls_dir, cert_filename) - with salt.utils.fopen(priv_key_path, 'wb+') as priv_key: + with salt.utils.files.fopen(priv_key_path, 'wb+') as priv_key: priv_key.write( OpenSSL.crypto.dump_privatekey(OpenSSL.crypto.FILETYPE_PEM, key) ) @@ -1190,7 +1194,7 @@ def create_self_signed_cert(tls_dir='tls', crt_path = '{0}/{1}/certs/{2}.crt'.format(cert_base_path(), tls_dir, cert_filename) - with salt.utils.fopen(crt_path, 'wb+') as crt: + with salt.utils.files.fopen(crt_path, 'wb+') as crt: crt.write( OpenSSL.crypto.dump_certificate( OpenSSL.crypto.FILETYPE_PEM, @@ -1355,13 +1359,13 @@ def create_ca_signed_cert(ca_name, maybe_fix_ssl_version(ca_name, cacert_path=cacert_path, ca_filename=ca_filename) - with salt.utils.fopen('{0}/{1}/{2}.crt'.format(cert_base_path(), + with salt.utils.files.fopen('{0}/{1}/{2}.crt'.format(cert_base_path(), ca_name, ca_filename)) as fhr: ca_cert = OpenSSL.crypto.load_certificate( OpenSSL.crypto.FILETYPE_PEM, fhr.read() ) - with salt.utils.fopen('{0}/{1}/{2}.key'.format(cert_base_path(), + with salt.utils.files.fopen('{0}/{1}/{2}.key'.format(cert_base_path(), ca_name, ca_filename)) as fhr: ca_key = OpenSSL.crypto.load_privatekey( @@ -1375,7 +1379,7 @@ def create_ca_signed_cert(ca_name, try: csr_path = '{0}/{1}.csr'.format(cert_path, csr_filename) - with salt.utils.fopen(csr_path) as fhr: + with salt.utils.files.fopen(csr_path) as fhr: req = OpenSSL.crypto.load_certificate_request( OpenSSL.crypto.FILETYPE_PEM, fhr.read()) @@ -1430,7 +1434,7 @@ def create_ca_signed_cert(ca_name, cert_full_path = '{0}/{1}.crt'.format(cert_path, cert_filename) - with salt.utils.fopen(cert_full_path, 'wb+') as crt: + with salt.utils.files.fopen(cert_full_path, 'wb+') as crt: crt.write( OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, cert)) @@ -1487,7 +1491,7 @@ def create_pkcs12(ca_name, CN, passphrase='', cacert_path=None, replace=False): return 'Certificate "{0}" already exists'.format(CN) try: - with salt.utils.fopen('{0}/{1}/{2}_ca_cert.crt'.format(cert_base_path(), + with salt.utils.files.fopen('{0}/{1}/{2}_ca_cert.crt'.format(cert_base_path(), ca_name, ca_name)) as fhr: ca_cert = OpenSSL.crypto.load_certificate( @@ -1498,14 +1502,14 @@ def create_pkcs12(ca_name, CN, passphrase='', cacert_path=None, replace=False): return 'There is no CA named "{0}"'.format(ca_name) try: - with salt.utils.fopen('{0}/{1}/certs/{2}.crt'.format(cert_base_path(), + with salt.utils.files.fopen('{0}/{1}/certs/{2}.crt'.format(cert_base_path(), ca_name, CN)) as fhr: cert = OpenSSL.crypto.load_certificate( OpenSSL.crypto.FILETYPE_PEM, fhr.read() ) - with salt.utils.fopen('{0}/{1}/certs/{2}.key'.format(cert_base_path(), + with salt.utils.files.fopen('{0}/{1}/certs/{2}.key'.format(cert_base_path(), ca_name, CN)) as fhr: key = OpenSSL.crypto.load_privatekey( @@ -1521,7 +1525,7 @@ def create_pkcs12(ca_name, CN, passphrase='', cacert_path=None, replace=False): pkcs12.set_ca_certificates([ca_cert]) pkcs12.set_privatekey(key) - with salt.utils.fopen('{0}/{1}/certs/{2}.p12'.format(cert_base_path(), + with salt.utils.files.fopen('{0}/{1}/certs/{2}.p12'.format(cert_base_path(), ca_name, CN), 'wb') as ofile: ofile.write(pkcs12.export(passphrase=passphrase)) @@ -1553,7 +1557,7 @@ def cert_info(cert_path, digest='sha256'): # format that OpenSSL returns dates in date_fmt = '%Y%m%d%H%M%SZ' - with salt.utils.fopen(cert_path) as cert_file: + with salt.utils.files.fopen(cert_path) as cert_file: cert = OpenSSL.crypto.load_certificate( OpenSSL.crypto.FILETYPE_PEM, cert_file.read() @@ -1562,21 +1566,21 @@ def cert_info(cert_path, digest='sha256'): issuer = {} for key, value in cert.get_issuer().get_components(): if isinstance(key, bytes): - key = salt.utils.to_str(key, __salt_system_encoding__) + key = salt.utils.stringutils.to_str(key, __salt_system_encoding__) if isinstance(value, bytes): - value = salt.utils.to_str(value, __salt_system_encoding__) + value = salt.utils.stringutils.to_str(value, __salt_system_encoding__) issuer[key] = value subject = {} for key, value in cert.get_subject().get_components(): if isinstance(key, bytes): - key = salt.utils.to_str(key, __salt_system_encoding__) + key = salt.utils.stringutils.to_str(key, __salt_system_encoding__) if isinstance(value, bytes): - value = salt.utils.to_str(value, __salt_system_encoding__) + value = salt.utils.stringutils.to_str(value, __salt_system_encoding__) subject[key] = value ret = { - 'fingerprint': salt.utils.to_str(cert.digest(digest)), + 'fingerprint': salt.utils.stringutils.to_str(cert.digest(digest)), 'subject': subject, 'issuer': issuer, 'serial_number': cert.get_serial_number(), @@ -1612,7 +1616,7 @@ def cert_info(cert_path, digest='sha256'): try: value = cert.get_signature_algorithm() if isinstance(value, bytes): - value = salt.utils.to_str(value, __salt_system_encoding__) + value = salt.utils.stringutils.to_str(value, __salt_system_encoding__) ret['signature_algorithm'] = value except AttributeError: # On py3 at least @@ -1668,7 +1672,7 @@ def create_empty_crl( return 'CRL "{0}" already exists'.format(crl_file) try: - with salt.utils.fopen('{0}/{1}/{2}.crt'.format( + with salt.utils.files.fopen('{0}/{1}/{2}.crt'.format( cert_base_path(), ca_name, ca_filename)) as fp_: @@ -1676,7 +1680,7 @@ def create_empty_crl( OpenSSL.crypto.FILETYPE_PEM, fp_.read() ) - with salt.utils.fopen('{0}/{1}/{2}.key'.format( + with salt.utils.files.fopen('{0}/{1}/{2}.key'.format( cert_base_path(), ca_name, ca_filename)) as fp_: @@ -1690,7 +1694,7 @@ def create_empty_crl( crl = OpenSSL.crypto.CRL() crl_text = crl.export(ca_cert, ca_key) - with salt.utils.fopen(crl_file, 'w') as f: + with salt.utils.files.fopen(crl_file, 'w') as f: f.write(crl_text) return 'Created an empty CRL: "{0}"'.format(crl_file) @@ -1754,7 +1758,7 @@ def revoke_cert( cert_filename = '{0}'.format(CN) try: - with salt.utils.fopen('{0}/{1}/{2}.crt'.format( + with salt.utils.files.fopen('{0}/{1}/{2}.crt'.format( cert_base_path(), ca_name, ca_filename)) as fp_: @@ -1762,7 +1766,7 @@ def revoke_cert( OpenSSL.crypto.FILETYPE_PEM, fp_.read() ) - with salt.utils.fopen('{0}/{1}/{2}.key'.format( + with salt.utils.files.fopen('{0}/{1}/{2}.key'.format( cert_base_path(), ca_name, ca_filename)) as fp_: @@ -1774,7 +1778,7 @@ def revoke_cert( return 'There is no CA named "{0}"'.format(ca_name) try: - with salt.utils.fopen('{}/{}.crt'.format(cert_path, + with salt.utils.files.fopen('{}/{}.crt'.format(cert_path, cert_filename)) as rfh: client_cert = OpenSSL.crypto.load_certificate( OpenSSL.crypto.FILETYPE_PEM, @@ -1805,7 +1809,7 @@ def revoke_cert( index_serial_subject) ret = {} - with salt.utils.fopen(index_file) as fp_: + with salt.utils.files.fopen(index_file) as fp_: for line in fp_: if index_r_data_pattern.match(line): revoke_date = line.split('\t')[2] @@ -1834,7 +1838,7 @@ def revoke_cert( crl = OpenSSL.crypto.CRL() - with salt.utils.fopen(index_file) as fp_: + with salt.utils.files.fopen(index_file) as fp_: for line in fp_: if line.startswith('R'): fields = line.split('\t') @@ -1860,7 +1864,7 @@ def revoke_cert( crl_file) return ret - with salt.utils.fopen(crl_file, 'w') as fp_: + with salt.utils.files.fopen(crl_file, 'w') as fp_: fp_.write(crl_text) return ('Revoked Certificate: "{0}/{1}.crt", ' diff --git a/salt/modules/trafficserver.py b/salt/modules/trafficserver.py index 19b49e732d0..7f3bdaab0a7 100644 --- a/salt/modules/trafficserver.py +++ b/salt/modules/trafficserver.py @@ -14,7 +14,9 @@ import logging import subprocess # Import salt libs -import salt.utils +import salt.utils.path +import salt.utils.stringutils +import salt.utils.versions __virtualname__ = 'trafficserver' @@ -22,14 +24,14 @@ log = logging.getLogger(__name__) def __virtual__(): - if salt.utils.which('traffic_ctl') or salt.utils.which('traffic_line'): + if salt.utils.path.which('traffic_ctl') or salt.utils.path.which('traffic_line'): return __virtualname__ return (False, 'trafficserver execution module not loaded: ' 'neither traffic_ctl nor traffic_line was found.') -_TRAFFICLINE = salt.utils.which('traffic_line') -_TRAFFICCTL = salt.utils.which('traffic_ctl') +_TRAFFICLINE = salt.utils.path.which('traffic_line') +_TRAFFICCTL = salt.utils.path.which('traffic_ctl') def _traffic_ctl(*args): @@ -57,7 +59,7 @@ def _subprocess(cmd): try: proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) - ret = salt.utils.to_str(proc.communicate()[0]).strip() + ret = salt.utils.stringutils.to_str(proc.communicate()[0]).strip() retcode = proc.wait() if ret: @@ -213,7 +215,7 @@ def match_var(regex): salt '*' trafficserver.match_var regex ''' - salt.utils.warn_until( + salt.utils.versions.warn_until( 'Fluorine', 'The \'match_var\' function has been deprecated and will be removed in Salt ' '{version}. Please use \'match_metric\' or \'match_config\' instead.' @@ -355,7 +357,7 @@ def read_var(*args): salt '*' trafficserver.read_var proxy.process.http.tcp_hit_count_stat ''' - salt.utils.warn_until( + salt.utils.versions.warn_until( 'Fluorine', 'The \'read_var\' function has been deprecated and will be removed in Salt ' '{version}. Please use \'read_metric\' or \'read_config\' instead.' @@ -384,7 +386,7 @@ def set_var(variable, value): salt '*' trafficserver.set_var proxy.config.http.server_ports ''' - salt.utils.warn_until( + salt.utils.versions.warn_until( 'Fluorine', 'The \'set_var\' function has been deprecated and will be removed in Salt ' '{version}. Please use \'set_config\' instead.' diff --git a/salt/modules/tuned.py b/salt/modules/tuned.py index 45d54b512b4..1f4889cf50f 100644 --- a/salt/modules/tuned.py +++ b/salt/modules/tuned.py @@ -13,7 +13,7 @@ from __future__ import absolute_import import re # Import Salt libs -import salt.utils +import salt.utils.path __func_alias__ = { 'list_': 'list', @@ -27,7 +27,7 @@ def __virtual__(): Check to see if tuned-adm binary is installed on the system ''' - tuned_adm = salt.utils.which('tuned-adm') + tuned_adm = salt.utils.path.which('tuned-adm') if not tuned_adm: return (False, 'The tuned execution module failed to load: the tuned-adm binary is not in the path.') return __virtualname__ diff --git a/salt/modules/twilio_notify.py b/salt/modules/twilio_notify.py index 520a7c0cc3c..334ecd167e2 100644 --- a/salt/modules/twilio_notify.py +++ b/salt/modules/twilio_notify.py @@ -21,8 +21,15 @@ import logging HAS_LIBS = False try: - from twilio.rest import TwilioRestClient - from twilio import TwilioRestException + import twilio + if twilio.__version__ > 5: + TWILIO_5 = False + from twilio.rest import Client as TwilioRestClient + from twilio.rest import TwilioException as TwilioRestException + else: + TWILIO_5 = True + from twilio.rest import TwilioRestClient + from twilio import TwilioRestException HAS_LIBS = True except ImportError: pass @@ -67,7 +74,10 @@ def send_sms(profile, body, to, from_): ret['message']['sid'] = None client = _get_twilio(profile) try: - message = client.sms.messages.create(body=body, to=to, from_=from_) + if TWILIO_5: + message = client.sms.messages.create(body=body, to=to, from_=from_) + else: + message = client.messages.create(body=body, to=to, from_=from_) except TwilioRestException as exc: ret['_error'] = {} ret['_error']['code'] = exc.code diff --git a/salt/modules/udev.py b/salt/modules/udev.py index 285f9f1fee3..758c2bb32a1 100644 --- a/salt/modules/udev.py +++ b/salt/modules/udev.py @@ -8,7 +8,7 @@ Manage and query udev info from __future__ import absolute_import import logging -import salt.utils +import salt.utils.path import salt.modules.cmdmod from salt.exceptions import CommandExecutionError @@ -23,7 +23,7 @@ def __virtual__(): ''' Only work when udevadm is installed. ''' - return salt.utils.which_bin(['udevadm']) is not None + return salt.utils.path.which_bin(['udevadm']) is not None def _parse_udevadm_info(udev_info): diff --git a/salt/modules/upstart.py b/salt/modules/upstart.py index 19bb822a506..dfd3a503381 100644 --- a/salt/modules/upstart.py +++ b/salt/modules/upstart.py @@ -55,6 +55,7 @@ import fnmatch # Import salt libs import salt.utils +import salt.utils.files import salt.modules.cmdmod import salt.utils.systemd @@ -109,7 +110,7 @@ def _default_runlevel(): # Try to get the "main" default. If this fails, throw up our # hands and just guess "2", because things are horribly broken try: - with salt.utils.fopen('/etc/init/rc-sysinit.conf') as fp_: + with salt.utils.files.fopen('/etc/init/rc-sysinit.conf') as fp_: for line in fp_: if line.startswith('env DEFAULT_RUNLEVEL'): runlevel = line.split('=')[-1].strip() @@ -118,7 +119,7 @@ def _default_runlevel(): # Look for an optional "legacy" override in /etc/inittab try: - with salt.utils.fopen('/etc/inittab') as fp_: + with salt.utils.files.fopen('/etc/inittab') as fp_: for line in fp_: if not line.startswith('#') and 'initdefault' in line: runlevel = line.split(':')[1] @@ -130,7 +131,7 @@ def _default_runlevel(): try: valid_strings = set( ('0', '1', '2', '3', '4', '5', '6', 's', 'S', '-s', 'single')) - with salt.utils.fopen('/proc/cmdline') as fp_: + with salt.utils.files.fopen('/proc/cmdline') as fp_: for line in fp_: for arg in line.strip().split(): if arg in valid_strings: @@ -182,7 +183,7 @@ def _upstart_is_disabled(name): ''' files = ['/etc/init/{0}.conf'.format(name), '/etc/init/{0}.override'.format(name)] for file_name in itertools.ifilter(os.path.isfile, files): - with salt.utils.fopen(file_name) as fp_: + with salt.utils.files.fopen(file_name) as fp_: if re.search(r'^\s*manual', fp_.read(), re.MULTILINE): return True return False @@ -492,7 +493,7 @@ def _upstart_disable(name): if _upstart_is_disabled(name): return _upstart_is_disabled(name) override = '/etc/init/{0}.override'.format(name) - with salt.utils.fopen(override, 'a') as ofile: + with salt.utils.files.fopen(override, 'a') as ofile: ofile.write('manual\n') return _upstart_is_disabled(name) @@ -506,7 +507,7 @@ def _upstart_enable(name): override = '/etc/init/{0}.override'.format(name) files = ['/etc/init/{0}.conf'.format(name), override] for file_name in itertools.ifilter(os.path.isfile, files): - with salt.utils.fopen(file_name, 'r+') as fp_: + with salt.utils.files.fopen(file_name, 'r+') as fp_: new_text = re.sub(r'^\s*manual\n?', '', fp_.read(), 0, re.MULTILINE) fp_.seek(0) fp_.write(new_text) diff --git a/salt/modules/useradd.py b/salt/modules/useradd.py index f69dad6aa0e..24900f34801 100644 --- a/salt/modules/useradd.py +++ b/salt/modules/useradd.py @@ -19,11 +19,14 @@ import logging import copy # Import salt libs -import salt.utils -import salt.utils.decorators as decorators -from salt.ext import six +import salt.utils # Can be removed when get_group_list is moved +import salt.utils.files +import salt.utils.decorators.path +import salt.utils.locales from salt.exceptions import CommandExecutionError -from salt.utils import locales + +# Import 3rd-party libs +from salt.ext import six log = logging.getLogger(__name__) @@ -59,10 +62,10 @@ def _get_gecos(name): # Assign empty strings for any unspecified trailing GECOS fields while len(gecos_field) < 4: gecos_field.append('') - return {'fullname': locales.sdecode(gecos_field[0]), - 'roomnumber': locales.sdecode(gecos_field[1]), - 'workphone': locales.sdecode(gecos_field[2]), - 'homephone': locales.sdecode(gecos_field[3])} + return {'fullname': salt.utils.locales.sdecode(gecos_field[0]), + 'roomnumber': salt.utils.locales.sdecode(gecos_field[1]), + 'workphone': salt.utils.locales.sdecode(gecos_field[2]), + 'homephone': salt.utils.locales.sdecode(gecos_field[3])} def _build_gecos(gecos_dict): @@ -138,7 +141,7 @@ def add(name, defs_file = '/etc/login.defs' if __grains__['kernel'] != 'OpenBSD': try: - with salt.utils.fopen(defs_file) as fp_: + with salt.utils.files.fopen(defs_file) as fp_: for line in fp_: if 'USERGROUPS_ENAB' not in line[:15]: continue @@ -158,7 +161,7 @@ def add(name, else: usermgmt_file = '/etc/usermgmt.conf' try: - with salt.utils.fopen(usermgmt_file) as fp_: + with salt.utils.files.fopen(usermgmt_file) as fp_: for line in fp_: if 'group' not in line[:5]: continue @@ -594,7 +597,7 @@ def _format_info(data): 'homephone': gecos_field[3]} -@decorators.which('id') +@salt.utils.decorators.path.which('id') def primary_group(name): ''' Return the primary group of the named user diff --git a/salt/modules/uwsgi.py b/salt/modules/uwsgi.py index d5df5e5826d..ec361cf69bf 100644 --- a/salt/modules/uwsgi.py +++ b/salt/modules/uwsgi.py @@ -12,7 +12,7 @@ from __future__ import absolute_import import json # Import Salt libs -import salt.utils +import salt.utils.path def __virtual__(): @@ -20,7 +20,7 @@ def __virtual__(): Only load the module if uwsgi is installed ''' cmd = 'uwsgi' - if salt.utils.which(cmd): + if salt.utils.path.which(cmd): return cmd return (False, 'The uwsgi execution module failed to load: the uwsgi binary is not in the path.') diff --git a/salt/modules/varnish.py b/salt/modules/varnish.py index 75050657ccd..cd325f24050 100644 --- a/salt/modules/varnish.py +++ b/salt/modules/varnish.py @@ -16,7 +16,7 @@ import logging import re # Import salt libs -import salt.utils +import salt.utils.path log = logging.getLogger(__name__) @@ -28,7 +28,7 @@ def __virtual__(): ''' Only load the module if varnish is installed ''' - if salt.utils.which('varnishd') and salt.utils.which('varnishadm'): + if salt.utils.path.which('varnishd') and salt.utils.path.which('varnishadm'): return __virtualname__ return (False, 'The varnish execution module failed to load: either varnishd or varnishadm is not in the path.') diff --git a/salt/modules/vault.py b/salt/modules/vault.py index f86925720da..c5dab85044b 100644 --- a/salt/modules/vault.py +++ b/salt/modules/vault.py @@ -31,6 +31,25 @@ Functions to interact with Hashicorp Vault. Currently only token auth is supported. The token must be able to create tokens with the policies that should be assigned to minions. Required. + You can still use the token via a OS environment variable via this + config example: + + .. code-block: yaml + + vault: + url: https://vault.service.domain:8200 + auth: + method: token + token: sdb://osenv/VAULT_TOKEN + osenv: + driver: env + + + And then export the VAULT_TOKEN variable in your OS: + + .. code-block: bash + export VAULT_TOKEN=11111111-1111-1111-1111-1111111111111 + policies Policies that are assigned to minions when requesting a token. These can either be static, eg saltstack/minions, or templated, eg diff --git a/salt/modules/vboxmanage.py b/salt/modules/vboxmanage.py index 962d44af2ea..642914f0706 100644 --- a/salt/modules/vboxmanage.py +++ b/salt/modules/vboxmanage.py @@ -22,11 +22,14 @@ import os.path import logging # pylint: disable=import-error,no-name-in-module -import salt.utils -from salt.ext.six import string_types +import salt.utils.files +import salt.utils.path from salt.exceptions import CommandExecutionError # pylint: enable=import-error,no-name-in-module +# Import 3rd-party libs +from salt.ext import six + LOG = logging.getLogger(__name__) UUID_RE = re.compile('[^{0}]'.format('a-zA-Z0-9._-')) @@ -55,7 +58,7 @@ def vboxcmd(): salt '*' vboxmanage.vboxcmd ''' - return salt.utils.which('VBoxManage') + return salt.utils.path.which('VBoxManage') def list_ostypes(): @@ -254,7 +257,7 @@ def create(name, params += ' --name {0}'.format(name) if groups: - if isinstance(groups, string_types): + if isinstance(groups, six.string_types): groups = [groups] if isinstance(groups, list): params += ' --groups {0}'.format(','.join(groups)) @@ -369,7 +372,7 @@ def clonevm(name=None, params += ' --name {0}'.format(new_name) if groups: - if isinstance(groups, string_types): + if isinstance(groups, six.string_types): groups = [groups] if isinstance(groups, list): params += ' --groups {0}'.format(','.join(groups)) @@ -457,7 +460,7 @@ def clonemedium(medium, params += ' ' + uuid_out elif file_out: try: - salt.utils.fopen(file_out, 'w').close() # pylint: disable=resource-leakage + salt.utils.files.fopen(file_out, 'w').close() # pylint: disable=resource-leakage os.unlink(file_out) params += ' ' + file_out except OSError: diff --git a/salt/modules/virt.py b/salt/modules/virt.py index f285ff1117e..33544937362 100644 --- a/salt/modules/virt.py +++ b/salt/modules/virt.py @@ -24,7 +24,7 @@ from xml.etree import ElementTree import yaml import jinja2 import jinja2.exceptions -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import StringIO as _StringIO # pylint: disable=import-error from xml.dom import minidom try: @@ -37,6 +37,8 @@ except ImportError: # Import salt libs import salt.utils import salt.utils.files +import salt.utils.path +import salt.utils.stringutils import salt.utils.templates import salt.utils.validate.net from salt.exceptions import CommandExecutionError, SaltInvocationError @@ -196,14 +198,14 @@ def _libvirt_creds(): stdout = subprocess.Popen(g_cmd, shell=True, stdout=subprocess.PIPE).communicate()[0] - group = salt.utils.to_str(stdout).split('"')[1] + group = salt.utils.stringutils.to_str(stdout).split('"')[1] except IndexError: group = 'root' try: stdout = subprocess.Popen(u_cmd, shell=True, stdout=subprocess.PIPE).communicate()[0] - user = salt.utils.to_str(stdout).split('"')[1] + user = salt.utils.stringutils.to_str(stdout).split('"')[1] except IndexError: user = 'root' return {'user': user, 'group': group} @@ -395,7 +397,7 @@ def _qemu_image_create(vm_name, sfn = __salt__['cp.cache_file'](disk_image, saltenv) qcow2 = False - if salt.utils.which('qemu-img'): + if salt.utils.path.which('qemu-img'): res = __salt__['cmd.run']('qemu-img info {}'.format(sfn)) imageinfo = yaml.load(res) qcow2 = imageinfo['file format'] == 'qcow2' @@ -1086,7 +1088,7 @@ def get_disks(vm_): ['qemu-img', 'info', disks[dev]['file']], shell=False, stdout=subprocess.PIPE).communicate()[0] - qemu_output = salt.utils.to_str(stdout) + qemu_output = salt.utils.stringutils.to_str(stdout) snapshots = False columns = None lines = qemu_output.strip().split('\n') @@ -1439,7 +1441,7 @@ def create_xml_path(path): salt '*' virt.create_xml_path ''' try: - with salt.utils.fopen(path, 'r') as fp_: + with salt.utils.files.fopen(path, 'r') as fp_: return create_xml_str(fp_.read()) except (OSError, IOError): return False @@ -1471,7 +1473,7 @@ def define_xml_path(path): ''' try: - with salt.utils.fopen(path, 'r') as fp_: + with salt.utils.files.fopen(path, 'r') as fp_: return define_xml_str(fp_.read()) except (OSError, IOError): return False @@ -1505,7 +1507,7 @@ def define_vol_xml_path(path): ''' try: - with salt.utils.fopen(path, 'r') as fp_: + with salt.utils.files.fopen(path, 'r') as fp_: return define_vol_xml_str(fp_.read()) except (OSError, IOError): return False @@ -1527,7 +1529,7 @@ def migrate_non_shared(vm_, target, ssh=False): stdout = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE).communicate()[0] - return salt.utils.to_str(stdout) + return salt.utils.stringutils.to_str(stdout) def migrate_non_shared_inc(vm_, target, ssh=False): @@ -1546,7 +1548,7 @@ def migrate_non_shared_inc(vm_, target, ssh=False): stdout = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE).communicate()[0] - return salt.utils.to_str(stdout) + return salt.utils.stringutils.to_str(stdout) def migrate(vm_, target, ssh=False): @@ -1565,7 +1567,7 @@ def migrate(vm_, target, ssh=False): stdout = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE).communicate()[0] - return salt.utils.to_str(stdout) + return salt.utils.stringutils.to_str(stdout) def seed_non_shared_migrate(disks, force=False): @@ -1698,7 +1700,7 @@ def is_kvm_hyper(): salt '*' virt.is_kvm_hyper ''' try: - with salt.utils.fopen('/proc/modules') as fp_: + with salt.utils.files.fopen('/proc/modules') as fp_: if 'kvm_' not in fp_.read(): return False except IOError: @@ -1724,7 +1726,7 @@ def is_xen_hyper(): # virtual_subtype isn't set everywhere. return False try: - with salt.utils.fopen('/proc/modules') as fp_: + with salt.utils.files.fopen('/proc/modules') as fp_: if 'xen_' not in fp_.read(): return False except (OSError, IOError): @@ -2158,7 +2160,7 @@ def cpu_baseline(full=False, migratable=False, out='libvirt'): if full and not getattr(libvirt, 'VIR_CONNECT_BASELINE_CPU_EXPAND_FEATURES', False): # Try do it by ourselves # Find the models in cpu_map.xml and iterate over them for as long as entries have submodels - with salt.utils.fopen('/usr/share/libvirt/cpu_map.xml', 'r') as cpu_map: + with salt.utils.files.fopen('/usr/share/libvirt/cpu_map.xml', 'r') as cpu_map: cpu_map = minidom.parse(cpu_map) cpu_model = cpu.getElementsByTagName('model')[0].childNodes[0].nodeValue diff --git a/salt/modules/virtualenv_mod.py b/salt/modules/virtualenv_mod.py index 77164a31a3e..20145c57e96 100644 --- a/salt/modules/virtualenv_mod.py +++ b/salt/modules/virtualenv_mod.py @@ -14,8 +14,9 @@ import os import sys # Import salt libs -import salt.utils import salt.utils.files +import salt.utils.path +import salt.utils.platform import salt.utils.verify from salt.exceptions import CommandExecutionError, SaltInvocationError from salt.ext.six import string_types @@ -29,7 +30,7 @@ KNOWN_BINARY_NAMES = frozenset([ log = logging.getLogger(__name__) __opts__ = { - 'venv_bin': salt.utils.which_bin(KNOWN_BINARY_NAMES) or 'virtualenv' + 'venv_bin': salt.utils.path.which_bin(KNOWN_BINARY_NAMES) or 'virtualenv' } __pillar__ = {} @@ -185,7 +186,7 @@ def create(path, cmd.append('--distribute') if python is not None and python.strip() != '': - if not salt.utils.which(python): + if not salt.utils.path.which(python): raise CommandExecutionError( 'Cannot find requested python ({0}).'.format(python) ) @@ -199,12 +200,10 @@ def create(path, for entry in extra_search_dir: cmd.append('--extra-search-dir={0}'.format(entry)) if never_download is True: - if virtualenv_version_info >= (1, 10): + if virtualenv_version_info >= (1, 10) and virtualenv_version_info < (14, 0, 0): log.info( - 'The virtualenv \'--never-download\' option has been ' - 'deprecated in virtualenv(>=1.10), as such, the ' - '\'never_download\' option to `virtualenv.create()` has ' - 'also been deprecated and it\'s not necessary anymore.' + '--never-download was deprecated in 1.10.0, but reimplemented in 14.0.0. ' + 'If this feature is needed, please install a supported virtualenv version.' ) else: cmd.append('--never-download') @@ -259,7 +258,7 @@ def create(path, return ret # Check if distribute and pip are already installed - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): venv_python = os.path.join(path, 'Scripts', 'python.exe') venv_pip = os.path.join(path, 'Scripts', 'pip.exe') venv_setuptools = os.path.join(path, 'Scripts', 'easy_install.exe') @@ -453,12 +452,12 @@ def get_resource_content(venv, def _install_script(source, cwd, python, user, saltenv='base', use_vt=False): - if not salt.utils.is_windows(): + if not salt.utils.platform.is_windows(): tmppath = salt.utils.files.mkstemp(dir=cwd) else: tmppath = __salt__['cp.cache_file'](source, saltenv) - if not salt.utils.is_windows(): + if not salt.utils.platform.is_windows(): fn_ = __salt__['cp.cache_file'](source, saltenv) shutil.copyfile(fn_, tmppath) os.chmod(tmppath, 0o500) diff --git a/salt/modules/vsphere.py b/salt/modules/vsphere.py index 2f0062572b9..c0745a4a592 100644 --- a/salt/modules/vsphere.py +++ b/salt/modules/vsphere.py @@ -169,14 +169,17 @@ import inspect from functools import wraps # Import Salt Libs -import salt.ext.six as six +from salt.ext import six import salt.utils -import salt.utils.vmware +import salt.utils.args +import salt.utils.dictupdate as dictupdate import salt.utils.http -from salt.utils import dictupdate -from salt.exceptions import CommandExecutionError, VMwareSaltError +import salt.utils.path +import salt.utils.vmware +import salt.utils.vsan +from salt.exceptions import CommandExecutionError, VMwareSaltError, \ + ArgumentValueError from salt.utils.decorators import depends, ignores_kwargs -from salt.utils import clean_kwargs # Import Third Party Libs try: @@ -185,7 +188,7 @@ try: except ImportError: HAS_PYVMOMI = False -esx_cli = salt.utils.which('esxcli') +esx_cli = salt.utils.path.which('esxcli') if esx_cli: HAS_ESX_CLI = True else: @@ -194,7 +197,7 @@ else: log = logging.getLogger(__name__) __virtualname__ = 'vsphere' -__proxyenabled__ = ['esxi'] +__proxyenabled__ = ['esxi', 'esxcluster', 'esxdatacenter'] def __virtual__(): @@ -203,7 +206,8 @@ def __virtual__(): def get_proxy_type(): ''' - Returns the proxy type + Returns the proxy type retrieved either from the pillar of from the proxy + minion's config. Returns ```` otherwise. CLI Example: @@ -211,7 +215,11 @@ def get_proxy_type(): salt '*' vsphere.get_proxy_type ''' - return __pillar__['proxy']['proxytype'] + if __pillar__.get('proxy', {}).get('proxytype'): + return __pillar__['proxy']['proxytype'] + if __opts__.get('proxy', {}).get('proxytype'): + return __opts__['proxy']['proxytype'] + return '' def _get_proxy_connection_details(): @@ -221,6 +229,10 @@ def _get_proxy_connection_details(): proxytype = get_proxy_type() if proxytype == 'esxi': details = __salt__['esxi.get_details']() + elif proxytype == 'esxcluster': + details = __salt__['esxcluster.get_details']() + elif proxytype == 'esxdatacenter': + details = __salt__['esxdatacenter.get_details']() else: raise CommandExecutionError('\'{0}\' proxy is not supported' ''.format(proxytype)) @@ -241,13 +253,14 @@ def supports_proxies(*proxy_types): Arbitrary list of strings with the supported types of proxies ''' def _supports_proxies(fn): + @wraps(fn) def __supports_proxies(*args, **kwargs): proxy_type = get_proxy_type() if proxy_type not in proxy_types: raise CommandExecutionError( '\'{0}\' proxy is not supported by function {1}' ''.format(proxy_type, fn.__name__)) - return fn(*args, **clean_kwargs(**kwargs)) + return fn(*args, **salt.utils.args.clean_kwargs(**kwargs)) return __supports_proxies return _supports_proxies @@ -258,6 +271,8 @@ def gets_service_instance_via_proxy(fn): proxy details and passes the connection (vim.ServiceInstance) to the decorated function. + Supported proxies: esxi, esxcluster, esxdatacenter. + Notes: 1. The decorated function must have a ``service_instance`` parameter or a ``**kwarg`` type argument (name of argument is not important); @@ -328,7 +343,7 @@ def gets_service_instance_via_proxy(fn): *connection_details) kwargs['service_instance'] = local_service_instance try: - ret = fn(*args, **clean_kwargs(**kwargs)) + ret = fn(*args, **salt.utils.args.clean_kwargs(**kwargs)) # Disconnect if connected in the decorator if local_service_instance: salt.utils.vmware.disconnect(local_service_instance) @@ -342,11 +357,16 @@ def gets_service_instance_via_proxy(fn): return _gets_service_instance_via_proxy -@supports_proxies('esxi') +@depends(HAS_PYVMOMI) +@supports_proxies('esxi', 'esxcluster', 'esxdatacenter') def get_service_instance_via_proxy(service_instance=None): ''' Returns a service instance to the proxied endpoint (vCenter/ESXi host). + service_instance + Service instance (vim.ServiceInstance) of the vCenter. + Default is None. + Note: Should be used by state functions not invoked directly. @@ -357,7 +377,8 @@ def get_service_instance_via_proxy(service_instance=None): return salt.utils.vmware.get_service_instance(*connection_details) -@supports_proxies('esxi') +@depends(HAS_PYVMOMI) +@supports_proxies('esxi', 'esxcluster', 'esxdatacenter') def disconnect(service_instance): ''' Disconnects from a vCenter or ESXi host @@ -1892,7 +1913,7 @@ def get_vsan_eligible_disks(host, username, password, protocol=None, port=None, @depends(HAS_PYVMOMI) -@supports_proxies('esxi') +@supports_proxies('esxi', 'esxcluster', 'esxdatacenter') @gets_service_instance_via_proxy def test_vcenter_connection(service_instance=None): ''' @@ -3580,6 +3601,196 @@ def vsan_enable(host, username, password, protocol=None, port=None, host_names=N return ret +@depends(HAS_PYVMOMI) +@supports_proxies('esxdatacenter', 'esxcluster') +@gets_service_instance_via_proxy +def list_datacenters_via_proxy(datacenter_names=None, service_instance=None): + ''' + Returns a list of dict representations of VMware datacenters. + Connection is done via the proxy details. + + Supported proxies: esxdatacenter + + datacenter_names + List of datacenter names. + Default is None. + + service_instance + Service instance (vim.ServiceInstance) of the vCenter. + Default is None. + + .. code-block:: bash + salt '*' vsphere.list_datacenters_via_proxy + + salt '*' vsphere.list_datacenters_via_proxy dc1 + + salt '*' vsphere.list_datacenters_via_proxy dc1,dc2 + + salt '*' vsphere.list_datacenters_via_proxy datacenter_names=[dc1, dc2] + ''' + if not datacenter_names: + dc_refs = salt.utils.vmware.get_datacenters(service_instance, + get_all_datacenters=True) + else: + dc_refs = salt.utils.vmware.get_datacenters(service_instance, + datacenter_names) + + return [{'name': salt.utils.vmware.get_managed_object_name(dc_ref)} + for dc_ref in dc_refs] + + +@depends(HAS_PYVMOMI) +@supports_proxies('esxdatacenter') +@gets_service_instance_via_proxy +def create_datacenter(datacenter_name, service_instance=None): + ''' + Creates a datacenter. + + Supported proxies: esxdatacenter + + datacenter_name + The datacenter name + + service_instance + Service instance (vim.ServiceInstance) of the vCenter. + Default is None. + + .. code-block:: bash + + salt '*' vsphere.create_datacenter dc1 + ''' + salt.utils.vmware.create_datacenter(service_instance, datacenter_name) + return {'create_datacenter': True} + + +def _get_cluster_dict(cluster_name, cluster_ref): + ''' + Returns a cluster dict representation from + a vim.ClusterComputeResource object. + + cluster_name + Name of the cluster + + cluster_ref + Reference to the cluster + ''' + + log.trace('Building a dictionary representation of cluster ' + '\'{0}\''.format(cluster_name)) + props = salt.utils.vmware.get_properties_of_managed_object( + cluster_ref, + properties=['configurationEx']) + res = {'ha': {'enabled': props['configurationEx'].dasConfig.enabled}, + 'drs': {'enabled': props['configurationEx'].drsConfig.enabled}} + # Convert HA properties of interest + ha_conf = props['configurationEx'].dasConfig + log.trace('ha_conf = {0}'.format(ha_conf)) + res['ha']['admission_control_enabled'] = ha_conf.admissionControlEnabled + if ha_conf.admissionControlPolicy and \ + isinstance(ha_conf.admissionControlPolicy, + vim.ClusterFailoverResourcesAdmissionControlPolicy): + pol = ha_conf.admissionControlPolicy + res['ha']['admission_control_policy'] = \ + {'cpu_failover_percent': pol.cpuFailoverResourcesPercent, + 'memory_failover_percent': pol.memoryFailoverResourcesPercent} + if ha_conf.defaultVmSettings: + def_vm_set = ha_conf.defaultVmSettings + res['ha']['default_vm_settings'] = \ + {'isolation_response': def_vm_set.isolationResponse, + 'restart_priority': def_vm_set.restartPriority} + res['ha']['hb_ds_candidate_policy'] = \ + ha_conf.hBDatastoreCandidatePolicy + if ha_conf.hostMonitoring: + res['ha']['host_monitoring'] = ha_conf.hostMonitoring + if ha_conf.option: + res['ha']['options'] = [{'key': o.key, 'value': o.value} + for o in ha_conf.option] + res['ha']['vm_monitoring'] = ha_conf.vmMonitoring + # Convert DRS properties + drs_conf = props['configurationEx'].drsConfig + log.trace('drs_conf = {0}'.format(drs_conf)) + res['drs']['vmotion_rate'] = 6 - drs_conf.vmotionRate + res['drs']['default_vm_behavior'] = drs_conf.defaultVmBehavior + # vm_swap_placement + res['vm_swap_placement'] = props['configurationEx'].vmSwapPlacement + # Convert VSAN properties + si = salt.utils.vmware.get_service_instance_from_managed_object( + cluster_ref) + + if salt.utils.vsan.vsan_supported(si): + # XXX The correct way of retrieving the VSAN data (on the if branch) + # is not supported before 60u2 vcenter + vcenter_info = salt.utils.vmware.get_service_info(si) + if int(vcenter_info.build) >= 3634794: # 60u2 + # VSAN API is fully supported by the VC starting with 60u2 + vsan_conf = salt.utils.vsan.get_cluster_vsan_info(cluster_ref) + log.trace('vsan_conf = {0}'.format(vsan_conf)) + res['vsan'] = {'enabled': vsan_conf.enabled, + 'auto_claim_storage': + vsan_conf.defaultConfig.autoClaimStorage} + if vsan_conf.dataEfficiencyConfig: + data_eff = vsan_conf.dataEfficiencyConfig + res['vsan'].update({ + # We force compression_enabled to be True/False + 'compression_enabled': + data_eff.compressionEnabled or False, + 'dedup_enabled': data_eff.dedupEnabled}) + else: # before 60u2 (no advanced vsan info) + if props['configurationEx'].vsanConfigInfo: + default_config = \ + props['configurationEx'].vsanConfigInfo.defaultConfig + res['vsan'] = { + 'enabled': props['configurationEx'].vsanConfigInfo.enabled, + 'auto_claim_storage': default_config.autoClaimStorage} + return res + + +@depends(HAS_PYVMOMI) +@supports_proxies('esxcluster', 'esxdatacenter') +@gets_service_instance_via_proxy +def list_cluster(datacenter=None, cluster=None, service_instance=None): + ''' + Returns a dict representation of an ESX cluster. + + datacenter + Name of datacenter containing the cluster. + Ignored if already contained by proxy details. + Default value is None. + + cluster + Name of cluster. + Ignored if already contained by proxy details. + Default value is None. + + service_instance + Service instance (vim.ServiceInstance) of the vCenter. + Default is None. + + .. code-block:: bash + + # vcenter proxy + salt '*' vsphere.list_cluster datacenter=dc1 cluster=cl1 + + # esxdatacenter proxy + salt '*' vsphere.list_cluster cluster=cl1 + + # esxcluster proxy + salt '*' vsphere.list_cluster + ''' + proxy_type = get_proxy_type() + if proxy_type == 'esxdatacenter': + dc_ref = _get_proxy_target(service_instance) + if not cluster: + raise ArgumentValueError('\'cluster\' needs to be specified') + cluster_ref = salt.utils.vmware.get_cluster(dc_ref, cluster) + elif proxy_type == 'esxcluster': + cluster_ref = _get_proxy_target(service_instance) + cluster = __salt__['esxcluster.get_details']()['cluster'] + log.trace('Retrieving representation of cluster \'{0}\' in a ' + '{1} proxy'.format(cluster, proxy_type)) + return _get_cluster_dict(cluster, cluster_ref) + + def _check_hosts(service_instance, host, host_names): ''' Helper function that checks to see if the host provided is a vCenter Server or @@ -3877,6 +4088,7 @@ def _set_syslog_config_helper(host, username, password, syslog_config, config_va return ret_dict +@depends(HAS_PYVMOMI) @ignores_kwargs('credstore') def add_host_to_dvs(host, username, password, vmknic_name, vmnic_name, dvs_name, target_portgroup_name, uplink_portgroup_name, @@ -4204,3 +4416,57 @@ def add_host_to_dvs(host, username, password, vmknic_name, vmnic_name, raise return ret + + +@depends(HAS_PYVMOMI) +@supports_proxies('esxcluster', 'esxdatacenter') +def _get_proxy_target(service_instance): + ''' + Returns the target object of a proxy. + + If the object doesn't exist a VMwareObjectRetrievalError is raised + + service_instance + Service instance (vim.ServiceInstance) of the vCenter/ESXi host. + ''' + proxy_type = get_proxy_type() + if not salt.utils.vmware.is_connection_to_a_vcenter(service_instance): + raise CommandExecutionError('\'_get_proxy_target\' not supported ' + 'when connected via the ESXi host') + reference = None + if proxy_type == 'esxcluster': + host, username, password, protocol, port, mechanism, principal, \ + domain, datacenter, cluster = _get_esxcluster_proxy_details() + + dc_ref = salt.utils.vmware.get_datacenter(service_instance, datacenter) + reference = salt.utils.vmware.get_cluster(dc_ref, cluster) + elif proxy_type == 'esxdatacenter': + # esxdatacenter proxy + host, username, password, protocol, port, mechanism, principal, \ + domain, datacenter = _get_esxdatacenter_proxy_details() + + reference = salt.utils.vmware.get_datacenter(service_instance, + datacenter) + log.trace('reference = {0}'.format(reference)) + return reference + + +def _get_esxdatacenter_proxy_details(): + ''' + Returns the running esxdatacenter's proxy details + ''' + det = __salt__['esxdatacenter.get_details']() + return det.get('vcenter'), det.get('username'), det.get('password'), \ + det.get('protocol'), det.get('port'), det.get('mechanism'), \ + det.get('principal'), det.get('domain'), det.get('datacenter') + + +def _get_esxcluster_proxy_details(): + ''' + Returns the running esxcluster's proxy details + ''' + det = __salt__['esxcluster.get_details']() + return det.get('vcenter'), det.get('username'), det.get('password'), \ + det.get('protocol'), det.get('port'), det.get('mechanism'), \ + det.get('principal'), det.get('domain'), det.get('datacenter'), \ + det.get('cluster') diff --git a/salt/modules/win_autoruns.py b/salt/modules/win_autoruns.py index a31f0946f04..e630e7209a0 100644 --- a/salt/modules/win_autoruns.py +++ b/salt/modules/win_autoruns.py @@ -4,12 +4,12 @@ Module for listing programs that automatically run on startup (very alpha...not tested on anything but my Win 7x64) ''' -# Import python libs +# Import Python libs from __future__ import absolute_import import os -# Import salt libs -import salt.utils +# Import Salt libs +import salt.utils.platform # Define a function alias in order not to shadow built-in's @@ -26,7 +26,7 @@ def __virtual__(): Only works on Windows systems ''' - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): return __virtualname__ return (False, "Module win_autoruns: module only works on Windows systems") diff --git a/salt/modules/win_certutil.py b/salt/modules/win_certutil.py index bf4c1843f90..c45adb3f769 100644 --- a/salt/modules/win_certutil.py +++ b/salt/modules/win_certutil.py @@ -14,7 +14,7 @@ import re import logging # Import Salt Libs -import salt.utils +import salt.utils.platform log = logging.getLogger(__name__) __virtualname__ = "certutil" @@ -24,7 +24,7 @@ def __virtual__(): ''' Only work on Windows ''' - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): return __virtualname__ return False @@ -42,9 +42,10 @@ def get_cert_serial(cert_file): salt '*' certutil.get_cert_serial ''' - cmd = "certutil.exe -verify {0}".format(cert_file) + cmd = "certutil.exe -silent -verify {0}".format(cert_file) out = __salt__['cmd.run'](cmd) - matches = re.search(r"Serial: (.*)", out) + # match serial number by paragraph to work with multiple languages + matches = re.search(r":\s*(\w*)\r\n\r\n", out) if matches is not None: return matches.groups()[0].strip() else: @@ -66,7 +67,8 @@ def get_stored_cert_serials(store): ''' cmd = "certutil.exe -store {0}".format(store) out = __salt__['cmd.run'](cmd) - matches = re.findall(r"Serial Number: (.*)\r", out) + # match serial numbers by header position to work with multiple languages + matches = re.findall(r"={16}\r\n.*:\s*(\w*)\r\n", out) return matches diff --git a/salt/modules/win_dacl.py b/salt/modules/win_dacl.py index a72da3be85d..e4421bd5e5a 100644 --- a/salt/modules/win_dacl.py +++ b/salt/modules/win_dacl.py @@ -16,8 +16,8 @@ import re # may also need to add the ability to take ownership of an object to set # permissions if the minion is running as a user and not LOCALSYSTEM -# Import salt libs -import salt.utils +# Import Salt libs +import salt.utils.platform from salt.exceptions import CommandExecutionError from salt.ext.six import string_types from salt.ext.six.moves import range # pylint: disable=redefined-builtin @@ -342,7 +342,7 @@ def __virtual__(): ''' Only works on Windows systems ''' - if salt.utils.is_windows() and HAS_WINDOWS_MODULES: + if salt.utils.platform.is_windows() and HAS_WINDOWS_MODULES: return __virtualname__ return (False, "Module win_dacl: module only works on Windows systems") diff --git a/salt/modules/win_disk.py b/salt/modules/win_disk.py index 52c95914bb7..278a3de90a2 100644 --- a/salt/modules/win_disk.py +++ b/salt/modules/win_disk.py @@ -6,13 +6,15 @@ Module for gathering disk information on Windows ''' from __future__ import absolute_import -# Import python libs +# Import Python libs import ctypes import string -# Import salt libs -import salt.ext.six as six -import salt.utils +# Import Salt libs +import salt.utils.platform + +# Import 3rd-party libs +from salt.ext import six try: import win32api @@ -33,7 +35,7 @@ def __virtual__(): ''' Only works on Windows systems ''' - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): return __virtualname__ return (False, "Module win_disk: module only works on Windows systems") diff --git a/salt/modules/win_dism.py b/salt/modules/win_dism.py index ef09af2e0a6..20fb33d1b81 100644 --- a/salt/modules/win_dism.py +++ b/salt/modules/win_dism.py @@ -7,12 +7,13 @@ Windows 10. ''' from __future__ import absolute_import -# Import python libs +# Import Python libs import re import logging -# Import salt libs -import salt.utils +# Import Salt libs +import salt.utils.platform +import salt.utils.versions log = logging.getLogger(__name__) __virtualname__ = "dism" @@ -22,7 +23,7 @@ def __virtual__(): ''' Only work on Windows ''' - if not salt.utils.is_windows(): + if not salt.utils.platform.is_windows(): return False, "Only available on Windows systems" return __virtualname__ @@ -73,7 +74,7 @@ def add_capability(capability, salt '*' dism.add_capability Tools.Graphics.DirectX~~~~0.0.1.0 ''' - if salt.utils.version_cmp(__grains__['osversion'], '10') == -1: + if salt.utils.versions.version_cmp(__grains__['osversion'], '10') == -1: raise NotImplementedError( '`install_capability` is not available on this version of Windows: ' '{0}'.format(__grains__['osversion'])) @@ -118,7 +119,7 @@ def remove_capability(capability, image=None, restart=False): salt '*' dism.remove_capability Tools.Graphics.DirectX~~~~0.0.1.0 ''' - if salt.utils.version_cmp(__grains__['osversion'], '10') == -1: + if salt.utils.versions.version_cmp(__grains__['osversion'], '10') == -1: raise NotImplementedError( '`uninstall_capability` is not available on this version of ' 'Windows: {0}'.format(__grains__['osversion'])) @@ -157,7 +158,7 @@ def get_capabilities(image=None): salt '*' dism.get_capabilities ''' - if salt.utils.version_cmp(__grains__['osversion'], '10') == -1: + if salt.utils.versions.version_cmp(__grains__['osversion'], '10') == -1: raise NotImplementedError( '`installed_capabilities` is not available on this version of ' 'Windows: {0}'.format(__grains__['osversion'])) @@ -197,7 +198,7 @@ def installed_capabilities(image=None): salt '*' dism.installed_capabilities ''' - if salt.utils.version_cmp(__grains__['osversion'], '10') == -1: + if salt.utils.versions.version_cmp(__grains__['osversion'], '10') == -1: raise NotImplementedError( '`installed_capabilities` is not available on this version of ' 'Windows: {0}'.format(__grains__['osversion'])) @@ -226,7 +227,7 @@ def available_capabilities(image=None): salt '*' dism.installed_capabilities ''' - if salt.utils.version_cmp(__grains__['osversion'], '10') == -1: + if salt.utils.versions.version_cmp(__grains__['osversion'], '10') == -1: raise NotImplementedError( '`installed_capabilities` is not available on this version of ' 'Windows: {0}'.format(__grains__['osversion'])) diff --git a/salt/modules/win_dns_client.py b/salt/modules/win_dns_client.py index 29e6997656c..e030a2ce6f7 100644 --- a/salt/modules/win_dns_client.py +++ b/salt/modules/win_dns_client.py @@ -4,11 +4,12 @@ Module for configuring DNS Client on Windows systems ''' from __future__ import absolute_import -# Import python libs +# Import Python libs import logging -# Import salt libs -import salt.utils +# Import Salt libs +import salt.utils.platform + try: import wmi except ImportError: @@ -21,7 +22,7 @@ def __virtual__(): ''' Only works on Windows systems ''' - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): return 'win_dns_client' return (False, "Module win_dns_client: module only works on Windows systems") diff --git a/salt/modules/win_dsc.py b/salt/modules/win_dsc.py index 90aef68630f..56a2b280527 100644 --- a/salt/modules/win_dsc.py +++ b/salt/modules/win_dsc.py @@ -17,13 +17,14 @@ the Minion. ''' from __future__ import absolute_import -# Import python libs +# Import Python libs import logging import json import os -# Import salt libs -import salt.utils +# Import Salt libs +import salt.utils.platform +import salt.utils.versions from salt.exceptions import CommandExecutionError, SaltInvocationError # Set up logging @@ -38,7 +39,7 @@ def __virtual__(): Set the system module of the kernel is Windows ''' # Verify Windows - if not salt.utils.is_windows(): + if not salt.utils.platform.is_windows(): log.debug('Module DSC: Only available on Windows systems') return False, 'Module DSC: Only available on Windows systems' @@ -49,7 +50,7 @@ def __virtual__(): return False, 'Module DSC: Requires PowerShell' # Verify PowerShell 5.0 or greater - if salt.utils.compare_versions(powershell_info['version'], '<', '5.0'): + if salt.utils.versions.compare(powershell_info['version'], '<', '5.0'): log.debug('Module DSC: Requires PowerShell 5 or later') return False, 'Module DSC: Requires PowerShell 5 or later' diff --git a/salt/modules/win_file.py b/salt/modules/win_file.py index 12d9319031a..26fa0a3c1a9 100644 --- a/salt/modules/win_file.py +++ b/salt/modules/win_file.py @@ -44,6 +44,7 @@ from salt.exceptions import CommandExecutionError, SaltInvocationError # Import salt libs import salt.utils import salt.utils.path +import salt.utils.platform from salt.modules.file import (check_hash, # pylint: disable=W0611 directory_exists, get_managed, check_managed, check_managed_changes, source_list, @@ -65,7 +66,7 @@ from salt.utils import namespaced_function as _namespaced_function HAS_WINDOWS_MODULES = False try: - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): import win32api import win32file import win32con @@ -83,7 +84,7 @@ except ImportError: HAS_WIN_DACL = False try: - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): import salt.utils.win_dacl HAS_WIN_DACL = True except ImportError: @@ -99,7 +100,7 @@ def __virtual__(): ''' Only works on Windows systems ''' - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): if HAS_WINDOWS_MODULES: # Load functions from file.py global get_managed, manage_file @@ -1687,9 +1688,6 @@ def check_perms(path, perms = changes[user]['perms'] try: - log.debug('*' * 68) - log.debug(perms) - log.debug('*' * 68) salt.utils.win_dacl.set_permissions( path, user, perms, 'deny', applies_to) ret['changes']['deny_perms'][user] = changes[user] diff --git a/salt/modules/win_firewall.py b/salt/modules/win_firewall.py index b7498b92f18..927ec231c09 100644 --- a/salt/modules/win_firewall.py +++ b/salt/modules/win_firewall.py @@ -4,11 +4,11 @@ Module for configuring Windows Firewall using ``netsh`` ''' from __future__ import absolute_import -# Import python libs +# Import Python libs import re -# Import salt libs -import salt.utils +# Import Salt libs +import salt.utils.platform from salt.exceptions import CommandExecutionError # Define the module's virtual name @@ -19,7 +19,7 @@ def __virtual__(): ''' Only works on Windows systems ''' - if not salt.utils.is_windows(): + if not salt.utils.platform.is_windows(): return False, "Module win_firewall: module only available on Windows" return __virtualname__ @@ -155,7 +155,7 @@ def get_rule(name='all'): salt '*' firewall.get_rule 'MyAppPort' ''' cmd = ['netsh', 'advfirewall', 'firewall', 'show', 'rule', - 'name="{0}"'.format(name)] + 'name={0}'.format(name)] ret = __salt__['cmd.run_all'](cmd, python_shell=False, ignore_retcode=True) if ret['retcode'] != 0: raise CommandExecutionError(ret['stdout']) @@ -245,7 +245,7 @@ def add_rule(name, localport, protocol='tcp', action='allow', dir='in', return True -def delete_rule(name, +def delete_rule(name=None, localport=None, protocol=None, dir=None, @@ -261,10 +261,11 @@ def delete_rule(name, name (str): The name of the rule to delete. If the name ``all`` is used you must specify additional parameters. - localport (Optional[str]): The port of the rule. Must specify a - protocol. + localport (Optional[str]): The port of the rule. If protocol is not + specified, protocol will be set to ``tcp`` - protocol (Optional[str]): The protocol of the rule. + protocol (Optional[str]): The protocol of the rule. Default is ``tcp`` + when ``localport`` is specified dir (Optional[str]): The direction of the rule. @@ -293,8 +294,9 @@ def delete_rule(name, # Delete a rule called 'allow80': salt '*' firewall.delete_rule allow80 ''' - cmd = ['netsh', 'advfirewall', 'firewall', 'delete', 'rule', - 'name={0}'.format(name)] + cmd = ['netsh', 'advfirewall', 'firewall', 'delete', 'rule'] + if name: + cmd.append('name={0}'.format(name)) if protocol: cmd.append('protocol={0}'.format(protocol)) if dir: @@ -305,6 +307,8 @@ def delete_rule(name, if protocol is None \ or ('icmpv4' not in protocol and 'icmpv6' not in protocol): if localport: + if not protocol: + cmd.append('protocol=tcp') cmd.append('localport={0}'.format(localport)) ret = __salt__['cmd.run_all'](cmd, python_shell=False, ignore_retcode=True) diff --git a/salt/modules/win_groupadd.py b/salt/modules/win_groupadd.py index d466380d703..954f153d2bb 100644 --- a/salt/modules/win_groupadd.py +++ b/salt/modules/win_groupadd.py @@ -10,8 +10,9 @@ Manage groups on Windows ''' from __future__ import absolute_import -# Import salt libs -import salt.utils +# Import Salt libs +import salt.utils.platform +import salt.utils.win_functions try: @@ -30,15 +31,23 @@ def __virtual__(): ''' Set the group module if the kernel is Windows ''' - if salt.utils.is_windows() and HAS_DEPENDENCIES: + if salt.utils.platform.is_windows() and HAS_DEPENDENCIES: return __virtualname__ return (False, "Module win_groupadd: module only works on Windows systems") -def add(name, gid=None, system=False): +def add(name, **kwargs): ''' Add the specified group + Args: + + name (str): + The name of the group to add + + Returns: + dict: A dictionary of results + CLI Example: .. code-block:: bash @@ -57,29 +66,32 @@ def add(name, gid=None, system=False): compObj = nt.GetObject('', 'WinNT://.,computer') newGroup = compObj.Create('group', name) newGroup.SetInfo() - ret['changes'].append(( - 'Successfully created group {0}' - ).format(name)) + ret['changes'].append('Successfully created group {0}'.format(name)) except pywintypes.com_error as com_err: ret['result'] = False if len(com_err.excepinfo) >= 2: friendly_error = com_err.excepinfo[2].rstrip('\r\n') - ret['comment'] = ( - 'Failed to create group {0}. {1}' - ).format(name, friendly_error) + ret['comment'] = 'Failed to create group {0}. {1}' \ + ''.format(name, friendly_error) else: ret['result'] = None - ret['comment'] = ( - 'The group {0} already exists.' - ).format(name) + ret['comment'] = 'The group {0} already exists.'.format(name) return ret -def delete(name): +def delete(name, **kwargs): ''' Remove the named group + Args: + + name (str): + The name of the group to remove + + Returns: + dict: A dictionary of results + CLI Example: .. code-block:: bash @@ -118,6 +130,14 @@ def info(name): ''' Return information about a group + Args: + + name (str): + The name of the group for which to get information + + Returns: + dict: A dictionary of information about the group + CLI Example: .. code-block:: bash @@ -151,6 +171,17 @@ def getent(refresh=False): ''' Return info on all groups + Args: + + refresh (bool): + Refresh the info for all groups in ``__context__``. If False only + the groups in ``__context__`` wil be returned. If True the + ``__context__`` will be refreshed with current data and returned. + Default is False + + Returns: + A list of groups and their information + CLI Example: .. code-block:: bash @@ -182,16 +213,26 @@ def getent(refresh=False): return ret -def adduser(name, username): +def adduser(name, username, **kwargs): ''' - add a user to a group + Add a user to a group + + Args: + + name (str): + The name of the group to modify + + username (str): + The name of the user to add to the group + + Returns: + dict: A dictionary of results CLI Example: .. code-block:: bash salt '*' group.adduser foo username - ''' ret = {'name': name, @@ -209,7 +250,7 @@ def adduser(name, username): '/', '\\').encode('ascii', 'backslashreplace').lower()) try: - if __fixlocaluser(username.lower()) not in existingMembers: + if salt.utils.win_functions.get_sam_name(username) not in existingMembers: if not __opts__['test']: groupObj.Add('WinNT://' + username.replace('\\', '/')) @@ -231,16 +272,26 @@ def adduser(name, username): return ret -def deluser(name, username): +def deluser(name, username, **kwargs): ''' - remove a user from a group + Remove a user from a group + + Args: + + name (str): + The name of the group to modify + + username (str): + The name of the user to remove from the group + + Returns: + dict: A dictionary of results CLI Example: .. code-block:: bash salt '*' group.deluser foo username - ''' ret = {'name': name, @@ -258,7 +309,7 @@ def deluser(name, username): '/', '\\').encode('ascii', 'backslashreplace').lower()) try: - if __fixlocaluser(username.lower()) in existingMembers: + if salt.utils.win_functions.get_sam_name(username) in existingMembers: if not __opts__['test']: groupObj.Remove('WinNT://' + username.replace('\\', '/')) @@ -280,16 +331,27 @@ def deluser(name, username): return ret -def members(name, members_list): +def members(name, members_list, **kwargs): ''' - remove a user from a group + Ensure a group contains only the members in the list + + Args: + + name (str): + The name of the group to modify + + members_list (str): + A single user or a comma separated list of users. The group will + contain only the users specified in this list. + + Returns: + dict: A dictionary of results CLI Example: .. code-block:: bash salt '*' group.members foo 'user1,user2,user3' - ''' ret = {'name': name, @@ -297,7 +359,7 @@ def members(name, members_list): 'changes': {'Users Added': [], 'Users Removed': []}, 'comment': []} - members_list = [__fixlocaluser(thisMember) for thisMember in members_list.lower().split(",")] + members_list = [salt.utils.win_functions.get_sam_name(m) for m in members_list.split(",")] if not isinstance(members_list, list): ret['result'] = False ret['comment'].append('Members is not a list object') @@ -364,27 +426,26 @@ def members(name, members_list): return ret -def __fixlocaluser(username): - ''' - prefixes a username w/o a backslash with the computername - - i.e. __fixlocaluser('Administrator') would return 'computername\administrator' - ''' - if '\\' not in username: - username = ('{0}\\{1}').format(__salt__['grains.get']('host'), username) - - return username.lower() - - def list_groups(refresh=False): ''' Return a list of groups + Args: + + refresh (bool): + Refresh the info for all groups in ``__context__``. If False only + the groups in ``__context__`` wil be returned. If True, the + ``__context__`` will be refreshed with current data and returned. + Default is False + + Returns: + list: A list of groups on the machine + CLI Example: .. code-block:: bash - salt '*' group.getent + salt '*' group.list_groups ''' if 'group.list_groups' in __context__ and not refresh: return __context__['group.getent'] diff --git a/salt/modules/win_iis.py b/salt/modules/win_iis.py index d0fe647b03e..e171827316f 100644 --- a/salt/modules/win_iis.py +++ b/salt/modules/win_iis.py @@ -18,10 +18,10 @@ import os import decimal # Import salt libs +import salt.utils.platform from salt.ext.six.moves import range from salt.exceptions import SaltInvocationError, CommandExecutionError from salt.ext import six -import salt.utils log = logging.getLogger(__name__) @@ -38,7 +38,7 @@ def __virtual__(): Load only on Windows Requires PowerShell and the WebAdministration module ''' - if not salt.utils.is_windows(): + if not salt.utils.platform.is_windows(): return False, 'Only available on Windows systems' powershell_info = __salt__['cmd.shell_info']('powershell', True) @@ -837,6 +837,9 @@ def create_cert_binding(name, site, hostheader='', ipaddress='*', port=443, # IIS 7.5 and earlier have different syntax for associating a certificate with a site # Modify IP spec to IIS 7.5 format iis7path = binding_path.replace(r"\*!", "\\0.0.0.0!") + # win 2008 uses the following format: ip!port and not ip!port! + if iis7path.endswith("!"): + iis7path = iis7path[:-1] ps_cmd = ['New-Item', '-Path', "'{0}'".format(iis7path), @@ -856,7 +859,7 @@ def create_cert_binding(name, site, hostheader='', ipaddress='*', port=443, new_cert_bindings = list_cert_bindings(site) - if binding_info not in new_cert_bindings(site): + if binding_info not in new_cert_bindings: log.error('Binding not present: {0}'.format(binding_info)) return False @@ -1255,6 +1258,9 @@ def set_container_setting(name, container, settings): salt '*' win_iis.set_container_setting name='MyTestPool' container='AppPools' settings="{'managedPipeLineMode': 'Integrated'}" ''' + + identityType_map2string = {'0': 'LocalSystem', '1': 'LocalService', '2': 'NetworkService', '3': 'SpecificUser', '4': 'ApplicationPoolIdentity'} + identityType_map2numeric = {'LocalSystem': '0', 'LocalService': '1', 'NetworkService': '2', 'SpecificUser': '3', 'ApplicationPoolIdentity': '4'} ps_cmd = list() container_path = r"IIS:\{0}\{1}".format(container, name) @@ -1281,6 +1287,10 @@ def set_container_setting(name, container, settings): except ValueError: value = "'{0}'".format(settings[setting]) + # Map to numeric to support server 2008 + if setting == 'processModel.identityType' and settings[setting] in identityType_map2numeric.keys(): + value = identityType_map2numeric[settings[setting]] + ps_cmd.extend(['Set-ItemProperty', '-Path', "'{0}'".format(container_path), '-Name', "'{0}'".format(setting), @@ -1300,6 +1310,10 @@ def set_container_setting(name, container, settings): failed_settings = dict() for setting in settings: + # map identity type from numeric to string for comparing + if setting == 'processModel.identityType' and settings[setting] in identityType_map2string.keys(): + settings[setting] = identityType_map2string[settings[setting]] + if str(settings[setting]) != str(new_settings[setting]): failed_settings[setting] = settings[setting] diff --git a/salt/modules/win_ip.py b/salt/modules/win_ip.py index dc7416b78eb..7f818e56e5a 100644 --- a/salt/modules/win_ip.py +++ b/salt/modules/win_ip.py @@ -4,13 +4,13 @@ The networking module for Windows based systems ''' from __future__ import absolute_import -# Import python libs +# Import Python libs import logging import time -# Import salt libs -import salt.utils +# Import Salt libs import salt.utils.network +import salt.utils.platform import salt.utils.validate.net from salt.exceptions import ( CommandExecutionError, @@ -29,7 +29,7 @@ def __virtual__(): ''' Confine this module to Windows systems ''' - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): return __virtualname__ return (False, "Module win_ip: module only works on Windows systems") diff --git a/salt/modules/win_lgpo.py b/salt/modules/win_lgpo.py index 6bf5d437d8f..9c970d19d39 100644 --- a/salt/modules/win_lgpo.py +++ b/salt/modules/win_lgpo.py @@ -38,21 +38,20 @@ Current known limitations - struct - salt.modules.reg ''' - -# Import python libs +# Import Python libs from __future__ import absolute_import import os import logging import re -# Import salt libs -import salt.utils -from salt.exceptions import CommandExecutionError -from salt.exceptions import SaltInvocationError +# Import Salt libs +import salt.utils.files +import salt.utils.platform import salt.utils.dictupdate as dictupdate +from salt.exceptions import CommandExecutionError, SaltInvocationError # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import range log = logging.getLogger(__name__) @@ -2649,7 +2648,7 @@ def __virtual__(): ''' Only works on Windows systems ''' - if salt.utils.is_windows() and HAS_WINDOWS_MODULES: + if salt.utils.platform.is_windows() and HAS_WINDOWS_MODULES: return __virtualname__ return False @@ -2696,7 +2695,7 @@ def _remove_unicode_encoding(xml_file): as lxml does not support that on a windows node currently see issue #38100 ''' - with salt.utils.fopen(xml_file, 'rb') as f: + with salt.utils.files.fopen(xml_file, 'rb') as f: xml_content = f.read() modified_xml = re.sub(r' encoding=[\'"]+unicode[\'"]+', '', xml_content.decode('utf-16'), count=1) xmltree = lxml.etree.parse(six.StringIO(modified_xml)) @@ -2835,7 +2834,8 @@ def _findOptionValueInSeceditFile(option): _reader = codecs.open(_tfile, 'r', encoding='utf-16') _secdata = _reader.readlines() _reader.close() - _ret = __salt__['file.remove'](_tfile) + if __salt__['file.file_exists'](_tfile): + _ret = __salt__['file.remove'](_tfile) for _line in _secdata: if _line.startswith(option): return True, _line.split('=')[1].strip() @@ -2856,16 +2856,20 @@ def _importSeceditConfig(infdata): _tInfFile = '{0}\\{1}'.format(__salt__['config.get']('cachedir'), 'salt-secedit-config-{0}.inf'.format(_d)) # make sure our temp files don't already exist - _ret = __salt__['file.remove'](_tSdbfile) - _ret = __salt__['file.remove'](_tInfFile) + if __salt__['file.file_exists'](_tSdbfile): + _ret = __salt__['file.remove'](_tSdbfile) + if __salt__['file.file_exists'](_tInfFile): + _ret = __salt__['file.remove'](_tInfFile) # add the inf data to the file, win_file sure could use the write() function _ret = __salt__['file.touch'](_tInfFile) _ret = __salt__['file.append'](_tInfFile, infdata) # run secedit to make the change _ret = __salt__['cmd.run']('secedit /configure /db {0} /cfg {1}'.format(_tSdbfile, _tInfFile)) # cleanup our temp files - _ret = __salt__['file.remove'](_tSdbfile) - _ret = __salt__['file.remove'](_tInfFile) + if __salt__['file.file_exists'](_tSdbfile): + _ret = __salt__['file.remove'](_tSdbfile) + if __salt__['file.file_exists'](_tInfFile): + _ret = __salt__['file.remove'](_tInfFile) return True except Exception as e: log.debug('error occurred while trying to import secedit data') @@ -3503,22 +3507,31 @@ def _checkAllAdmxPolicies(policy_class, not_configured_policies.remove(policy_item) for not_configured_policy in not_configured_policies: - policy_vals[not_configured_policy.attrib['name']] = 'Not Configured' + not_configured_policy_namespace = not_configured_policy.nsmap[not_configured_policy.prefix] + if not_configured_policy_namespace not in policy_vals: + policy_vals[not_configured_policy_namespace] = {} + policy_vals[not_configured_policy_namespace][not_configured_policy.attrib['name']] = 'Not Configured' if return_full_policy_names: - full_names[not_configured_policy.attrib['name']] = _getFullPolicyName( + if not_configured_policy_namespace not in full_names: + full_names[not_configured_policy_namespace] = {} + full_names[not_configured_policy_namespace][not_configured_policy.attrib['name']] = _getFullPolicyName( not_configured_policy, not_configured_policy.attrib['name'], return_full_policy_names, adml_policy_resources) log.debug('building hierarchy for non-configured item {0}'.format(not_configured_policy.attrib['name'])) - hierarchy[not_configured_policy.attrib['name']] = _build_parent_list(not_configured_policy, - admx_policy_definitions, - return_full_policy_names, - adml_policy_resources) + if not_configured_policy_namespace not in hierarchy: + hierarchy[not_configured_policy_namespace] = {} + hierarchy[not_configured_policy_namespace][not_configured_policy.attrib['name']] = _build_parent_list( + not_configured_policy, + admx_policy_definitions, + return_full_policy_names, + adml_policy_resources) for admx_policy in admx_policies: this_key = None this_valuename = None this_policyname = None + this_policynamespace = None this_policy_setting = 'Not Configured' element_only_enabled_disabled = True explicit_enable_disable_value_setting = False @@ -3537,6 +3550,7 @@ def _checkAllAdmxPolicies(policy_class, log.error('policy item {0} does not have the required "name" ' 'attribute'.format(admx_policy.attrib)) break + this_policynamespace = admx_policy.nsmap[admx_policy.prefix] if ENABLED_VALUE_XPATH(admx_policy) and this_policy_setting == 'Not Configured': element_only_enabled_disabled = False explicit_enable_disable_value_setting = True @@ -3548,7 +3562,9 @@ def _checkAllAdmxPolicies(policy_class, policy_filedata): this_policy_setting = 'Enabled' log.debug('{0} is enabled'.format(this_policyname)) - policy_vals[this_policyname] = this_policy_setting + if this_policynamespace not in policy_vals: + policy_vals[this_policynamespace] = {} + policy_vals[this_policynamespace][this_policyname] = this_policy_setting if DISABLED_VALUE_XPATH(admx_policy) and this_policy_setting == 'Not Configured': element_only_enabled_disabled = False explicit_enable_disable_value_setting = True @@ -3560,21 +3576,27 @@ def _checkAllAdmxPolicies(policy_class, policy_filedata): this_policy_setting = 'Disabled' log.debug('{0} is disabled'.format(this_policyname)) - policy_vals[this_policyname] = this_policy_setting + if this_policynamespace not in policy_vals: + policy_vals[this_policynamespace] = {} + policy_vals[this_policynamespace][this_policyname] = this_policy_setting if ENABLED_LIST_XPATH(admx_policy) and this_policy_setting == 'Not Configured': element_only_enabled_disabled = False explicit_enable_disable_value_setting = True if _checkListItem(admx_policy, this_policyname, this_key, ENABLED_LIST_XPATH, policy_filedata): this_policy_setting = 'Enabled' log.debug('{0} is enabled'.format(this_policyname)) - policy_vals[this_policyname] = this_policy_setting + if this_policynamespace not in policy_vals: + policy_vals[this_policynamespace] = {} + policy_vals[this_policynamespace][this_policyname] = this_policy_setting if DISABLED_LIST_XPATH(admx_policy) and this_policy_setting == 'Not Configured': element_only_enabled_disabled = False explicit_enable_disable_value_setting = True if _checkListItem(admx_policy, this_policyname, this_key, DISABLED_LIST_XPATH, policy_filedata): this_policy_setting = 'Disabled' log.debug('{0} is disabled'.format(this_policyname)) - policy_vals[this_policyname] = this_policy_setting + if this_policynamespace not in policy_vals: + policy_vals[this_policynamespace] = {} + policy_vals[this_policynamespace][this_policyname] = this_policy_setting if not explicit_enable_disable_value_setting and this_valuename: # the policy has a key/valuename but no explicit enabled/Disabled @@ -3587,7 +3609,9 @@ def _checkAllAdmxPolicies(policy_class, policy_filedata): this_policy_setting = 'Enabled' log.debug('{0} is enabled'.format(this_policyname)) - policy_vals[this_policyname] = this_policy_setting + if this_policynamespace not in policy_vals: + policy_vals[this_policynamespace] = {} + policy_vals[this_policynamespace][this_policyname] = this_policy_setting elif _regexSearchRegPolData(re.escape(_buildKnownDataSearchString(this_key, this_valuename, 'REG_DWORD', @@ -3596,7 +3620,9 @@ def _checkAllAdmxPolicies(policy_class, policy_filedata): this_policy_setting = 'Disabled' log.debug('{0} is disabled'.format(this_policyname)) - policy_vals[this_policyname] = this_policy_setting + if this_policynamespace not in policy_vals: + policy_vals[this_policynamespace] = {} + policy_vals[this_policynamespace][this_policyname] = this_policy_setting if ELEMENTS_XPATH(admx_policy): if element_only_enabled_disabled or this_policy_setting == 'Enabled': @@ -3794,65 +3820,84 @@ def _checkAllAdmxPolicies(policy_class, and len(configured_elements.keys()) == len(required_elements.keys()): if policy_disabled_elements == len(required_elements.keys()): log.debug('{0} is disabled by all enum elements'.format(this_policyname)) - policy_vals[this_policyname] = 'Disabled' + if this_policynamespace not in policy_vals: + policy_vals[this_policynamespace] = {} + policy_vals[this_policynamespace][this_policyname] = 'Disabled' else: - policy_vals[this_policyname] = configured_elements + if this_policynamespace not in policy_vals: + policy_vals[this_policynamespace] = {} + policy_vals[this_policynamespace][this_policyname] = configured_elements log.debug('{0} is enabled by enum elements'.format(this_policyname)) else: if this_policy_setting == 'Enabled': - policy_vals[this_policyname] = configured_elements - if return_full_policy_names and this_policyname in policy_vals: - full_names[this_policyname] = _getFullPolicyName( + if this_policynamespace not in policy_vals: + policy_vals[this_policynamespace] = {} + policy_vals[this_policynamespace][this_policyname] = configured_elements + if return_full_policy_names and this_policynamespace in policy_vals and this_policyname in policy_vals[this_policynamespace]: + if this_policynamespace not in full_names: + full_names[this_policynamespace] = {} + full_names[this_policynamespace][this_policyname] = _getFullPolicyName( admx_policy, admx_policy.attrib['name'], return_full_policy_names, adml_policy_resources) - if this_policyname in policy_vals: - hierarchy[this_policyname] = _build_parent_list(admx_policy, + if this_policynamespace in policy_vals and this_policyname in policy_vals[this_policynamespace]: + if this_policynamespace not in hierarchy: + hierarchy[this_policynamespace] = {} + hierarchy[this_policynamespace][this_policyname] = _build_parent_list(admx_policy, admx_policy_definitions, return_full_policy_names, adml_policy_resources) if policy_vals and return_full_policy_names and not hierarchical_return: unpathed_dict = {} pathed_dict = {} - for policy_item in list(policy_vals): - if full_names[policy_item] in policy_vals: - # add this item with the path'd full name - full_path_list = hierarchy[policy_item] + for policy_namespace in list(policy_vals): + for policy_item in list(policy_vals[policy_namespace]): + if full_names[policy_namespace][policy_item] in policy_vals[policy_namespace]: + # add this item with the path'd full name + full_path_list = hierarchy[policy_namespace][policy_item] + full_path_list.reverse() + full_path_list.append(full_names[policy_namespace][policy_item]) + policy_vals['\\'.join(full_path_list)] = policy_vals[policy_namespace].pop(policy_item) + pathed_dict[full_names[policy_namespace][policy_item]] = True + else: + policy_vals[policy_namespace][full_names[policy_namespace][policy_item]] = policy_vals[policy_namespace].pop(policy_item) + if policy_namespace not in unpathed_dict: + unpathed_dict[policy_namespace] = {} + unpathed_dict[policy_namespace][full_names[policy_namespace][policy_item]] = policy_item + # go back and remove any "unpathed" policies that need a full path + for path_needed in unpathed_dict[policy_namespace]: + # remove the item with the same full name and re-add it w/a path'd version + full_path_list = hierarchy[policy_namespace][unpathed_dict[policy_namespace][path_needed]] full_path_list.reverse() - full_path_list.append(full_names[policy_item]) - policy_vals['\\'.join(full_path_list)] = policy_vals.pop(policy_item) - pathed_dict[full_names[policy_item]] = True - else: - policy_vals[full_names[policy_item]] = policy_vals.pop(policy_item) - unpathed_dict[full_names[policy_item]] = policy_item - # go back and remove any "unpathed" policies that need a full path - for path_needed in unpathed_dict: - # remove the item with the same full name and re-add it w/a path'd version - full_path_list = hierarchy[unpathed_dict[path_needed]] - full_path_list.reverse() - full_path_list.append(path_needed) - log.debug('full_path_list == {0}'.format(full_path_list)) - policy_vals['\\'.join(full_path_list)] = policy_vals.pop(path_needed) + full_path_list.append(path_needed) + log.debug('full_path_list == {0}'.format(full_path_list)) + policy_vals['\\'.join(full_path_list)] = policy_vals[policy_namespace].pop(path_needed) + for policy_namespace in list(policy_vals): + if policy_vals[policy_namespace] == {}: + policy_vals.pop(policy_namespace) if policy_vals and hierarchical_return: if hierarchy: - for hierarchy_item in hierarchy: - if hierarchy_item in policy_vals: - tdict = {} - first_item = True - for item in hierarchy[hierarchy_item]: - newdict = {} - if first_item: - h_policy_name = hierarchy_item - if return_full_policy_names: - h_policy_name = full_names[hierarchy_item] - newdict[item] = {h_policy_name: policy_vals.pop(hierarchy_item)} - first_item = False - else: - newdict[item] = tdict - tdict = newdict - if tdict: - policy_vals = dictupdate.update(policy_vals, tdict) + for policy_namespace in hierarchy: + for hierarchy_item in hierarchy[policy_namespace]: + if hierarchy_item in policy_vals[policy_namespace]: + tdict = {} + first_item = True + for item in hierarchy[policy_namespace][hierarchy_item]: + newdict = {} + if first_item: + h_policy_name = hierarchy_item + if return_full_policy_names: + h_policy_name = full_names[policy_namespace][hierarchy_item] + newdict[item] = {h_policy_name: policy_vals[policy_namespace].pop(hierarchy_item)} + first_item = False + else: + newdict[item] = tdict + tdict = newdict + if tdict: + policy_vals = dictupdate.update(policy_vals, tdict) + if policy_namespace in policy_vals and policy_vals[policy_namespace] == {}: + policy_vals.pop(policy_namespace) policy_vals = { module_policy_data.admx_registry_classes[policy_class]['lgpo_section']: { 'Administrative Templates': policy_vals @@ -3938,7 +3983,7 @@ def _read_regpol_file(reg_pol_path): ''' returndata = None if os.path.exists(reg_pol_path): - with salt.utils.fopen(reg_pol_path, 'rb') as pol_file: + with salt.utils.files.fopen(reg_pol_path, 'rb') as pol_file: returndata = pol_file.read() returndata = returndata.decode('utf-16-le') return returndata @@ -3982,78 +4027,77 @@ def _write_regpol_data(data_to_write, gpt_extension_guid: admx registry extension guid for the class ''' try: - if data_to_write: - reg_pol_header = u'\u5250\u6765\x01\x00' - if not os.path.exists(policy_file_path): - ret = __salt__['file.makedirs'](policy_file_path) - with salt.utils.fopen(policy_file_path, 'wb') as pol_file: - if not data_to_write.startswith(reg_pol_header): - pol_file.write(reg_pol_header.encode('utf-16-le')) - pol_file.write(data_to_write.encode('utf-16-le')) - try: - gpt_ini_data = '' - if os.path.exists(gpt_ini_path): - with salt.utils.fopen(gpt_ini_path, 'rb') as gpt_file: - gpt_ini_data = gpt_file.read() - if not _regexSearchRegPolData(r'\[General\]\r\n', gpt_ini_data): - gpt_ini_data = '[General]\r\n' + gpt_ini_data - if _regexSearchRegPolData(r'{0}='.format(re.escape(gpt_extension)), gpt_ini_data): - # ensure the line contains the ADM guid - gpt_ext_loc = re.search(r'^{0}=.*\r\n'.format(re.escape(gpt_extension)), - gpt_ini_data, - re.IGNORECASE | re.MULTILINE) - gpt_ext_str = gpt_ini_data[gpt_ext_loc.start():gpt_ext_loc.end()] - if not _regexSearchRegPolData(r'{0}'.format(re.escape(gpt_extension_guid)), - gpt_ext_str): - gpt_ext_str = gpt_ext_str.split('=') - gpt_ext_str[1] = gpt_extension_guid + gpt_ext_str[1] - gpt_ext_str = '='.join(gpt_ext_str) - gpt_ini_data = gpt_ini_data[0:gpt_ext_loc.start()] + gpt_ext_str + gpt_ini_data[gpt_ext_loc.end():] - else: - general_location = re.search(r'^\[General\]\r\n', - gpt_ini_data, - re.IGNORECASE | re.MULTILINE) - gpt_ini_data = "{0}{1}={2}\r\n{3}".format( - gpt_ini_data[general_location.start():general_location.end()], - gpt_extension, gpt_extension_guid, - gpt_ini_data[general_location.end():]) - # https://technet.microsoft.com/en-us/library/cc978247.aspx - if _regexSearchRegPolData(r'Version=', gpt_ini_data): - version_loc = re.search(r'^Version=.*\r\n', - gpt_ini_data, - re.IGNORECASE | re.MULTILINE) - version_str = gpt_ini_data[version_loc.start():version_loc.end()] - version_str = version_str.split('=') - version_nums = struct.unpack('>2H', struct.pack('>I', int(version_str[1]))) - if gpt_extension.lower() == 'gPCMachineExtensionNames'.lower(): - version_nums = (version_nums[0], version_nums[1] + 1) - elif gpt_extension.lower() == 'gPCUserExtensionNames'.lower(): - version_nums = (version_nums[0] + 1, version_nums[1]) - version_num = struct.unpack('>I', struct.pack('>2H', *version_nums))[0] - gpt_ini_data = "{0}{1}={2}\r\n{3}".format( - gpt_ini_data[0:version_loc.start()], - 'Version', version_num, - gpt_ini_data[version_loc.end():]) - else: - general_location = re.search(r'^\[General\]\r\n', - gpt_ini_data, - re.IGNORECASE | re.MULTILINE) - if gpt_extension.lower() == 'gPCMachineExtensionNames'.lower(): - version_nums = (0, 1) - elif gpt_extension.lower() == 'gPCUserExtensionNames'.lower(): - version_nums = (1, 0) - gpt_ini_data = "{0}{1}={2}\r\n{3}".format( - gpt_ini_data[general_location.start():general_location.end()], - 'Version', - int("{0}{1}".format(str(version_nums[0]).zfill(4), str(version_nums[1]).zfill(4)), 16), - gpt_ini_data[general_location.end():]) - if gpt_ini_data: - with salt.utils.fopen(gpt_ini_path, 'wb') as gpt_file: - gpt_file.write(gpt_ini_data) - except Exception as e: - msg = 'An error occurred attempting to write to {0}, the exception was {1}'.format( - gpt_ini_path, e) - raise CommandExecutionError(msg) + reg_pol_header = u'\u5250\u6765\x01\x00' + if not os.path.exists(policy_file_path): + ret = __salt__['file.makedirs'](policy_file_path) + with salt.utils.files.fopen(policy_file_path, 'wb') as pol_file: + if not data_to_write.startswith(reg_pol_header): + pol_file.write(reg_pol_header.encode('utf-16-le')) + pol_file.write(data_to_write.encode('utf-16-le')) + try: + gpt_ini_data = '' + if os.path.exists(gpt_ini_path): + with salt.utils.files.fopen(gpt_ini_path, 'rb') as gpt_file: + gpt_ini_data = gpt_file.read() + if not _regexSearchRegPolData(r'\[General\]\r\n', gpt_ini_data): + gpt_ini_data = '[General]\r\n' + gpt_ini_data + if _regexSearchRegPolData(r'{0}='.format(re.escape(gpt_extension)), gpt_ini_data): + # ensure the line contains the ADM guid + gpt_ext_loc = re.search(r'^{0}=.*\r\n'.format(re.escape(gpt_extension)), + gpt_ini_data, + re.IGNORECASE | re.MULTILINE) + gpt_ext_str = gpt_ini_data[gpt_ext_loc.start():gpt_ext_loc.end()] + if not _regexSearchRegPolData(r'{0}'.format(re.escape(gpt_extension_guid)), + gpt_ext_str): + gpt_ext_str = gpt_ext_str.split('=') + gpt_ext_str[1] = gpt_extension_guid + gpt_ext_str[1] + gpt_ext_str = '='.join(gpt_ext_str) + gpt_ini_data = gpt_ini_data[0:gpt_ext_loc.start()] + gpt_ext_str + gpt_ini_data[gpt_ext_loc.end():] + else: + general_location = re.search(r'^\[General\]\r\n', + gpt_ini_data, + re.IGNORECASE | re.MULTILINE) + gpt_ini_data = "{0}{1}={2}\r\n{3}".format( + gpt_ini_data[general_location.start():general_location.end()], + gpt_extension, gpt_extension_guid, + gpt_ini_data[general_location.end():]) + # https://technet.microsoft.com/en-us/library/cc978247.aspx + if _regexSearchRegPolData(r'Version=', gpt_ini_data): + version_loc = re.search(r'^Version=.*\r\n', + gpt_ini_data, + re.IGNORECASE | re.MULTILINE) + version_str = gpt_ini_data[version_loc.start():version_loc.end()] + version_str = version_str.split('=') + version_nums = struct.unpack('>2H', struct.pack('>I', int(version_str[1]))) + if gpt_extension.lower() == 'gPCMachineExtensionNames'.lower(): + version_nums = (version_nums[0], version_nums[1] + 1) + elif gpt_extension.lower() == 'gPCUserExtensionNames'.lower(): + version_nums = (version_nums[0] + 1, version_nums[1]) + version_num = struct.unpack('>I', struct.pack('>2H', *version_nums))[0] + gpt_ini_data = "{0}{1}={2}\r\n{3}".format( + gpt_ini_data[0:version_loc.start()], + 'Version', version_num, + gpt_ini_data[version_loc.end():]) + else: + general_location = re.search(r'^\[General\]\r\n', + gpt_ini_data, + re.IGNORECASE | re.MULTILINE) + if gpt_extension.lower() == 'gPCMachineExtensionNames'.lower(): + version_nums = (0, 1) + elif gpt_extension.lower() == 'gPCUserExtensionNames'.lower(): + version_nums = (1, 0) + gpt_ini_data = "{0}{1}={2}\r\n{3}".format( + gpt_ini_data[general_location.start():general_location.end()], + 'Version', + int("{0}{1}".format(str(version_nums[0]).zfill(4), str(version_nums[1]).zfill(4)), 16), + gpt_ini_data[general_location.end():]) + if gpt_ini_data: + with salt.utils.files.fopen(gpt_ini_path, 'wb') as gpt_file: + gpt_file.write(gpt_ini_data) + except Exception as e: + msg = 'An error occurred attempting to write to {0}, the exception was {1}'.format( + gpt_ini_path, e) + raise CommandExecutionError(msg) except Exception as e: msg = 'An error occurred attempting to write to {0}, the exception was {1}'.format(policy_file_path, e) raise CommandExecutionError(msg) @@ -4117,6 +4161,7 @@ def _policyFileReplaceOrAppend(this_string, policy_data, append_only=False): def _writeAdminTemplateRegPolFile(admtemplate_data, + admtemplate_namespace_data, admx_policy_definitions=None, adml_policy_resources=None, display_language='en-US', @@ -4133,7 +4178,7 @@ def _writeAdminTemplateRegPolFile(admtemplate_data, existing_data = '' base_policy_settings = {} policy_data = _policy_info() - policySearchXpath = etree.XPath('//*[@*[local-name() = "id"] = $id or @*[local-name() = "name"] = $id]') + policySearchXpath = '//ns1:*[@id = "{0}" or @name = "{0}"]' try: if admx_policy_definitions is None or adml_policy_resources is None: admx_policy_definitions, adml_policy_resources = _processPolicyDefinitions( @@ -4145,298 +4190,303 @@ def _writeAdminTemplateRegPolFile(admtemplate_data, hierarchical_return=False, return_not_configured=False) log.debug('preparing to loop through policies requested to be configured') - for adm_policy in admtemplate_data: - if str(admtemplate_data[adm_policy]).lower() == 'not configured': - if adm_policy in base_policy_settings: - base_policy_settings.pop(adm_policy) - else: - log.debug('adding {0} to base_policy_settings'.format(adm_policy)) - base_policy_settings[adm_policy] = admtemplate_data[adm_policy] - for admPolicy in base_policy_settings: - log.debug('working on admPolicy {0}'.format(admPolicy)) - explicit_enable_disable_value_setting = False - this_key = None - this_valuename = None - if str(base_policy_settings[admPolicy]).lower() == 'disabled': - log.debug('time to disable {0}'.format(admPolicy)) - this_policy = policySearchXpath(admx_policy_definitions, id=admPolicy) - if this_policy: - this_policy = this_policy[0] - if 'class' in this_policy.attrib: - if this_policy.attrib['class'] == registry_class or this_policy.attrib['class'] == 'Both': - if 'key' in this_policy.attrib: - this_key = this_policy.attrib['key'] - else: - msg = 'policy item {0} does not have the required "key" attribute' - log.error(msg.format(this_policy.attrib)) - break - if 'valueName' in this_policy.attrib: - this_valuename = this_policy.attrib['valueName'] - if DISABLED_VALUE_XPATH(this_policy): - # set the disabled value in the registry.pol file - explicit_enable_disable_value_setting = True - disabled_value_string = _checkValueItemParent(this_policy, - admPolicy, - this_key, - this_valuename, - DISABLED_VALUE_XPATH, - None, - check_deleted=False, - test_item=False) - existing_data = _policyFileReplaceOrAppend(disabled_value_string, - existing_data) - if DISABLED_LIST_XPATH(this_policy): - explicit_enable_disable_value_setting = True - disabled_list_strings = _checkListItem(this_policy, - admPolicy, - this_key, - DISABLED_LIST_XPATH, - None, - test_items=False) - log.debug('working with disabledList portion of {0}'.format(admPolicy)) - existing_data = _policyFileReplaceOrAppendList(disabled_list_strings, + for adm_namespace in admtemplate_data: + for adm_policy in admtemplate_data[adm_namespace]: + if str(admtemplate_data[adm_namespace][adm_policy]).lower() == 'not configured': + if adm_policy in base_policy_settings[adm_namespace]: + base_policy_settings[adm_namespace].pop(adm_policy) + else: + log.debug('adding {0} to base_policy_settings'.format(adm_policy)) + if adm_namespace not in base_policy_settings: + base_policy_settings[adm_namespace] = {} + base_policy_settings[adm_namespace][adm_policy] = admtemplate_data[adm_namespace][adm_policy] + for adm_namespace in base_policy_settings: + for admPolicy in base_policy_settings[adm_namespace]: + log.debug('working on admPolicy {0}'.format(admPolicy)) + explicit_enable_disable_value_setting = False + this_key = None + this_valuename = None + if str(base_policy_settings[adm_namespace][admPolicy]).lower() == 'disabled': + log.debug('time to disable {0}'.format(admPolicy)) + this_policy = admx_policy_definitions.xpath(policySearchXpath.format(admPolicy), namespaces={'ns1': adm_namespace}) + if this_policy: + this_policy = this_policy[0] + if 'class' in this_policy.attrib: + if this_policy.attrib['class'] == registry_class or this_policy.attrib['class'] == 'Both': + if 'key' in this_policy.attrib: + this_key = this_policy.attrib['key'] + else: + msg = 'policy item {0} does not have the required "key" attribute' + log.error(msg.format(this_policy.attrib)) + break + if 'valueName' in this_policy.attrib: + this_valuename = this_policy.attrib['valueName'] + if DISABLED_VALUE_XPATH(this_policy): + # set the disabled value in the registry.pol file + explicit_enable_disable_value_setting = True + disabled_value_string = _checkValueItemParent(this_policy, + admPolicy, + this_key, + this_valuename, + DISABLED_VALUE_XPATH, + None, + check_deleted=False, + test_item=False) + existing_data = _policyFileReplaceOrAppend(disabled_value_string, existing_data) - if not explicit_enable_disable_value_setting and this_valuename: - disabled_value_string = _buildKnownDataSearchString(this_key, - this_valuename, - 'REG_DWORD', - None, - check_deleted=True) - existing_data = _policyFileReplaceOrAppend(disabled_value_string, - existing_data) - if ELEMENTS_XPATH(this_policy): - log.debug('checking elements of {0}'.format(admPolicy)) - for elements_item in ELEMENTS_XPATH(this_policy): - for child_item in elements_item.getchildren(): - child_key = this_key - child_valuename = this_valuename - if 'key' in child_item.attrib: - child_key = child_item.attrib['key'] - if 'valueName' in child_item.attrib: - child_valuename = child_item.attrib['valueName'] - if etree.QName(child_item).localname == 'boolean' \ - and (TRUE_LIST_XPATH(child_item) or FALSE_LIST_XPATH(child_item)): - # WARNING: no OOB adm files use true/falseList items - # this has not been fully vetted - temp_dict = {'trueList': TRUE_LIST_XPATH, 'falseList': FALSE_LIST_XPATH} - for this_list in temp_dict: - disabled_list_strings = _checkListItem( - child_item, - admPolicy, - child_key, - temp_dict[this_list], - None, - test_items=False) - log.debug('working with {1} portion of {0}'.format( - admPolicy, - this_list)) - existing_data = _policyFileReplaceOrAppendList( - disabled_list_strings, - existing_data) - elif etree.QName(child_item).localname == 'boolean' \ - or etree.QName(child_item).localname == 'decimal' \ - or etree.QName(child_item).localname == 'text' \ - or etree.QName(child_item).localname == 'longDecimal' \ - or etree.QName(child_item).localname == 'multiText' \ - or etree.QName(child_item).localname == 'enum': - disabled_value_string = _processValueItem(child_item, - child_key, - child_valuename, - this_policy, - elements_item, - check_deleted=True) - msg = 'I have disabled value string of {0}' - log.debug(msg.format(disabled_value_string)) - existing_data = _policyFileReplaceOrAppend( - disabled_value_string, - existing_data) - elif etree.QName(child_item).localname == 'list': - disabled_value_string = _processValueItem(child_item, - child_key, - child_valuename, - this_policy, - elements_item, - check_deleted=True) - msg = 'I have disabled value string of {0}' - log.debug(msg.format(disabled_value_string)) - existing_data = _policyFileReplaceOrAppend( - disabled_value_string, - existing_data) - else: - msg = 'policy {0} was found but it does not appear to be valid for the class {1}' - log.error(msg.format(admPolicy, registry_class)) - else: - msg = 'policy item {0} does not have the requried "class" attribute' - log.error(msg.format(this_policy.attrib)) - else: - log.debug('time to enable and set the policy "{0}"'.format(admPolicy)) - this_policy = policySearchXpath(admx_policy_definitions, id=admPolicy) - if this_policy: - this_policy = this_policy[0] - if 'class' in this_policy.attrib: - if this_policy.attrib['class'] == registry_class or this_policy.attrib['class'] == 'Both': - if 'key' in this_policy.attrib: - this_key = this_policy.attrib['key'] - else: - msg = 'policy item {0} does not have the required "key" attribute' - log.error(msg.format(this_policy.attrib)) - break - if 'valueName' in this_policy.attrib: - this_valuename = this_policy.attrib['valueName'] - - if ENABLED_VALUE_XPATH(this_policy): - explicit_enable_disable_value_setting = True - enabled_value_string = _checkValueItemParent(this_policy, - admPolicy, - this_key, - this_valuename, - ENABLED_VALUE_XPATH, - None, - check_deleted=False, - test_item=False) - existing_data = _policyFileReplaceOrAppend( - enabled_value_string, - existing_data) - if ENABLED_LIST_XPATH(this_policy): - explicit_enable_disable_value_setting = True - enabled_list_strings = _checkListItem(this_policy, - admPolicy, - this_key, - ENABLED_LIST_XPATH, - None, - test_items=False) - log.debug('working with enabledList portion of {0}'.format(admPolicy)) - existing_data = _policyFileReplaceOrAppendList( - enabled_list_strings, - existing_data) - if not explicit_enable_disable_value_setting and this_valuename: - enabled_value_string = _buildKnownDataSearchString(this_key, - this_valuename, - 'REG_DWORD', - '1', - check_deleted=False) - existing_data = _policyFileReplaceOrAppend( - enabled_value_string, - existing_data) - if ELEMENTS_XPATH(this_policy): - for elements_item in ELEMENTS_XPATH(this_policy): - for child_item in elements_item.getchildren(): - child_key = this_key - child_valuename = this_valuename - if 'key' in child_item.attrib: - child_key = child_item.attrib['key'] - if 'valueName' in child_item.attrib: - child_valuename = child_item.attrib['valueName'] - if child_item.attrib['id'] in base_policy_settings[admPolicy]: - if etree.QName(child_item).localname == 'boolean' and ( - TRUE_LIST_XPATH(child_item) or FALSE_LIST_XPATH(child_item)): - list_strings = [] - if base_policy_settings[admPolicy][child_item.attrib['id']]: - list_strings = _checkListItem(child_item, - admPolicy, - child_key, - TRUE_LIST_XPATH, - None, - test_items=False) - log.debug('working with trueList portion of {0}'.format(admPolicy)) - else: - list_strings = _checkListItem(child_item, - admPolicy, - child_key, - FALSE_LIST_XPATH, - None, - test_items=False) - existing_data = _policyFileReplaceOrAppendList( - list_strings, - existing_data) - if etree.QName(child_item).localname == 'boolean' and ( - TRUE_VALUE_XPATH(child_item) or FALSE_VALUE_XPATH(child_item)): - value_string = '' - if base_policy_settings[admPolicy][child_item.attrib['id']]: - value_string = _checkValueItemParent(child_item, - admPolicy, - child_key, - child_valuename, - TRUE_VALUE_XPATH, - None, - check_deleted=False, - test_item=False) - else: - value_string = _checkValueItemParent(child_item, - admPolicy, - child_key, - child_valuename, - FALSE_VALUE_XPATH, - None, - check_deleted=False, - test_item=False) - existing_data = _policyFileReplaceOrAppend( - value_string, - existing_data) + if DISABLED_LIST_XPATH(this_policy): + explicit_enable_disable_value_setting = True + disabled_list_strings = _checkListItem(this_policy, + admPolicy, + this_key, + DISABLED_LIST_XPATH, + None, + test_items=False) + log.debug('working with disabledList portion of {0}'.format(admPolicy)) + existing_data = _policyFileReplaceOrAppendList(disabled_list_strings, + existing_data) + if not explicit_enable_disable_value_setting and this_valuename: + disabled_value_string = _buildKnownDataSearchString(this_key, + this_valuename, + 'REG_DWORD', + None, + check_deleted=True) + existing_data = _policyFileReplaceOrAppend(disabled_value_string, + existing_data) + if ELEMENTS_XPATH(this_policy): + log.debug('checking elements of {0}'.format(admPolicy)) + for elements_item in ELEMENTS_XPATH(this_policy): + for child_item in elements_item.getchildren(): + child_key = this_key + child_valuename = this_valuename + if 'key' in child_item.attrib: + child_key = child_item.attrib['key'] + if 'valueName' in child_item.attrib: + child_valuename = child_item.attrib['valueName'] if etree.QName(child_item).localname == 'boolean' \ + and (TRUE_LIST_XPATH(child_item) or FALSE_LIST_XPATH(child_item)): + # WARNING: no OOB adm files use true/falseList items + # this has not been fully vetted + temp_dict = {'trueList': TRUE_LIST_XPATH, 'falseList': FALSE_LIST_XPATH} + for this_list in temp_dict: + disabled_list_strings = _checkListItem( + child_item, + admPolicy, + child_key, + temp_dict[this_list], + None, + test_items=False) + log.debug('working with {1} portion of {0}'.format( + admPolicy, + this_list)) + existing_data = _policyFileReplaceOrAppendList( + disabled_list_strings, + existing_data) + elif etree.QName(child_item).localname == 'boolean' \ or etree.QName(child_item).localname == 'decimal' \ or etree.QName(child_item).localname == 'text' \ or etree.QName(child_item).localname == 'longDecimal' \ - or etree.QName(child_item).localname == 'multiText': - enabled_value_string = _processValueItem( - child_item, - child_key, - child_valuename, - this_policy, - elements_item, - check_deleted=False, - this_element_value=base_policy_settings[admPolicy][child_item.attrib['id']]) - msg = 'I have enabled value string of {0}' - log.debug(msg.format([enabled_value_string])) + or etree.QName(child_item).localname == 'multiText' \ + or etree.QName(child_item).localname == 'enum': + disabled_value_string = _processValueItem(child_item, + child_key, + child_valuename, + this_policy, + elements_item, + check_deleted=True) + msg = 'I have disabled value string of {0}' + log.debug(msg.format(disabled_value_string)) existing_data = _policyFileReplaceOrAppend( - enabled_value_string, + disabled_value_string, existing_data) - elif etree.QName(child_item).localname == 'enum': - for enum_item in child_item.getchildren(): - if base_policy_settings[admPolicy][child_item.attrib['id']] == \ - _getAdmlDisplayName(adml_policy_resources, - enum_item.attrib['displayName'] - ).strip(): - enabled_value_string = _checkValueItemParent( - enum_item, - child_item.attrib['id'], - child_key, - child_valuename, - VALUE_XPATH, - None, - check_deleted=False, - test_item=False) - existing_data = _policyFileReplaceOrAppend( - enabled_value_string, - existing_data) - if VALUE_LIST_XPATH(enum_item): - enabled_list_strings = _checkListItem(enum_item, - admPolicy, - child_key, - VALUE_LIST_XPATH, - None, - test_items=False) - msg = 'working with valueList portion of {0}' - log.debug(msg.format(child_item.attrib['id'])) - existing_data = _policyFileReplaceOrAppendList( - enabled_list_strings, - existing_data) - break elif etree.QName(child_item).localname == 'list': - enabled_value_string = _processValueItem( - child_item, - child_key, - child_valuename, - this_policy, - elements_item, - check_deleted=False, - this_element_value=base_policy_settings[admPolicy][child_item.attrib['id']]) - msg = 'I have enabled value string of {0}' - log.debug(msg.format([enabled_value_string])) + disabled_value_string = _processValueItem(child_item, + child_key, + child_valuename, + this_policy, + elements_item, + check_deleted=True) + msg = 'I have disabled value string of {0}' + log.debug(msg.format(disabled_value_string)) existing_data = _policyFileReplaceOrAppend( - enabled_value_string, - existing_data, - append_only=True) + disabled_value_string, + existing_data) + else: + msg = 'policy {0} was found but it does not appear to be valid for the class {1}' + log.error(msg.format(admPolicy, registry_class)) + else: + msg = 'policy item {0} does not have the requried "class" attribute' + log.error(msg.format(this_policy.attrib)) + else: + log.debug('time to enable and set the policy "{0}"'.format(admPolicy)) + this_policy = admx_policy_definitions.xpath(policySearchXpath.format(admPolicy), namespaces={'ns1': adm_namespace}) + log.debug('found this_policy == {0}'.format(this_policy)) + if this_policy: + this_policy = this_policy[0] + if 'class' in this_policy.attrib: + if this_policy.attrib['class'] == registry_class or this_policy.attrib['class'] == 'Both': + if 'key' in this_policy.attrib: + this_key = this_policy.attrib['key'] + else: + msg = 'policy item {0} does not have the required "key" attribute' + log.error(msg.format(this_policy.attrib)) + break + if 'valueName' in this_policy.attrib: + this_valuename = this_policy.attrib['valueName'] + + if ENABLED_VALUE_XPATH(this_policy): + explicit_enable_disable_value_setting = True + enabled_value_string = _checkValueItemParent(this_policy, + admPolicy, + this_key, + this_valuename, + ENABLED_VALUE_XPATH, + None, + check_deleted=False, + test_item=False) + existing_data = _policyFileReplaceOrAppend( + enabled_value_string, + existing_data) + if ENABLED_LIST_XPATH(this_policy): + explicit_enable_disable_value_setting = True + enabled_list_strings = _checkListItem(this_policy, + admPolicy, + this_key, + ENABLED_LIST_XPATH, + None, + test_items=False) + log.debug('working with enabledList portion of {0}'.format(admPolicy)) + existing_data = _policyFileReplaceOrAppendList( + enabled_list_strings, + existing_data) + if not explicit_enable_disable_value_setting and this_valuename: + enabled_value_string = _buildKnownDataSearchString(this_key, + this_valuename, + 'REG_DWORD', + '1', + check_deleted=False) + existing_data = _policyFileReplaceOrAppend( + enabled_value_string, + existing_data) + if ELEMENTS_XPATH(this_policy): + for elements_item in ELEMENTS_XPATH(this_policy): + for child_item in elements_item.getchildren(): + child_key = this_key + child_valuename = this_valuename + if 'key' in child_item.attrib: + child_key = child_item.attrib['key'] + if 'valueName' in child_item.attrib: + child_valuename = child_item.attrib['valueName'] + if child_item.attrib['id'] in base_policy_settings[adm_namespace][admPolicy]: + if etree.QName(child_item).localname == 'boolean' and ( + TRUE_LIST_XPATH(child_item) or FALSE_LIST_XPATH(child_item)): + list_strings = [] + if base_policy_settings[adm_namespace][admPolicy][child_item.attrib['id']]: + list_strings = _checkListItem(child_item, + admPolicy, + child_key, + TRUE_LIST_XPATH, + None, + test_items=False) + log.debug('working with trueList portion of {0}'.format(admPolicy)) + else: + list_strings = _checkListItem(child_item, + admPolicy, + child_key, + FALSE_LIST_XPATH, + None, + test_items=False) + existing_data = _policyFileReplaceOrAppendList( + list_strings, + existing_data) + elif etree.QName(child_item).localname == 'boolean' and ( + TRUE_VALUE_XPATH(child_item) or FALSE_VALUE_XPATH(child_item)): + value_string = '' + if base_policy_settings[adm_namespace][admPolicy][child_item.attrib['id']]: + value_string = _checkValueItemParent(child_item, + admPolicy, + child_key, + child_valuename, + TRUE_VALUE_XPATH, + None, + check_deleted=False, + test_item=False) + else: + value_string = _checkValueItemParent(child_item, + admPolicy, + child_key, + child_valuename, + FALSE_VALUE_XPATH, + None, + check_deleted=False, + test_item=False) + existing_data = _policyFileReplaceOrAppend( + value_string, + existing_data) + elif etree.QName(child_item).localname == 'boolean' \ + or etree.QName(child_item).localname == 'decimal' \ + or etree.QName(child_item).localname == 'text' \ + or etree.QName(child_item).localname == 'longDecimal' \ + or etree.QName(child_item).localname == 'multiText': + enabled_value_string = _processValueItem( + child_item, + child_key, + child_valuename, + this_policy, + elements_item, + check_deleted=False, + this_element_value=base_policy_settings[adm_namespace][admPolicy][child_item.attrib['id']]) + msg = 'I have enabled value string of {0}' + log.debug(msg.format([enabled_value_string])) + existing_data = _policyFileReplaceOrAppend( + enabled_value_string, + existing_data) + elif etree.QName(child_item).localname == 'enum': + for enum_item in child_item.getchildren(): + if base_policy_settings[adm_namespace][admPolicy][child_item.attrib['id']] == \ + _getAdmlDisplayName(adml_policy_resources, + enum_item.attrib['displayName'] + ).strip(): + enabled_value_string = _checkValueItemParent( + enum_item, + child_item.attrib['id'], + child_key, + child_valuename, + VALUE_XPATH, + None, + check_deleted=False, + test_item=False) + existing_data = _policyFileReplaceOrAppend( + enabled_value_string, + existing_data) + if VALUE_LIST_XPATH(enum_item): + enabled_list_strings = _checkListItem(enum_item, + admPolicy, + child_key, + VALUE_LIST_XPATH, + None, + test_items=False) + msg = 'working with valueList portion of {0}' + log.debug(msg.format(child_item.attrib['id'])) + existing_data = _policyFileReplaceOrAppendList( + enabled_list_strings, + existing_data) + break + elif etree.QName(child_item).localname == 'list': + enabled_value_string = _processValueItem( + child_item, + child_key, + child_valuename, + this_policy, + elements_item, + check_deleted=False, + this_element_value=base_policy_settings[adm_namespace][admPolicy][child_item.attrib['id']]) + msg = 'I have enabled value string of {0}' + log.debug(msg.format([enabled_value_string])) + existing_data = _policyFileReplaceOrAppend( + enabled_value_string, + existing_data, + append_only=True) _write_regpol_data(existing_data, policy_data.admx_registry_classes[registry_class]['policy_path'], policy_data.gpt_ini_path, @@ -4552,6 +4602,7 @@ def _lookup_admin_template(policy_name, if admx_policy_definitions is None or adml_policy_resources is None: admx_policy_definitions, adml_policy_resources = _processPolicyDefinitions( display_language=adml_language) + admx_search_results = [] admx_search_results = ADMX_SEARCH_XPATH(admx_policy_definitions, policy_name=policy_name, registry_class=policy_class) @@ -4591,11 +4642,42 @@ def _lookup_admin_template(policy_name, if adml_search_results: multiple_adml_entries = False suggested_policies = '' + adml_to_remove = [] if len(adml_search_results) > 1: multiple_adml_entries = True for adml_search_result in adml_search_results: if not getattr(adml_search_result, 'text', '').strip() == policy_name: - adml_search_results.remove(adml_search_result) + adml_to_remove.append(adml_search_result) + else: + if hierarchy: + display_name_searchval = '$({0}.{1})'.format( + adml_search_result.tag.split('}')[1], + adml_search_result.attrib['id']) + #policy_search_string = '//{0}:policy[@*[local-name() = "displayName"] = "{1}" and (@*[local-name() = "class"] = "Both" or @*[local-name() = "class"] = "{2}") ]'.format( + policy_search_string = '//{0}:policy[@displayName = "{1}" and (@class = "Both" or @class = "{2}") ]'.format( + adml_search_result.prefix, + display_name_searchval, + policy_class) + admx_results = [] + admx_search_results = admx_policy_definitions.xpath(policy_search_string, namespaces=adml_search_result.nsmap) + for search_result in admx_search_results: + log.debug('policy_name == {0}'.format(policy_name)) + this_hierarchy = _build_parent_list(search_result, + admx_policy_definitions, + True, + adml_policy_resources) + this_hierarchy.reverse() + if hierarchy != this_hierarchy: + adml_to_remove.append(adml_search_result) + else: + admx_results.append(search_result) + if len(admx_results) == 1: + admx_search_results = admx_results + for adml in adml_to_remove: + if adml in adml_search_results: + adml_search_results.remove(adml) + if len(adml_search_results) == 1 and multiple_adml_entries: + multiple_adml_entries = False for adml_search_result in adml_search_results: dmsg = 'found an ADML entry matching the string! {0} -- {1}' log.debug(dmsg.format(adml_search_result.tag, @@ -4604,10 +4686,11 @@ def _lookup_admin_template(policy_name, adml_search_result.tag.split('}')[1], adml_search_result.attrib['id']) log.debug('searching for displayName == {0}'.format(display_name_searchval)) - admx_search_results = ADMX_DISPLAYNAME_SEARCH_XPATH( - admx_policy_definitions, - display_name=display_name_searchval, - registry_class=policy_class) + if not admx_search_results: + admx_search_results = ADMX_DISPLAYNAME_SEARCH_XPATH( + admx_policy_definitions, + display_name=display_name_searchval, + registry_class=policy_class) if admx_search_results: if len(admx_search_results) == 1 or hierarchy and not multiple_adml_entries: found = False @@ -4619,6 +4702,7 @@ def _lookup_admin_template(policy_name, True, adml_policy_resources) this_hierarchy.reverse() + log.debug('testing {0} == {1}'.format(hierarchy, this_hierarchy)) if hierarchy == this_hierarchy: found = True else: @@ -5077,6 +5161,7 @@ def set_(computer_policy=None, user_policy=None, if policies[p_class]: for policy_name in policies[p_class]: _pol = None + policy_namespace = None policy_key_name = policy_name if policy_name in _policydata.policies[p_class]['policies']: _pol = _policydata.policies[p_class]['policies'][policy_name] @@ -5126,16 +5211,19 @@ def set_(computer_policy=None, user_policy=None, adml_policy_resources=admlPolicyResources) if success: policy_name = the_policy.attrib['name'] - _admTemplateData[policy_name] = _value + policy_namespace = the_policy.nsmap[the_policy.prefix] + if policy_namespace not in _admTemplateData: + _admTemplateData[policy_namespace] = {} + _admTemplateData[policy_namespace][policy_name] = _value else: raise SaltInvocationError(msg) - if policy_name in _admTemplateData and the_policy is not None: - log.debug('setting == {0}'.format(_admTemplateData[policy_name]).lower()) - log.debug('{0}'.format(str(_admTemplateData[policy_name]).lower())) - if str(_admTemplateData[policy_name]).lower() != 'disabled' \ - and str(_admTemplateData[policy_name]).lower() != 'not configured': + if policy_namespace and policy_name in _admTemplateData[policy_namespace] and the_policy is not None: + log.debug('setting == {0}'.format(_admTemplateData[policy_namespace][policy_name]).lower()) + log.debug('{0}'.format(str(_admTemplateData[policy_namespace][policy_name]).lower())) + if str(_admTemplateData[policy_namespace][policy_name]).lower() != 'disabled' \ + and str(_admTemplateData[policy_namespace][policy_name]).lower() != 'not configured': if ELEMENTS_XPATH(the_policy): - if isinstance(_admTemplateData[policy_name], dict): + if isinstance(_admTemplateData[policy_namespace][policy_name], dict): for elements_item in ELEMENTS_XPATH(the_policy): for child_item in elements_item.getchildren(): # check each element @@ -5146,9 +5234,9 @@ def set_(computer_policy=None, user_policy=None, True, admlPolicyResources) log.debug('id attribute == "{0}" this_element_name == "{1}"'.format(child_item.attrib['id'], this_element_name)) - if this_element_name in _admTemplateData[policy_name]: + if this_element_name in _admTemplateData[policy_namespace][policy_name]: temp_element_name = this_element_name - elif child_item.attrib['id'] in _admTemplateData[policy_name]: + elif child_item.attrib['id'] in _admTemplateData[policy_namespace][policy_name]: temp_element_name = child_item.attrib['id'] else: msg = ('Element "{0}" must be included' @@ -5156,12 +5244,12 @@ def set_(computer_policy=None, user_policy=None, raise SaltInvocationError(msg.format(this_element_name, policy_name)) if 'required' in child_item.attrib \ and child_item.attrib['required'].lower() == 'true': - if not _admTemplateData[policy_name][temp_element_name]: + if not _admTemplateData[policy_namespace][policy_name][temp_element_name]: msg = 'Element "{0}" requires a value to be specified' raise SaltInvocationError(msg.format(temp_element_name)) if etree.QName(child_item).localname == 'boolean': if not isinstance( - _admTemplateData[policy_name][temp_element_name], + _admTemplateData[policy_namespace][policy_name][temp_element_name], bool): msg = 'Element {0} requires a boolean True or False' raise SaltInvocationError(msg.format(temp_element_name)) @@ -5173,9 +5261,9 @@ def set_(computer_policy=None, user_policy=None, min_val = int(child_item.attrib['minValue']) if 'maxValue' in child_item.attrib: max_val = int(child_item.attrib['maxValue']) - if int(_admTemplateData[policy_name][temp_element_name]) \ + if int(_admTemplateData[policy_namespace][policy_name][temp_element_name]) \ < min_val or \ - int(_admTemplateData[policy_name][temp_element_name]) \ + int(_admTemplateData[policy_namespace][policy_name][temp_element_name]) \ > max_val: msg = 'Element "{0}" value must be between {1} and {2}' raise SaltInvocationError(msg.format(temp_element_name, @@ -5185,7 +5273,7 @@ def set_(computer_policy=None, user_policy=None, # make sure the value is in the enumeration found = False for enum_item in child_item.getchildren(): - if _admTemplateData[policy_name][temp_element_name] == \ + if _admTemplateData[policy_namespace][policy_name][temp_element_name] == \ _getAdmlDisplayName( admlPolicyResources, enum_item.attrib['displayName']).strip(): @@ -5199,33 +5287,33 @@ def set_(computer_policy=None, user_policy=None, and child_item.attrib['explicitValue'].lower() == \ 'true': if not isinstance( - _admTemplateData[policy_name][temp_element_name], + _admTemplateData[policy_namespace][policy_name][temp_element_name], dict): msg = ('Each list item of element "{0}" ' 'requires a dict value') msg = msg.format(temp_element_name) raise SaltInvocationError(msg) elif not isinstance( - _admTemplateData[policy_name][temp_element_name], + _admTemplateData[policy_namespace][policy_name][temp_element_name], list): msg = 'Element "{0}" requires a list value' msg = msg.format(temp_element_name) raise SaltInvocationError(msg) elif etree.QName(child_item).localname == 'multiText': if not isinstance( - _admTemplateData[policy_name][temp_element_name], + _admTemplateData[policy_namespace][policy_name][temp_element_name], list): msg = 'Element "{0}" requires a list value' msg = msg.format(temp_element_name) raise SaltInvocationError(msg) - _admTemplateData[policy_name][child_item.attrib['id']] = \ - _admTemplateData[policy_name].pop(temp_element_name) + _admTemplateData[policy_namespace][policy_name][child_item.attrib['id']] = \ + _admTemplateData[policy_namespace][policy_name].pop(temp_element_name) else: msg = 'The policy "{0}" has elements which must be configured' msg = msg.format(policy_name) raise SaltInvocationError(msg) else: - if str(_admTemplateData[policy_name]).lower() != 'enabled': + if str(_admTemplateData[policy_namespace][policy_name]).lower() != 'enabled': msg = ('The policy {0} must either be "Enabled", ' '"Disabled", or "Not Configured"') msg = msg.format(policy_name) diff --git a/salt/modules/win_license.py b/salt/modules/win_license.py index 10598424fdd..ce990b79598 100644 --- a/salt/modules/win_license.py +++ b/salt/modules/win_license.py @@ -13,7 +13,7 @@ import re import logging # Import Salt Libs -import salt.utils +import salt.utils.platform log = logging.getLogger(__name__) __virtualname__ = 'license' @@ -23,7 +23,7 @@ def __virtual__(): ''' Only work on Windows ''' - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): return __virtualname__ return False diff --git a/salt/modules/win_network.py b/salt/modules/win_network.py index 3ee8fc8c5d3..d01ea6d3e87 100644 --- a/salt/modules/win_network.py +++ b/salt/modules/win_network.py @@ -4,18 +4,16 @@ Module for gathering and managing network information ''' from __future__ import absolute_import -# Import python libs +# Import Python libs import re - -# Import salt libs -import salt.utils import hashlib import datetime import socket -# Import salt libs +# Import Salt libs import salt.utils import salt.utils.network +import salt.utils.platform import salt.utils.validate.net from salt.modules.network import (wol, get_hostname, interface, interface_ip, subnets6, ip_in_subnet, convert_cidr, @@ -44,7 +42,7 @@ def __virtual__(): ''' Only works on Windows systems ''' - if not salt.utils.is_windows(): + if not salt.utils.platform.is_windows(): return False, "Module win_network: Only available on Windows" if not HAS_DEPENDENCIES: diff --git a/salt/modules/win_ntp.py b/salt/modules/win_ntp.py index 11051b9a76f..3a3109aee35 100644 --- a/salt/modules/win_ntp.py +++ b/salt/modules/win_ntp.py @@ -6,11 +6,11 @@ Management of NTP servers on Windows ''' from __future__ import absolute_import -# Import python libs +# Import Python libs import logging -# Import salt libs -import salt.utils +# Import Salt libs +import salt.utils.platform log = logging.getLogger(__name__) @@ -22,7 +22,7 @@ def __virtual__(): ''' This only supports Windows ''' - if not salt.utils.is_windows(): + if not salt.utils.platform.is_windows(): return (False, "Module win_system: module only works on Windows systems") return __virtualname__ diff --git a/salt/modules/win_path.py b/salt/modules/win_path.py index 3b446f7ceaa..fa57f53d180 100644 --- a/salt/modules/win_path.py +++ b/salt/modules/win_path.py @@ -8,13 +8,16 @@ http://support.microsoft.com/kb/104011 ''' from __future__ import absolute_import -# Python Libs +# Import Python libs import logging -import re import os -from salt.ext.six.moves import map +import re -# Third party libs +# Import Salt libs +import salt.utils.platform + +# Import 3rd-party libs +from salt.ext.six.moves import map try: from win32con import HWND_BROADCAST, WM_SETTINGCHANGE from win32api import SendMessage @@ -22,9 +25,6 @@ try: except ImportError: HAS_WIN32 = False -# Import salt libs -import salt.utils - # Settings log = logging.getLogger(__name__) @@ -33,7 +33,7 @@ def __virtual__(): ''' Load only on Windows ''' - if salt.utils.is_windows() and HAS_WIN32: + if salt.utils.platform.is_windows() and HAS_WIN32: return 'win_path' return (False, "Module win_path: module only works on Windows systems") @@ -51,7 +51,7 @@ def rehash(): CLI Example: - ... code-block:: bash + .. code-block:: bash salt '*' win_path.rehash ''' diff --git a/salt/modules/win_pkg.py b/salt/modules/win_pkg.py index 510776e3184..c729e07ef5c 100644 --- a/salt/modules/win_pkg.py +++ b/salt/modules/win_pkg.py @@ -42,7 +42,7 @@ import time from functools import cmp_to_key # Import third party libs -import salt.ext.six as six +from salt.ext import six # pylint: disable=import-error,no-name-in-module from salt.ext.six.moves.urllib.parse import urlparse as _urlparse @@ -50,8 +50,12 @@ from salt.ext.six.moves.urllib.parse import urlparse as _urlparse from salt.exceptions import (CommandExecutionError, SaltInvocationError, SaltRenderError) -import salt.utils +import salt.utils # Can be removed once is_true, get_hash, compare_dicts are moved +import salt.utils.args +import salt.utils.files import salt.utils.pkg +import salt.utils.platform +import salt.utils.versions import salt.syspaths import salt.payload from salt.exceptions import MinionError @@ -67,7 +71,7 @@ def __virtual__(): ''' Set the virtual pkg module if the os is Windows ''' - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): return __virtualname__ return (False, "Module win_pkg: module only works on Windows systems") @@ -146,7 +150,7 @@ def latest_version(*names, **kwargs): # check, whether latest available version # is newer than latest installed version - if salt.utils.compare_versions(ver1=str(latest_available), + if salt.utils.versions.compare(ver1=str(latest_available), oper='>', ver2=str(latest_installed)): log.debug('Upgrade of {0} from {1} to {2} ' @@ -762,7 +766,7 @@ def genrepo(**kwargs): ) serial = salt.payload.Serial(__opts__) mode = 'w+' if six.PY2 else 'wb+' - with salt.utils.fopen(repo_details.winrepo_file, mode) as repo_cache: + with salt.utils.files.fopen(repo_details.winrepo_file, mode) as repo_cache: repo_cache.write(serial.dumps(ret)) # save reading it back again. ! this breaks due to utf8 issues #__context__['winrepo.data'] = ret @@ -945,27 +949,51 @@ def install(name=None, refresh=False, pkgs=None, **kwargs): packages listed under ``pkgs`` will be installed via a single command. + You can specify a version by passing the item as a dict: + + CLI Example: + + .. code-block:: bash + + # will install the latest version of foo and bar + salt '*' pkg.install pkgs='["foo", "bar"]' + + # will install the latest version of foo and version 1.2.3 of bar + salt '*' pkg.install pkgs='["foo", {"bar": "1.2.3"}]' + Kwargs: version (str): - The specific version to install. If omitted, the latest version - will be installed. If passed with multiple install, the version - will apply to all packages. Recommended for single installation - only. + The specific version to install. If omitted, the latest version will + be installed. Recommend for use when installing a single package. - cache_file (str): - A single file to copy down for use with the installer. Copied to - the same location as the installer. Use this over ``cache_dir`` if - there are many files in the directory and you only need a specific - file and don't want to cache additional files that may reside in - the installer directory. Only applies to files on ``salt://`` + If passed with a list of packages in the ``pkgs`` parameter, the + version will be ignored. - cache_dir (bool): - True will copy the contents of the installer directory. This is - useful for installations that are not a single file. Only applies - to directories on ``salt://`` + CLI Example: - saltenv (str): Salt environment. Default 'base' + .. code-block:: bash + + # Version is ignored + salt '*' pkg.install pkgs="['foo', 'bar']" version=1.2.3 + + If passed with a comma separated list in the ``name`` parameter, the + version will apply to all packages in the list. + + CLI Example: + + .. code-block:: bash + + # Version 1.2.3 will apply to packages foo and bar + salt '*' pkg.install foo,bar version=1.2.3 + + extra_install_flags (str): + Additional install flags that will be appended to the + ``install_flags`` defined in the software definition file. Only + applies when single package is passed. + + saltenv (str): + Salt environment. Default 'base' report_reboot_exit_codes (bool): If the installer exits with a recognized exit code indicating that @@ -1061,13 +1089,22 @@ def install(name=None, refresh=False, pkgs=None, **kwargs): # "sources" argument pkg_params = __salt__['pkg_resource.parse_targets'](name, pkgs, **kwargs)[0] + if len(pkg_params) > 1: + if kwargs.get('extra_install_flags') is not None: + log.warning('\'extra_install_flags\' argument will be ignored for ' + 'multiple package targets') + + # Windows expects an Options dictionary containing 'version' + for pkg in pkg_params: + pkg_params[pkg] = {'version': pkg_params[pkg]} + if pkg_params is None or len(pkg_params) == 0: log.error('No package definition found') return {} if not pkgs and len(pkg_params) == 1: - # Only use the 'version' param if 'name' was not specified as a - # comma-separated list + # Only use the 'version' param if a single item was passed to the 'name' + # parameter pkg_params = { name: { 'version': kwargs.get('version'), @@ -1092,11 +1129,15 @@ def install(name=None, refresh=False, pkgs=None, **kwargs): ret[pkg_name] = 'Unable to locate package {0}'.format(pkg_name) continue - # Get the version number passed or the latest available + # Get the version number passed or the latest available (must be a string) version_num = '' if options: - if options.get('version') is not None: - version_num = str(options.get('version')) + version_num = options.get('version', '') + # Using the salt cmdline with version=5.3 might be interpreted + # as a float it must be converted to a string in order for + # string matching to work. + if not isinstance(version_num, six.string_types) and version_num is not None: + version_num = str(version_num) if not version_num: version_num = _get_latest_pkg_version(pkginfo) @@ -1232,6 +1273,18 @@ def install(name=None, refresh=False, pkgs=None, **kwargs): #Compute msiexec string use_msiexec, msiexec = _get_msiexec(pkginfo[version_num].get('msiexec', False)) + # Build cmd and arguments + # cmd and arguments must be separated for use with the task scheduler + if use_msiexec: + cmd = msiexec + arguments = ['/i', cached_pkg] + if pkginfo[version_num].get('allusers', True): + arguments.append('ALLUSERS="1"') + arguments.extend(salt.utils.shlex_split(install_flags)) + else: + cmd = cached_pkg + arguments = salt.utils.shlex_split(install_flags) + # Install the software # Check Use Scheduler Option if pkginfo[version_num].get('use_scheduler', False): @@ -1242,10 +1295,10 @@ def install(name=None, refresh=False, pkgs=None, **kwargs): arguments = ['/i', cached_pkg] if pkginfo['version_num'].get('allusers', True): arguments.append('ALLUSERS="1"') - arguments.extend(salt.utils.shlex_split(install_flags)) + arguments.extend(salt.utils.args.shlex_split(install_flags)) else: cmd = cached_pkg - arguments = salt.utils.shlex_split(install_flags) + arguments = salt.utils.args.shlex_split(install_flags) # Create Scheduled Task __salt__['task.create_task'](name='update-salt-software', @@ -1260,21 +1313,46 @@ def install(name=None, refresh=False, pkgs=None, **kwargs): start_time='01:00', ac_only=False, stop_if_on_batteries=False) + # Run Scheduled Task - if not __salt__['task.run_wait'](name='update-salt-software'): - log.error('Failed to install {0}'.format(pkg_name)) - log.error('Scheduled Task failed to run') - ret[pkg_name] = {'install status': 'failed'} - else: - # Build the install command - cmd = [] - if use_msiexec: - cmd.extend([msiexec, '/i', cached_pkg]) - if pkginfo[version_num].get('allusers', True): - cmd.append('ALLUSERS="1"') + # Special handling for installing salt + if re.search(r'salt[\s_.-]*minion', + pkg_name, + flags=re.IGNORECASE + re.UNICODE) is not None: + ret[pkg_name] = {'install status': 'task started'} + if not __salt__['task.run'](name='update-salt-software'): + log.error('Failed to install {0}'.format(pkg_name)) + log.error('Scheduled Task failed to run') + ret[pkg_name] = {'install status': 'failed'} + else: + + # Make sure the task is running, try for 5 secs + from time import time + t_end = time() + 5 + while time() < t_end: + task_running = __salt__['task.status']( + 'update-salt-software') == 'Running' + if task_running: + break + + if not task_running: + log.error( + 'Failed to install {0}'.format(pkg_name)) + log.error('Scheduled Task failed to run') + ret[pkg_name] = {'install status': 'failed'} + + # All other packages run with task scheduler else: - cmd.append(cached_pkg) - cmd.extend(salt.utils.shlex_split(install_flags)) + if not __salt__['task.run_wait'](name='update-salt-software'): + log.error('Failed to install {0}'.format(pkg_name)) + log.error('Scheduled Task failed to run') + ret[pkg_name] = {'install status': 'failed'} + else: + + # Combine cmd and arguments + cmd = [cmd] + cmd.extend(arguments) + # Launch the command result = __salt__['cmd.run_all'](cmd, cache_path, @@ -1423,6 +1501,11 @@ def remove(name=None, pkgs=None, version=None, **kwargs): continue if version_num is not None: + # Using the salt cmdline with version=5.3 might be interpreted + # as a float it must be converted to a string in order for + # string matching to work. + if not isinstance(version_num, six.string_types) and version_num is not None: + version_num = str(version_num) if version_num not in pkginfo and 'latest' in pkginfo: version_num = 'latest' elif 'latest' in pkginfo: @@ -1540,10 +1623,10 @@ def remove(name=None, pkgs=None, version=None, **kwargs): if use_msiexec: cmd = msiexec arguments = ['/x'] - arguments.extend(salt.utils.shlex_split(uninstall_flags)) + arguments.extend(salt.utils.args.shlex_split(uninstall_flags)) else: cmd = expanded_cached_pkg - arguments = salt.utils.shlex_split(uninstall_flags) + arguments = salt.utils.args.shlex_split(uninstall_flags) # Create Scheduled Task __salt__['task.create_task'](name='update-salt-software', @@ -1570,7 +1653,7 @@ def remove(name=None, pkgs=None, version=None, **kwargs): cmd.extend([msiexec, '/x', expanded_cached_pkg]) else: cmd.append(expanded_cached_pkg) - cmd.extend(salt.utils.shlex_split(uninstall_flags)) + cmd.extend(salt.utils.args.shlex_split(uninstall_flags)) # Launch the command result = __salt__['cmd.run_all']( cmd, @@ -1684,7 +1767,7 @@ def get_repo_data(saltenv='base'): try: serial = salt.payload.Serial(__opts__) - with salt.utils.fopen(repo_details.winrepo_file, 'rb') as repofile: + with salt.utils.files.fopen(repo_details.winrepo_file, 'rb') as repofile: try: repodata = serial.loads(repofile.read()) or {} __context__['winrepo.data'] = repodata @@ -1758,4 +1841,4 @@ def compare_versions(ver1='', oper='==', ver2=''): salt '*' pkg.compare_versions 1.2 >= 1.3 ''' - return salt.utils.compare_versions(ver1, oper, ver2) + return salt.utils.versions.compare(ver1, oper, ver2) diff --git a/salt/modules/win_pki.py b/salt/modules/win_pki.py index 5b046b364a4..329da531f0a 100644 --- a/salt/modules/win_pki.py +++ b/salt/modules/win_pki.py @@ -1,22 +1,31 @@ # -*- coding: utf-8 -*- ''' -Microsoft certificate management via the Pki PowerShell module. +Microsoft certificate management via the PKI Client PowerShell module. +https://technet.microsoft.com/en-us/itpro/powershell/windows/pkiclient/pkiclient + +The PKI Client PowerShell module is only available on Windows 8+ and Windows +Server 2012+. +https://technet.microsoft.com/en-us/library/hh848636(v=wps.620).aspx :platform: Windows +:depends: + - PowerShell 4 + - PKI Client Module (Windows 8+ / Windows Server 2012+) + .. versionadded:: 2016.11.0 ''' -# Import python libs +# Import Python libs from __future__ import absolute_import +import ast import json import logging +import os # Import salt libs -from salt.exceptions import SaltInvocationError -import ast -import os -import salt.utils +import salt.utils.platform import salt.utils.powershell +from salt.exceptions import SaltInvocationError _DEFAULT_CONTEXT = 'LocalMachine' _DEFAULT_FORMAT = 'cer' @@ -29,11 +38,17 @@ __virtualname__ = 'win_pki' def __virtual__(): ''' - Only works on Windows systems with the PKI PowerShell module installed. + Requires Windows + Requires Windows 8+ / Windows Server 2012+ + Requires PowerShell + Requires PKI Client PowerShell module installed. ''' - if not salt.utils.is_windows(): + if not salt.utils.platform.is_windows(): return False, 'Only available on Windows Systems' + if salt.utils.version_cmp(__grains__['osversion'], '6.2.9200') == -1: + return False, 'Only available on Windows 8+ / Windows Server 2012 +' + if not __salt__['cmd.shell_info']('powershell')['installed']: return False, 'Powershell not available' @@ -155,7 +170,11 @@ def get_certs(context=_DEFAULT_CONTEXT, store=_DEFAULT_STORE): if key not in blacklist_keys: cert_info[key.lower()] = item[key] - cert_info['dnsnames'] = [name['Unicode'] for name in item['DnsNameList']] + names = item.get('DnsNameList', None) + if isinstance(names, list): + cert_info['dnsnames'] = [name.get('Unicode') for name in names] + else: + cert_info['dnsnames'] = [] ret[item['Thumbprint']] = cert_info return ret diff --git a/salt/modules/win_psget.py b/salt/modules/win_psget.py index c310349efdb..a3d020795c3 100644 --- a/salt/modules/win_psget.py +++ b/salt/modules/win_psget.py @@ -10,13 +10,14 @@ Support for PowerShell ''' from __future__ import absolute_import, unicode_literals -# Import python libs +# Import Python libs import copy import logging import json -# Import salt libs -import salt.utils +# Import Salt libs +import salt.utils.platform +import salt.utils.versions from salt.exceptions import CommandExecutionError # Set up logging @@ -31,7 +32,7 @@ def __virtual__(): Set the system module of the kernel is Windows ''' # Verify Windows - if not salt.utils.is_windows(): + if not salt.utils.platform.is_windows(): log.debug('Module PSGet: Only available on Windows systems') return False, 'Module PSGet: Only available on Windows systems' @@ -42,7 +43,7 @@ def __virtual__(): return False, 'Module PSGet: Requires PowerShell' # Verify PowerShell 5.0 or greater - if salt.utils.compare_versions(powershell_info['version'], '<', '5.0'): + if salt.utils.versions.compare(powershell_info['version'], '<', '5.0'): log.debug('Module PSGet: Requires PowerShell 5 or newer') return False, 'Module PSGet: Requires PowerShell 5 or newer.' diff --git a/salt/modules/win_repo.py b/salt/modules/win_repo.py index bf2a05d6500..975ba40496d 100644 --- a/salt/modules/win_repo.py +++ b/salt/modules/win_repo.py @@ -16,6 +16,8 @@ import os # Import salt libs import salt.output import salt.utils +import salt.utils.path +import salt.utils.platform import salt.loader import salt.template from salt.exceptions import CommandExecutionError, SaltRenderError @@ -45,7 +47,7 @@ def __virtual__(): ''' Set the winrepo module if the OS is Windows ''' - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): global _genrepo, _update_git_repos _genrepo = salt.utils.namespaced_function(_genrepo, globals()) _update_git_repos = \ @@ -113,7 +115,7 @@ def update_git_repos(clean=False): salt-call winrepo.update_git_repos ''' - if not salt.utils.which('git'): + if not salt.utils.path.which('git'): raise CommandExecutionError( 'Git for Windows is not installed, or not configured to be ' 'accessible from the Command Prompt' diff --git a/salt/modules/win_servermanager.py b/salt/modules/win_servermanager.py index f1aa4ce172a..46c3b1d9535 100644 --- a/salt/modules/win_servermanager.py +++ b/salt/modules/win_servermanager.py @@ -1,11 +1,15 @@ # -*- coding: utf-8 -*- ''' -Manage Windows features via the ServerManager powershell module +Manage Windows features via the ServerManager powershell module. Can list +available and installed roles/features. Can install and remove roles/features. + +:maintainer: Shane Lee +:platform: Windows Server 2008R2 or greater +:depends: PowerShell module ``ServerManager`` ''' # Import Python libs from __future__ import absolute_import -import ast import json import logging @@ -14,9 +18,10 @@ try: except ImportError: from pipes import quote as _cmd_quote -# Import salt libs -import salt.utils +# Import Salt libs +import salt.utils.platform import salt.utils.powershell +import salt.utils.versions log = logging.getLogger(__name__) @@ -27,21 +32,10 @@ def __virtual__(): ''' Load only on windows with servermanager module ''' - def _module_present(): - ''' - Check for the presence of the ServerManager module. - ''' - cmd = r"[Bool] (Get-Module -ListAvailable | Where-Object { $_.Name -eq 'ServerManager' })" - cmd_ret = __salt__['cmd.run_all'](cmd, shell='powershell', python_shell=True) - - if cmd_ret['retcode'] == 0: - return ast.literal_eval(cmd_ret['stdout']) + if not salt.utils.platform.is_windows(): return False - if not salt.utils.is_windows(): - return False - - if salt.utils.version_cmp(__grains__['osversion'], '6.1.7600') == -1: + if salt.utils.versions.version_cmp(__grains__['osversion'], '6.1.7600') == -1: return False, 'Failed to load win_servermanager module: ' \ 'Requires Remote Server Administration Tools which ' \ 'is only available on Windows 2008 R2 and later.' @@ -57,7 +51,7 @@ def __virtual__(): def _pshell_json(cmd, cwd=None): ''' Execute the desired powershell command and ensure that it returns data - in json format and load that into python + in JSON format and load that into python ''' cmd = 'Import-Module ServerManager; {0}'.format(cmd) if 'convertto-json' not in cmd.lower(): @@ -75,8 +69,9 @@ def list_available(): ''' List available features to install - :return: A list of available features - :rtype: list + Returns: + str: A list of available features as returned by the + ``Get-WindowsFeature`` PowerShell command CLI Example: @@ -96,8 +91,8 @@ def list_installed(): List installed features. Supported on Windows Server 2008 and Windows 8 and newer. - :return: A list of installed features - :rtype: list + Returns: + dict: A dictionary of installed features CLI Example: @@ -120,7 +115,7 @@ def list_installed(): def install(feature, recurse=False, restart=False, source=None, exclude=None): - ''' + r''' Install a feature .. note:: @@ -131,83 +126,157 @@ def install(feature, recurse=False, restart=False, source=None, exclude=None): Some features take a long time to complete un/installation, set -t with a long timeout - :param str feature: The name of the feature to install + Args: - :param bool recurse: Install all sub-features. Default is False + feature (str, list): + The name of the feature(s) to install. This can be a single feature, + a string of features in a comma delimited list (no spaces), or a + list of features. - :param str source: Path to the source files if missing from the target - system. None means that the system will use windows update services to - find the required files. Default is None + .. versionadded:: Oxygen + Added the ability to pass a list of features to be installed. - :param bool restart: Restarts the computer when installation is complete, if - required by the role/feature installed. Default is False + recurse (Options[bool]): + Install all sub-features. Default is False - :param str exclude: The name of the feature to exclude when installing the - named feature. + restart (Optional[bool]): + Restarts the computer when installation is complete, if required by + the role/feature installed. Will also trigger a reboot if an item + in ``exclude`` requires a reboot to be properly removed. Default is + False - .. note:: - As there is no exclude option for the ``Add-WindowsFeature`` - command, the feature will be installed with other sub-features and - will then be removed. + source (Optional[str]): + Path to the source files if missing from the target system. None + means that the system will use windows update services to find the + required files. Default is None - :param bool restart: Restarts the computer when installation is complete, if required by the role feature installed. + exclude (Optional[str]): + The name of the feature to exclude when installing the named + feature. This can be a single feature, a string of features in a + comma-delimited list (no spaces), or a list of features. - :return: A dictionary containing the results of the install - :rtype: dict + .. warning:: + As there is no exclude option for the ``Add-WindowsFeature`` + or ``Install-WindowsFeature`` PowerShell commands the features + named in ``exclude`` will be installed with other sub-features + and will then be removed. **If the feature named in ``exclude`` + is not a sub-feature of one of the installed items it will still + be removed.** + + Returns: + dict: A dictionary containing the results of the install CLI Example: .. code-block:: bash + # Install the Telnet Client passing a single string salt '*' win_servermanager.install Telnet-Client - salt '*' win_servermanager.install SNMP-Service True - salt '*' win_servermanager.install TFTP-Client source=d:\\side-by-side - ''' - # Use Install-WindowsFeature on Windows 8 (osversion 6.2) and later minions. Includes Windows 2012+. - # Default to Add-WindowsFeature for earlier releases of Windows. - # The newer command makes management tools optional so add them for partity with old behavior. + # Install the TFTP Client and the SNMP Service passing a comma-delimited + # string. Install all sub-features + salt '*' win_servermanager.install TFTP-Client,SNMP-Service recurse=True + + # Install the TFTP Client from d:\side-by-side + salt '*' win_servermanager.install TFTP-Client source=d:\\side-by-side + + # Install the XPS Viewer, SNMP Service, and Remote Access passing a + # list. Install all sub-features, but exclude the Web Server + salt '*' win_servermanager.install "['XPS-Viewer', 'SNMP-Service', 'RemoteAccess']" True recurse=True exclude="Web-Server" + ''' + # If it is a list of features, make it a comma delimited string + if isinstance(feature, list): + feature = ','.join(feature) + + # Use Install-WindowsFeature on Windows 2012 (osversion 6.2) and later + # minions. Default to Add-WindowsFeature for earlier releases of Windows. + # The newer command makes management tools optional so add them for parity + # with old behavior. command = 'Add-WindowsFeature' management_tools = '' - if salt.utils.version_cmp(__grains__['osversion'], '6.2') >= 0: + if salt.utils.versions.version_cmp(__grains__['osversion'], '6.2') >= 0: command = 'Install-WindowsFeature' management_tools = '-IncludeManagementTools' - sub = '' - if recurse: - sub = '-IncludeAllSubFeature' - - rst = '' - if restart: - rst = '-Restart' - - src = '' - if source is not None: - src = '-Source {0}'.format(source) - - cmd = '{0} -Name {1} {2} {3} {4} {5} ' \ - '-ErrorAction SilentlyContinue ' \ - '-WarningAction SilentlyContinue'.format(command, - _cmd_quote(feature), - sub, - src, - rst, - management_tools) + cmd = '{0} -Name {1} {2} {3} {4} ' \ + '-WarningAction SilentlyContinue'\ + .format(command, _cmd_quote(feature), management_tools, + '-IncludeAllSubFeature' if recurse else '', + '' if source is None else '-Source {0}'.format(source)) out = _pshell_json(cmd) + # Uninstall items in the exclude list + # The Install-WindowsFeature command doesn't have the concept of an exclude + # list. So you install first, then remove if exclude is not None: - remove(exclude, restart=restart) + removed = remove(exclude) + # Results are stored in a list of dictionaries in `FeatureResult` if out['FeatureResult']: - return {'ExitCode': out['ExitCode'], - 'DisplayName': out['FeatureResult'][0]['DisplayName'], - 'RestartNeeded': out['FeatureResult'][0]['RestartNeeded'], - 'Success': out['Success']} + ret = {'ExitCode': out['ExitCode'], + 'RestartNeeded': False, + 'Restarted': False, + 'Features': {}, + 'Success': out['Success']} + + # FeatureResult is a list of dicts, so each item is a dict + for item in out['FeatureResult']: + ret['Features'][item['Name']] = { + 'DisplayName': item['DisplayName'], + 'Message': item['Message'], + 'RestartNeeded': item['RestartNeeded'], + 'SkipReason': item['SkipReason'], + 'Success': item['Success'] + } + + if item['RestartNeeded']: + ret['RestartNeeded'] = True + + # Only items that installed are in the list of dictionaries + # Add 'Already installed' for features that aren't in the list of dicts + for item in feature.split(','): + if item not in ret['Features']: + ret['Features'][item] = {'Message': 'Already installed'} + + # Some items in the exclude list were removed after installation + # Show what was done, update the dict + if exclude is not None: + # Features is a dict, so it only iterates over the keys + for item in removed['Features']: + if item in ret['Features']: + ret['Features'][item] = { + 'Message': 'Removed after installation (exclude)', + 'DisplayName': removed['Features'][item]['DisplayName'], + 'RestartNeeded': removed['Features'][item]['RestartNeeded'], + 'SkipReason': removed['Features'][item]['SkipReason'], + 'Success': removed['Features'][item]['Success'] + } + + # Exclude items might need a restart + if removed['Features'][item]['RestartNeeded']: + ret['RestartNeeded'] = True + + # Restart here if needed + if restart: + if ret['RestartNeeded']: + if __salt__['system.restart'](in_seconds=True): + ret['Restarted'] = True + + return ret + else: - return {'ExitCode': out['ExitCode'], - 'DisplayName': '{0} (already installed)'.format(feature), - 'RestartNeeded': False, - 'Success': out['Success']} + + # If we get here then all features were already installed + ret = {'ExitCode': out['ExitCode'], + 'Features': {}, + 'RestartNeeded': False, + 'Restarted': False, + 'Success': out['Success']} + + for item in feature.split(','): + ret['Features'][item] = {'Message': 'Already installed'} + + return ret def remove(feature, remove_payload=False, restart=False): @@ -221,17 +290,26 @@ def remove(feature, remove_payload=False, restart=False): take a while to complete installation/uninstallation, so it is a good idea to use the ``-t`` option to set a longer timeout. - :param str feature: The name of the feature to remove + Args: - :param bool remove_payload: True will cause the feature to be removed from - the side-by-side store (``%SystemDrive%:\Windows\WinSxS``). Default is - False + feature (str, list): + The name of the feature(s) to remove. This can be a single feature, + a string of features in a comma delimited list (no spaces), or a + list of features. - :param bool restart: Restarts the computer when uninstall is complete, if - required by the role/feature removed. Default is False + .. versionadded:: Oxygen + Added the ability to pass a list of features to be removed. - :return: A dictionary containing the results of the uninstall - :rtype: dict + remove_payload (Optional[bool]): + True will cause the feature to be removed from the side-by-side + store (``%SystemDrive%:\Windows\WinSxS``). Default is False + + restart (Optional[bool]): + Restarts the computer when uninstall is complete, if required by the + role/feature removed. Default is False + + Returns: + dict: A dictionary containing the results of the uninstall CLI Example: @@ -239,31 +317,67 @@ def remove(feature, remove_payload=False, restart=False): salt -t 600 '*' win_servermanager.remove Telnet-Client ''' - mgmt_tools = '' - if salt.utils.version_cmp(__grains__['osversion'], '6.2') >= 0: - mgmt_tools = '-IncludeManagementTools' + # If it is a list of features, make it a comma delimited string + if isinstance(feature, list): + feature = ','.join(feature) - rmv = '' - if remove_payload: - rmv = '-Remove' + # Use Uninstall-WindowsFeature on Windows 2012 (osversion 6.2) and later + # minions. Default to Remove-WindowsFeature for earlier releases of Windows. + # The newer command makes management tools optional so add them for parity + # with old behavior. + command = 'Remove-WindowsFeature' + management_tools = '' + _remove_payload = '' + if salt.utils.versions.version_cmp(__grains__['osversion'], '6.2') >= 0: + command = 'Uninstall-WindowsFeature' + management_tools = '-IncludeManagementTools' - rst = '' - if restart: - rst = '-Restart' + # Only available with the `Uninstall-WindowsFeature` command + if remove_payload: + _remove_payload = '-Remove' - cmd = 'Remove-WindowsFeature -Name {0} {1} {2} {3} ' \ - '-ErrorAction SilentlyContinue ' \ + cmd = '{0} -Name {1} {2} {3} {4} ' \ '-WarningAction SilentlyContinue'\ - .format(_cmd_quote(feature), mgmt_tools, rmv, rst) + .format(command, _cmd_quote(feature), management_tools, + _remove_payload, + '-Restart' if restart else '') out = _pshell_json(cmd) + # Results are stored in a list of dictionaries in `FeatureResult` if out['FeatureResult']: - return {'ExitCode': out['ExitCode'], - 'DisplayName': out['FeatureResult'][0]['DisplayName'], - 'RestartNeeded': out['FeatureResult'][0]['RestartNeeded'], - 'Success': out['Success']} + ret = {'ExitCode': out['ExitCode'], + 'RestartNeeded': False, + 'Restarted': False, + 'Features': {}, + 'Success': out['Success']} + + for item in out['FeatureResult']: + ret['Features'][item['Name']] = { + 'DisplayName': item['DisplayName'], + 'Message': item['Message'], + 'RestartNeeded': item['RestartNeeded'], + 'SkipReason': item['SkipReason'], + 'Success': item['Success'] + } + + # Only items that installed are in the list of dictionaries + # Add 'Not installed' for features that aren't in the list of dicts + for item in feature.split(','): + if item not in ret['Features']: + ret['Features'][item] = {'Message': 'Not installed'} + + return ret + else: - return {'ExitCode': out['ExitCode'], - 'DisplayName': '{0} (not installed)'.format(feature), - 'RestartNeeded': False, - 'Success': out['Success']} + + # If we get here then none of the features were installed + ret = {'ExitCode': out['ExitCode'], + 'Features': {}, + 'RestartNeeded': False, + 'Restarted': False, + 'Success': out['Success']} + + for item in feature.split(','): + ret['Features'][item] = {'Message': 'Not installed'} + + return ret diff --git a/salt/modules/win_service.py b/salt/modules/win_service.py index 6fdf7580783..8c017c89e71 100644 --- a/salt/modules/win_service.py +++ b/salt/modules/win_service.py @@ -5,13 +5,15 @@ Windows Service module. .. versionchanged:: 2016.11.0 - Rewritten to use PyWin32 ''' -# Import python libs +# Import Python libs from __future__ import absolute_import -import salt.utils -import time -import logging import fnmatch +import logging import re +import time + +# Import Salt libs +import salt.utils.platform from salt.exceptions import CommandExecutionError # Import 3rd party libs @@ -93,7 +95,7 @@ def __virtual__(): ''' Only works on Windows systems with PyWin32 installed ''' - if not salt.utils.is_windows(): + if not salt.utils.platform.is_windows(): return False, 'Module win_service: module only works on Windows.' if not HAS_WIN32_MODS: @@ -811,13 +813,27 @@ def modify(name, return changes -def enable(name, **kwargs): +def enable(name, start_type='auto', start_delayed=False, **kwargs): ''' Enable the named service to start at boot Args: name (str): The name of the service to enable. + start_type (str): Specifies the service start type. Valid options are as + follows: + + - boot: Device driver that is loaded by the boot loader + - system: Device driver that is started during kernel initialization + - auto: Service that automatically starts + - manual: Service must be started manually + - disabled: Service cannot be started + + start_delayed (bool): Set the service to Auto(Delayed Start). Only valid + if the start_type is set to ``Auto``. If service_type is not passed, + but the service is already set to ``Auto``, then the flag will be + set. + Returns: bool: ``True`` if successful, ``False`` otherwise @@ -827,8 +843,13 @@ def enable(name, **kwargs): salt '*' service.enable ''' - modify(name, start_type='Auto') - return info(name)['StartType'] == 'Auto' + + modify(name, start_type=start_type, start_delayed=start_delayed) + svcstat = info(name) + if start_type.lower() == 'auto': + return svcstat['StartType'].lower() == start_type.lower() and svcstat['StartTypeDelayed'] == start_delayed + else: + return svcstat['StartType'].lower() == start_type.lower() def disable(name, **kwargs): diff --git a/salt/modules/win_shadow.py b/salt/modules/win_shadow.py index dfbaa691c0e..c6318bfcbe1 100644 --- a/salt/modules/win_shadow.py +++ b/salt/modules/win_shadow.py @@ -8,9 +8,11 @@ Manage the shadow file *'shadow.info' is not available*), see :ref:`here `. ''' +# Import Python libs from __future__ import absolute_import -import salt.utils +# Import Salt libs +import salt.utils.platform # Define the module's virtual name __virtualname__ = 'shadow' @@ -20,7 +22,7 @@ def __virtual__(): ''' Only works on Windows systems ''' - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): return __virtualname__ return (False, 'Module win_shadow: module only works on Windows systems.') diff --git a/salt/modules/win_smtp_server.py b/salt/modules/win_smtp_server.py index 4ccdf7e7bf1..399d7f085f9 100644 --- a/salt/modules/win_smtp_server.py +++ b/salt/modules/win_smtp_server.py @@ -23,12 +23,13 @@ from __future__ import absolute_import import logging import re -# Import 3rd-party libs -import salt.ext.six as six - -# Import salt libs +# Import Salt libs from salt.exceptions import SaltInvocationError -import salt.utils +import salt.utils.args +import salt.utils.platform + +# Import 3rd-party libs +from salt.ext import six try: import wmi @@ -49,7 +50,7 @@ def __virtual__(): ''' Only works on Windows systems. ''' - if salt.utils.is_windows() and _HAS_MODULE_DEPENDENCIES: + if salt.utils.platform.is_windows() and _HAS_MODULE_DEPENDENCIES: return __virtualname__ return False @@ -102,7 +103,7 @@ def _normalize_server_settings(**settings): Convert setting values that had been improperly converted to a dict back to a string. ''' ret = dict() - settings = salt.utils.clean_kwargs(**settings) + settings = salt.utils.args.clean_kwargs(**settings) for setting in settings: if isinstance(settings[setting], dict): diff --git a/salt/modules/win_snmp.py b/salt/modules/win_snmp.py index 15653a75107..52ce41e4acf 100644 --- a/salt/modules/win_snmp.py +++ b/salt/modules/win_snmp.py @@ -3,14 +3,13 @@ Module for managing SNMP service settings on Windows servers. The Windows feature 'SNMP-Service' must be installed. ''' - -# Import python libs +# Import Python libs from __future__ import absolute_import import logging -# Import salt libs +# Import Salt libs +import salt.utils.platform from salt.exceptions import SaltInvocationError -import salt.utils # Import 3rd party libs from salt.ext import six @@ -42,7 +41,7 @@ def __virtual__(): ''' Only works on Windows systems. ''' - if not salt.utils.is_windows(): + if not salt.utils.platform.is_windows(): return False, 'Module win_snmp: Requires Windows' if not __salt__['reg.read_value'](_HKEY, _SNMP_KEY)['success']: @@ -303,6 +302,11 @@ def get_community_names(): # Windows SNMP service GUI. if isinstance(current_values, list): for current_value in current_values: + + # Ignore error values + if not isinstance(current_value, dict): + continue + permissions = str() for permission_name in _PERMISSION_TYPES: if current_value['vdata'] == _PERMISSION_TYPES[permission_name]: diff --git a/salt/modules/win_status.py b/salt/modules/win_status.py index 7197152a066..9accd79ce82 100644 --- a/salt/modules/win_status.py +++ b/salt/modules/win_status.py @@ -18,8 +18,9 @@ import subprocess log = logging.getLogger(__name__) # Import Salt Libs -import salt.utils import salt.utils.event +import salt.utils.platform +import salt.utils.stringutils from salt.utils.network import host_to_ips as _host_to_ips from salt.utils import namespaced_function as _namespaced_function @@ -31,7 +32,7 @@ import copy # Import 3rd Party Libs try: - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): import wmi import salt.utils.winapi HAS_WMI = True @@ -41,7 +42,7 @@ except ImportError: HAS_WMI = False HAS_PSUTIL = False -if salt.utils.is_windows(): +if salt.utils.platform.is_windows(): import psutil HAS_PSUTIL = True @@ -53,7 +54,7 @@ def __virtual__(): ''' Only works on Windows systems with WMI and WinAPI ''' - if not salt.utils.is_windows(): + if not salt.utils.platform.is_windows(): return False, 'win_status.py: Requires Windows' if not HAS_WMI: @@ -215,8 +216,8 @@ def _get_process_info(proc): ''' Return process information ''' - cmd = salt.utils.to_str(proc.CommandLine or '') - name = salt.utils.to_str(proc.Name) + cmd = salt.utils.stringutils.to_str(proc.CommandLine or '') + name = salt.utils.stringutils.to_str(proc.Name) info = dict( cmd=cmd, name=name, @@ -230,13 +231,13 @@ def _get_process_owner(process): domain, error_code, user = None, None, None try: domain, error_code, user = process.GetOwner() - owner['user'] = salt.utils.to_str(user) - owner['user_domain'] = salt.utils.to_str(domain) + owner['user'] = salt.utils.stringutils.to_str(user) + owner['user_domain'] = salt.utils.stringutils.to_str(domain) except Exception as exc: pass if not error_code and all((user, domain)): - owner['user'] = salt.utils.to_str(user) - owner['user_domain'] = salt.utils.to_str(domain) + owner['user'] = salt.utils.stringutils.to_str(user) + owner['user_domain'] = salt.utils.stringutils.to_str(domain) elif process.ProcessId in [0, 4] and error_code == 2: # Access Denied for System Idle Process and System owner['user'] = 'SYSTEM' @@ -303,7 +304,7 @@ def master(master=None, connected=True): log.error('Failed netstat') raise - lines = salt.utils.to_str(data).split('\n') + lines = salt.utils.stringutils.to_str(data).split('\n') for line in lines: if 'ESTABLISHED' not in line: continue diff --git a/salt/modules/win_system.py b/salt/modules/win_system.py index a92f0923e55..15652d8493e 100644 --- a/salt/modules/win_system.py +++ b/salt/modules/win_system.py @@ -14,13 +14,20 @@ Support for reboot, shutdown, etc ''' from __future__ import absolute_import -# Import python libs +# Import Python libs +import ctypes import logging import time -import ctypes from datetime import datetime -# Import 3rd Party Libs +# Import salt libs +import salt.utils +import salt.utils.locales +import salt.utils.platform +from salt.exceptions import CommandExecutionError + +# Import 3rd-party Libs +from salt.ext import six try: import pythoncom import wmi @@ -33,11 +40,6 @@ try: except ImportError: HAS_WIN32NET_MODS = False -# Import salt libs -import salt.utils -import salt.utils.locales -import salt.ext.six as six - # Set up logging log = logging.getLogger(__name__) @@ -49,7 +51,7 @@ def __virtual__(): ''' Only works on Windows Systems with Win32 Modules ''' - if not salt.utils.is_windows(): + if not salt.utils.platform.is_windows(): return False, 'Module win_system: Requires Windows' if not HAS_WIN32NET_MODS: @@ -273,7 +275,7 @@ def shutdown(message=None, timeout=5, force_close=True, reboot=False, # pylint: if only_on_pending_reboot and not get_pending_reboot(): return False - if message and not isinstance(message, str): + if message and not isinstance(message, six.string_types): message = message.decode('utf-8') try: win32api.InitiateSystemShutdown('127.0.0.1', message, timeout, @@ -622,7 +624,11 @@ def join_domain(domain, .. versionadded:: 2015.8.2/2015.5.7 Returns: - dict: Dictionary if successful, otherwise False + dict: Dictionary if successful + + Raises: + CommandExecutionError: Raises an error if _join_domain returns anything + other than 0 CLI Example: @@ -651,10 +657,60 @@ def join_domain(domain, return 'Must specify a password if you pass a username' # remove any escape characters - if isinstance(account_ou, str): + if isinstance(account_ou, six.string_types): account_ou = account_ou.split('\\') account_ou = ''.join(account_ou) + err = _join_domain(domain=domain, username=username, password=password, + account_ou=account_ou, account_exists=account_exists) + + if not err: + ret = {'Domain': domain, + 'Restart': False} + if restart: + ret['Restart'] = reboot() + return ret + + raise CommandExecutionError(win32api.FormatMessage(err).rstrip()) + + +def _join_domain(domain, + username=None, + password=None, + account_ou=None, + account_exists=False): + ''' + Helper function to join the domain. + + Args: + domain (str): The domain to which the computer should be joined, e.g. + ``example.com`` + + username (str): Username of an account which is authorized to join + computers to the specified domain. Need to be either fully qualified + like ``user@domain.tld`` or simply ``user`` + + password (str): Password of the specified user + + account_ou (str): The DN of the OU below which the account for this + computer should be created when joining the domain, e.g. + ``ou=computers,ou=departm_432,dc=my-company,dc=com`` + + account_exists (bool): If set to ``True`` the computer will only join + the domain if the account already exists. If set to ``False`` the + computer account will be created if it does not exist, otherwise it + will use the existing account. Default is False. + + Returns: + int: + + :param domain: + :param username: + :param password: + :param account_ou: + :param account_exists: + :return: + ''' NETSETUP_JOIN_DOMAIN = 0x1 # pylint: disable=invalid-name NETSETUP_ACCOUNT_CREATE = 0x2 # pylint: disable=invalid-name NETSETUP_DOMAIN_JOIN_IF_JOINED = 0x20 # pylint: disable=invalid-name @@ -670,23 +726,13 @@ def join_domain(domain, pythoncom.CoInitialize() conn = wmi.WMI() comp = conn.Win32_ComputerSystem()[0] - err = comp.JoinDomainOrWorkgroup(Name=domain, - Password=password, - UserName=username, - AccountOU=account_ou, - FJoinOptions=join_options) - # you have to do this because JoinDomainOrWorkgroup returns a strangely - # formatted value that looks like (0,) - if not err[0]: - ret = {'Domain': domain, - 'Restart': False} - if restart: - ret['Restart'] = reboot() - return ret - - log.error(win32api.FormatMessage(err[0]).rstrip()) - return False + # Return the results of the command as an error + # JoinDomainOrWorkgroup returns a strangely formatted value that looks like + # (0,) so return the first item + return comp.JoinDomainOrWorkgroup( + Name=domain, Password=password, UserName=username, AccountOU=account_ou, + FJoinOptions=join_options)[0] def unjoin_domain(username=None, @@ -919,7 +965,11 @@ def set_system_date_time(years=None, seconds (int): Seconds digit: 0 - 59 Returns: - bool: True if successful, otherwise False. + bool: True if successful + + Raises: + CommandExecutionError: Raises an error if ``SetLocalTime`` function + fails CLI Example: @@ -972,12 +1022,15 @@ def set_system_date_time(years=None, system_time.wSecond = int(seconds) system_time_ptr = ctypes.pointer(system_time) succeeded = ctypes.windll.kernel32.SetLocalTime(system_time_ptr) - return succeeded is not 0 - except OSError: + if succeeded is not 0: + return True + else: + log.error('Failed to set local time') + raise CommandExecutionError( + win32api.FormatMessage(succeeded).rstrip()) + except OSError as err: log.error('Failed to set local time') - return False - - return True + raise CommandExecutionError(err) def get_system_date(): diff --git a/salt/modules/win_task.py b/salt/modules/win_task.py index 22285223e4f..c41410adeab 100644 --- a/salt/modules/win_task.py +++ b/salt/modules/win_task.py @@ -9,15 +9,16 @@ You can add and edit existing tasks. You can add and clear triggers and actions. You can list all tasks, folders, triggers, and actions. ''' - -# Import python libs +# Import Python libs from __future__ import absolute_import -import salt.utils -from datetime import datetime import logging import time +from datetime import datetime -# Import 3rd Party Libraries +# Import Salt libs +import salt.utils.platform + +# Import 3rd-party libraries try: import pythoncom import win32com.client @@ -161,7 +162,7 @@ def __virtual__(): ''' Only works on Windows systems ''' - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): if not HAS_DEPENDENCIES: log.warning('Could not load dependencies for {0}'.format(__virtualname__)) return __virtualname__ @@ -1259,7 +1260,7 @@ def status(name, location='\\'): task_service = win32com.client.Dispatch("Schedule.Service") task_service.Connect() - # get the folder to delete the folder from + # get the folder where the task is defined task_folder = task_service.GetFolder(location) task = task_folder.GetTask(name) diff --git a/salt/modules/win_timezone.py b/salt/modules/win_timezone.py index 5ad07d9a207..4c7b4d2881f 100644 --- a/salt/modules/win_timezone.py +++ b/salt/modules/win_timezone.py @@ -4,11 +4,14 @@ Module for managing timezone on Windows systems. ''' from __future__ import absolute_import -# Import python libs -import salt.utils +# Import Python libs import logging import re +# Import Salt libs +import salt.utils.path +import salt.utils.platform + log = logging.getLogger(__name__) # Maybe put in a different file ... ? %-0 @@ -458,7 +461,7 @@ def __virtual__(): ''' Only load on windows ''' - if salt.utils.is_windows() and salt.utils.which('tzutil'): + if salt.utils.platform.is_windows() and salt.utils.path.which('tzutil'): return __virtualname__ return (False, "Module win_timezone: tzutil not found or is not on Windows client") diff --git a/salt/modules/win_update.py b/salt/modules/win_update.py index 3226683caec..4b75e66a825 100644 --- a/salt/modules/win_update.py +++ b/salt/modules/win_update.py @@ -74,9 +74,10 @@ except ImportError: HAS_DEPENDENCIES = False # pylint: enable=import-error -# Import salt libs -import salt.utils +# Import Salt libs +import salt.utils.platform import salt.utils.locales +import salt.utils.versions log = logging.getLogger(__name__) @@ -85,8 +86,8 @@ def __virtual__(): ''' Only works on Windows systems ''' - if salt.utils.is_windows() and HAS_DEPENDENCIES: - salt.utils.warn_until( + if salt.utils.platform.is_windows() and HAS_DEPENDENCIES: + salt.utils.versions.warn_until( 'Fluorine', 'The \'win_update\' module is being deprecated and will be removed ' 'in Salt {version}. Please use the \'win_wua\' module instead.' @@ -383,7 +384,7 @@ class PyWinUpdater(object): update_dict = {} for f in update_com_fields: v = getattr(update, f) - if not any([isinstance(v, bool), isinstance(v, str)]): + if not any([isinstance(v, bool), isinstance(v, six.string_types)]): # Fields that require special evaluation. if f in simple_enums: v = [x for x in v] diff --git a/salt/modules/win_useradd.py b/salt/modules/win_useradd.py index 50aebade96c..9c11e312266 100644 --- a/salt/modules/win_useradd.py +++ b/salt/modules/win_useradd.py @@ -23,21 +23,24 @@ Module for managing Windows Users .. note:: This currently only works with local user accounts, not domain accounts ''' +# Import Python libs from __future__ import absolute_import -from datetime import datetime +import logging import time +from datetime import datetime try: from shlex import quote as _cmd_quote # pylint: disable=E0611 except: # pylint: disable=W0702 from pipes import quote as _cmd_quote -# Import salt libs +# Import Salt libs import salt.utils +import salt.utils.args +import salt.utils.platform from salt.ext import six from salt.ext.six import string_types from salt.exceptions import CommandExecutionError -import logging log = logging.getLogger(__name__) @@ -64,7 +67,7 @@ def __virtual__(): ''' Requires Windows and Windows Modules ''' - if not salt.utils.is_windows(): + if not salt.utils.platform.is_windows(): return False, 'Module win_useradd: Windows Only' if not HAS_WIN32NET_MODS: @@ -587,10 +590,10 @@ def chhome(name, home, **kwargs): name = _to_unicode(name) home = _to_unicode(home) - kwargs = salt.utils.clean_kwargs(**kwargs) + kwargs = salt.utils.args.clean_kwargs(**kwargs) persist = kwargs.pop('persist', False) if kwargs: - salt.utils.invalid_kwargs(kwargs) + salt.utils.args.invalid_kwargs(kwargs) if persist: log.info('Ignoring unsupported \'persist\' argument to user.chhome') diff --git a/salt/modules/win_wua.py b/salt/modules/win_wua.py index 638452f2ae9..5549b3e2bfa 100644 --- a/salt/modules/win_wua.py +++ b/salt/modules/win_wua.py @@ -2,24 +2,67 @@ ''' Module for managing Windows Updates using the Windows Update Agent. +List updates on the system using the following functions: + +- :ref:`available` +- :ref:`list` + +This is an easy way to find additional information about updates available to +to the system, such as the GUID, KB number, or description. + +Once you have the GUID or a KB number for the update you can get information +about the update, download, install, or uninstall it using these functions: + +- :ref:`get` +- :ref:`download` +- :ref:`install` +- :ref:`uninstall` + +The get function expects a name in the form of a GUID, KB, or Title and should +return information about a single update. The other functions accept either a +single item or a list of items for downloading/installing/uninstalling a +specific list of items. + +The :ref:`list` and :ref:`get` functions are utility functions. In addition to +returning information about updates they can also download and install updates +by setting ``download=True`` or ``install=True``. So, with :ref:`list` for +example, you could run the function with the filters you want to see what is +available. Then just add ``install=True`` to install everything on that list. + +If you want to download, install, or uninstall specific updates, use +:ref:`download`, :ref:`install`, or :ref:`uninstall`. To update your system +with the latest updates use :ref:`list` and set ``install=True`` + +You can also adjust the Windows Update settings using the :ref:`set_wu_settings` +function. This function is only supported on the following operating systems: + +- Windows Vista / Server 2008 +- Windows 7 / Server 2008R2 +- Windows 8 / Server 2012 +- Windows 8.1 / Server 2012R2 + +As of Windows 10 and Windows Server 2016, the ability to modify the Windows +Update settings has been restricted. The settings can be modified in the Local +Group Policy using the ``lgpo`` module. + .. versionadded:: 2015.8.0 :depends: - salt.utils.win_update ''' - # Import Python libs from __future__ import absolute_import from __future__ import unicode_literals import logging # Import Salt libs -from salt.ext import six -import salt.utils +import salt.utils.platform +import salt.utils.versions import salt.utils.win_update from salt.exceptions import CommandExecutionError # Import 3rd-party libs +from salt.ext import six try: import pythoncom import win32com.client @@ -34,7 +77,7 @@ def __virtual__(): ''' Only works on Windows systems with PyWin32 ''' - if not salt.utils.is_windows(): + if not salt.utils.platform.is_windows(): return False, 'WUA: Only available on Window systems' if not HAS_PYWIN32: @@ -54,36 +97,40 @@ def available(software=True, skip_mandatory=False, skip_reboot=False, categories=None, - severities=None, - ): + severities=None,): ''' .. versionadded:: 2017.7.0 - List updates that match the passed criteria. + List updates that match the passed criteria. This allows for more filter + options than :func:`list`. Good for finding a specific GUID or KB. Args: - software (bool): Include software updates in the results (default is - True) + software (bool): + Include software updates in the results (default is True) - drivers (bool): Include driver updates in the results (default is False) + drivers (bool): + Include driver updates in the results (default is False) summary (bool): - - True: Return a summary of updates available for each category. - - False (default): Return a detailed list of available updates. + - True: Return a summary of updates available for each category. + - False (default): Return a detailed list of available updates. - skip_installed (bool): Skip updates that are already installed. Default - is False. + skip_installed (bool): + Skip updates that are already installed. Default is False. - skip_hidden (bool): Skip updates that have been hidden. Default is True. + skip_hidden (bool): + Skip updates that have been hidden. Default is True. - skip_mandatory (bool): Skip mandatory updates. Default is False. + skip_mandatory (bool): + Skip mandatory updates. Default is False. - skip_reboot (bool): Skip updates that require a reboot. Default is - False. + skip_reboot (bool): + Skip updates that require a reboot. Default is False. - categories (list): Specify the categories to list. Must be passed as a - list. All categories returned by default. + categories (list): + Specify the categories to list. Must be passed as a list. All + categories returned by default. Categories include the following: @@ -101,8 +148,9 @@ def available(software=True, * Windows 8.1 and later drivers * Windows Defender - severities (list): Specify the severities to include. Must be passed as - a list. All severities returned by default. + severities (list): + Specify the severities to include. Must be passed as a list. All + severities returned by default. Severities include the following: @@ -152,28 +200,30 @@ def available(software=True, salt '*' win_wua.available # List all updates with categories of Critical Updates and Drivers - salt '*' win_wua.available categories=['Critical Updates','Drivers'] + salt '*' win_wua.available categories=["Critical Updates","Drivers"] # List all Critical Security Updates - salt '*' win_wua.available categories=['Security Updates'] severities=['Critical'] + salt '*' win_wua.available categories=["Security Updates"] severities=["Critical"] # List all updates with a severity of Critical - salt '*' win_wua.available severities=['Critical'] + salt '*' win_wua.available severities=["Critical"] # A summary of all available updates salt '*' win_wua.available summary=True # A summary of all Feature Packs and Windows 8.1 Updates - salt '*' win_wua.available categories=['Feature Packs','Windows 8.1'] summary=True + salt '*' win_wua.available categories=["Feature Packs","Windows 8.1"] summary=True ''' # Create a Windows Update Agent instance wua = salt.utils.win_update.WindowsUpdateAgent() # Look for available - updates = wua.available(skip_hidden, skip_installed, skip_mandatory, - skip_reboot, software, drivers, categories, - severities) + updates = wua.available( + skip_hidden=skip_hidden, skip_installed=skip_installed, + skip_mandatory=skip_mandatory, skip_reboot=skip_reboot, + software=software, drivers=drivers, categories=categories, + severities=severities) # Return results as Summary or Details return updates.summary() if summary else updates.list() @@ -183,23 +233,29 @@ def list_update(name, download=False, install=False): ''' .. deprecated:: 2017.7.0 Use :func:`get` instead + Returns details for all updates that match the search criteria Args: - name (str): The name of the update you're searching for. This can be the - GUID, a KB number, or any part of the name of the update. GUIDs and - KBs are preferred. Run ``list_updates`` to get the GUID for the update - you're looking for. - download (bool): Download the update returned by this function. Run this - function first to see if the update exists, then set ``download=True`` - to download the update. + name (str): + The name of the update you're searching for. This can be the GUID, a + KB number, or any part of the name of the update. GUIDs and KBs are + preferred. Run ``list_updates`` to get the GUID for the update + you're looking for. - install (bool): Install the update returned by this function. Run this - function first to see if the update exists, then set ``install=True`` to - install the update. + download (bool): + Download the update returned by this function. Run this function + first to see if the update exists, then set ``download=True`` to + download the update. + + install (bool): + Install the update returned by this function. Run this function + first to see if the update exists, then set ``install=True`` to + install the update. Returns: + dict: Returns a dict containing a list of updates that match the name if download and install are both set to False. Should usually be a single update, but can return multiple if a partial name is given. @@ -247,7 +303,7 @@ def list_update(name, download=False, install=False): # Not all updates have an associated KB salt '*' win_wua.list_update 'Microsoft Camera Codec Pack' ''' - salt.utils.warn_until( + salt.utils.versions.warn_until( 'Fluorine', 'This function is replaced by \'get\' as of Salt 2017.7.0. This' 'warning will be removed in Salt Fluorine.') @@ -258,23 +314,28 @@ def get(name, download=False, install=False): ''' .. versionadded:: 2017.7.0 - Returns details for all updates that match the search criteria + Returns details for the named update Args: - name (str): The name of the update you're searching for. This can be the - GUID, a KB number, or any part of the name of the update. GUIDs and - KBs are preferred. Run ``list`` to get the GUID for the update - you're looking for. - download (bool): Download the update returned by this function. Run this - function first to see if the update exists, then set ``download=True`` - to download the update. + name (str): + The name of the update you're searching for. This can be the GUID, a + KB number, or any part of the name of the update. GUIDs and KBs are + preferred. Run ``list`` to get the GUID for the update you're + looking for. - install (bool): Install the update returned by this function. Run this - function first to see if the update exists, then set ``install=True`` to - install the update. + download (bool): + Download the update returned by this function. Run this function + first to see if the update exists, then set ``download=True`` to + download the update. + + install (bool): + Install the update returned by this function. Run this function + first to see if the update exists, then set ``install=True`` to + install the update. Returns: + dict: Returns a dict containing a list of updates that match the name if download and install are both set to False. Should usually be a single update, but can return multiple if a partial name is given. @@ -357,30 +418,35 @@ def list_updates(software=True, install is True the same list will be downloaded and/or installed. Args: - software (bool): Include software updates in the results (default is - True) - drivers (bool): Include driver updates in the results (default is False) + software (bool): + Include software updates in the results (default is True) + + drivers (bool): + Include driver updates in the results (default is False) summary (bool): - - True: Return a summary of updates available for each category. - - False (default): Return a detailed list of available updates. + - True: Return a summary of updates available for each category. + - False (default): Return a detailed list of available updates. - skip_installed (bool): Skip installed updates in the results (default is - False) + skip_installed (bool): + Skip installed updates in the results (default is False) - download (bool): (Overrides reporting functionality) Download the list - of updates returned by this function. Run this function first with - ``download=False`` to see what will be downloaded, then set - ``download=True`` to download the updates. + download (bool): + (Overrides reporting functionality) Download the list of updates + returned by this function. Run this function first with + ``download=False`` to see what will be downloaded, then set + ``download=True`` to download the updates. - install (bool): (Overrides reporting functionality) Install the list of - updates returned by this function. Run this function first with - ``install=False`` to see what will be installed, then set - ``install=True`` to install the updates. + install (bool): + (Overrides reporting functionality) Install the list of updates + returned by this function. Run this function first with + ``install=False`` to see what will be installed, then set + ``install=True`` to install the updates. - categories (list): Specify the categories to list. Must be passed as a - list. All categories returned by default. + categories (list): + Specify the categories to list. Must be passed as a list. All + categories returned by default. Categories include the following: @@ -398,8 +464,9 @@ def list_updates(software=True, * Windows 8.1 and later drivers * Windows Defender - severities (list): Specify the severities to include. Must be passed as - a list. All severities returned by default. + severities (list): + Specify the severities to include. Must be passed as a list. All + severities returned by default. Severities include the following: @@ -463,7 +530,7 @@ def list_updates(software=True, # A summary of all Feature Packs and Windows 8.1 Updates salt '*' win_wua.list_updates categories=['Feature Packs','Windows 8.1'] summary=True ''' - salt.utils.warn_until( + salt.utils.versions.warn_until( 'Fluorine', 'This function is replaced by \'list\' as of Salt 2017.7.0. This' 'warning will be removed in Salt Fluorine.') @@ -486,30 +553,35 @@ def list(software=True, install is True the same list will be downloaded and/or installed. Args: - software (bool): Include software updates in the results (default is - True) - drivers (bool): Include driver updates in the results (default is False) + software (bool): + Include software updates in the results (default is True) + + drivers (bool): + Include driver updates in the results (default is False) summary (bool): - - True: Return a summary of updates available for each category. - - False (default): Return a detailed list of available updates. + - True: Return a summary of updates available for each category. + - False (default): Return a detailed list of available updates. - skip_installed (bool): Skip installed updates in the results (default is - False) + skip_installed (bool): + Skip installed updates in the results (default is False) - download (bool): (Overrides reporting functionality) Download the list - of updates returned by this function. Run this function first with - ``download=False`` to see what will be downloaded, then set - ``download=True`` to download the updates. + download (bool): + (Overrides reporting functionality) Download the list of updates + returned by this function. Run this function first with + ``download=False`` to see what will be downloaded, then set + ``download=True`` to download the updates. - install (bool): (Overrides reporting functionality) Install the list of - updates returned by this function. Run this function first with - ``install=False`` to see what will be installed, then set - ``install=True`` to install the updates. + install (bool): + (Overrides reporting functionality) Install the list of updates + returned by this function. Run this function first with + ``install=False`` to see what will be installed, then set + ``install=True`` to install the updates. - categories (list): Specify the categories to list. Must be passed as a - list. All categories returned by default. + categories (list): + Specify the categories to list. Must be passed as a list. All + categories returned by default. Categories include the following: @@ -527,8 +599,9 @@ def list(software=True, * Windows 8.1 and later drivers * Windows Defender - severities (list): Specify the severities to include. Must be passed as - a list. All severities returned by default. + severities (list): + Specify the severities to include. Must be passed as a list. All + severities returned by default. Severities include the following: @@ -575,22 +648,22 @@ def list(software=True, .. code-block:: bash # Normal Usage (list all software updates) - salt '*' win_wua.list_updates + salt '*' win_wua.list # List all updates with categories of Critical Updates and Drivers - salt '*' win_wua.list_updates categories=['Critical Updates','Drivers'] + salt '*' win_wua.list categories=['Critical Updates','Drivers'] # List all Critical Security Updates - salt '*' win_wua.list_updates categories=['Security Updates'] severities=['Critical'] + salt '*' win_wua.list categories=['Security Updates'] severities=['Critical'] # List all updates with a severity of Critical - salt '*' win_wua.list_updates severities=['Critical'] + salt '*' win_wua.list severities=['Critical'] # A summary of all available updates - salt '*' win_wua.list_updates summary=True + salt '*' win_wua.list summary=True # A summary of all Feature Packs and Windows 8.1 Updates - salt '*' win_wua.list_updates categories=['Feature Packs','Windows 8.1'] summary=True + salt '*' win_wua.list categories=['Feature Packs','Windows 8.1'] summary=True ''' # Create a Windows Update Agent instance wua = salt.utils.win_update.WindowsUpdateAgent() @@ -604,11 +677,11 @@ def list(software=True, # Download if download or install: - ret['Download'] = wua.download(updates.updates) + ret['Download'] = wua.download(updates) # Install if install: - ret['Install'] = wua.install(updates.updates) + ret['Install'] = wua.install(updates) if not ret: return updates.summary() if summary else updates.list() @@ -625,13 +698,16 @@ def download_update(name): Args: - name (str): The name of the update to download. This can be a GUID, a KB - number, or any part of the name. To ensure a single item is matched the - GUID is preferred. + name (str): + The name of the update to download. This can be a GUID, a KB number, + or any part of the name. To ensure a single item is matched the GUID + is preferred. - .. note:: If more than one result is returned an error will be raised. + .. note:: + If more than one result is returned an error will be raised. Returns: + dict: A dictionary containing the results of the download CLI Examples: @@ -641,9 +717,8 @@ def download_update(name): salt '*' win_wua.download_update 12345678-abcd-1234-abcd-1234567890ab salt '*' win_wua.download_update KB12312321 - ''' - salt.utils.warn_until( + salt.utils.versions.warn_until( 'Fluorine', 'This function is replaced by \'download\' as of Salt 2017.7.0. This' 'warning will be removed in Salt Fluorine.') @@ -660,8 +735,9 @@ def download_updates(names): Args: - names (list): A list of updates to download. This can be any combination - of GUIDs, KB numbers, or names. GUIDs or KBs are preferred. + names (list): + A list of updates to download. This can be any combination of GUIDs, + KB numbers, or names. GUIDs or KBs are preferred. Returns: @@ -672,9 +748,9 @@ def download_updates(names): .. code-block:: bash # Normal Usage - salt '*' win_wua.download guid=['12345678-abcd-1234-abcd-1234567890ab', 'KB2131233'] + salt '*' win_wua.download_updates guid=['12345678-abcd-1234-abcd-1234567890ab', 'KB2131233'] ''' - salt.utils.warn_until( + salt.utils.versions.warn_until( 'Fluorine', 'This function is replaced by \'download\' as of Salt 2017.7.0. This' 'warning will be removed in Salt Fluorine.') @@ -690,9 +766,14 @@ def download(names): Args: - names (str, list): A single update or a list of updates to download. - This can be any combination of GUIDs, KB numbers, or names. GUIDs or KBs - are preferred. + names (str, list): + A single update or a list of updates to download. This can be any + combination of GUIDs, KB numbers, or names. GUIDs or KBs are + preferred. + + .. note:: + An error will be raised if there are more results than there are items + in the names parameter Returns: @@ -703,7 +784,7 @@ def download(names): .. code-block:: bash # Normal Usage - salt '*' win_wua.download guid=['12345678-abcd-1234-abcd-1234567890ab', 'KB2131233'] + salt '*' win_wua.download names=['12345678-abcd-1234-abcd-1234567890ab', 'KB2131233'] ''' # Create a Windows Update Agent instance wua = salt.utils.win_update.WindowsUpdateAgent() @@ -714,6 +795,13 @@ def download(names): if updates.count() == 0: raise CommandExecutionError('No updates found') + # Make sure it's a list so count comparison is correct + if isinstance(names, six.string_types): + names = [names] + + if isinstance(names, six.integer_types): + names = [str(names)] + if updates.count() > len(names): raise CommandExecutionError('Multiple updates found, names need to be ' 'more specific') @@ -734,10 +822,12 @@ def install_update(name): number, or any part of the name. To ensure a single item is matched the GUID is preferred. - .. note:: If no results or more than one result is returned an error - will be raised. + .. note:: + If no results or more than one result is returned an error will be + raised. Returns: + dict: A dictionary containing the results of the install CLI Examples: @@ -748,7 +838,7 @@ def install_update(name): salt '*' win_wua.install_update KB12312231 ''' - salt.utils.warn_until( + salt.utils.versions.warn_until( 'Fluorine', 'This function is replaced by \'install\' as of Salt 2017.7.0. This' 'warning will be removed in Salt Fluorine.') @@ -779,7 +869,7 @@ def install_updates(names): # Normal Usage salt '*' win_wua.install_updates guid=['12345678-abcd-1234-abcd-1234567890ab', 'KB12323211'] ''' - salt.utils.warn_until( + salt.utils.versions.warn_until( 'Fluorine', 'This function is replaced by \'install\' as of Salt 2017.7.0. This' 'warning will be removed in Salt Fluorine.') @@ -795,9 +885,14 @@ def install(names): Args: - names (str, list): A single update or a list of updates to install. - This can be any combination of GUIDs, KB numbers, or names. GUIDs or KBs - are preferred. + names (str, list): + A single update or a list of updates to install. This can be any + combination of GUIDs, KB numbers, or names. GUIDs or KBs are + preferred. + + .. note:: + An error will be raised if there are more results than there are items + in the names parameter Returns: @@ -808,7 +903,7 @@ def install(names): .. code-block:: bash # Normal Usage - salt '*' win_wua.install_updates guid=['12345678-abcd-1234-abcd-1234567890ab', 'KB12323211'] + salt '*' win_wua.install KB12323211 ''' # Create a Windows Update Agent instance wua = salt.utils.win_update.WindowsUpdateAgent() @@ -819,6 +914,13 @@ def install(names): if updates.count() == 0: raise CommandExecutionError('No updates found') + # Make sure it's a list so count comparison is correct + if isinstance(names, six.string_types): + names = [names] + + if isinstance(names, six.integer_types): + names = [str(names)] + if updates.count() > len(names): raise CommandExecutionError('Multiple updates found, names need to be ' 'more specific') @@ -834,9 +936,10 @@ def uninstall(names): Args: - names (str, list): A single update or a list of updates to uninstall. - This can be any combination of GUIDs, KB numbers, or names. GUIDs or KBs - are preferred. + names (str, list): + A single update or a list of updates to uninstall. This can be any + combination of GUIDs, KB numbers, or names. GUIDs or KBs are + preferred. Returns: @@ -875,33 +978,50 @@ def set_wu_settings(level=None, Change Windows Update settings. If no parameters are passed, the current value will be returned. - :param int level: - Number from 1 to 4 indicating the update level: + Supported: + - Windows Vista / Server 2008 + - Windows 7 / Server 2008R2 + - Windows 8 / Server 2012 + - Windows 8.1 / Server 2012R2 + + .. note: + Microsoft began using the Unified Update Platform (UUP) starting with + Windows 10 / Server 2016. The Windows Update settings have changed and + the ability to 'Save' Windows Update settings has been removed. Windows + Update settings are read-only. See MSDN documentation: + https://msdn.microsoft.com/en-us/library/aa385829(v=vs.85).aspx + + Args: + + level (int): + Number from 1 to 4 indicating the update level: + 1. Never check for updates 2. Check for updates but let me choose whether to download and install them 3. Download updates but let me choose whether to install them 4. Install updates automatically - :param bool recommended: - Boolean value that indicates whether to include optional or recommended - updates when a search for updates and installation of updates is - performed. - :param bool featured: - Boolean value that indicates whether to display notifications for - featured updates. + recommended (bool): + Boolean value that indicates whether to include optional or + recommended updates when a search for updates and installation of + updates is performed. - :param bool elevated: - Boolean value that indicates whether non-administrators can perform some - update-related actions without administrator approval. + featured (bool): + Boolean value that indicates whether to display notifications for + featured updates. - :param bool msupdate: - Boolean value that indicates whether to turn on Microsoft Update for - other Microsoft products + elevated (bool): + Boolean value that indicates whether non-administrators can perform + some update-related actions without administrator approval. + + msupdate (bool): + Boolean value that indicates whether to turn on Microsoft Update for + other Microsoft products + + day (str): + Days of the week on which Automatic Updates installs or uninstalls + updates. Accepted values: - :param str day: - Days of the week on which Automatic Updates installs or uninstalls - updates. - Accepted values: - Everyday - Monday - Tuesday @@ -910,21 +1030,43 @@ def set_wu_settings(level=None, - Friday - Saturday - :param str time: - Time at which Automatic Updates installs or uninstalls updates. Must be - in the ##:## 24hr format, eg. 3:00 PM would be 15:00 + time (str): + Time at which Automatic Updates installs or uninstalls updates. Must + be in the ##:## 24hr format, eg. 3:00 PM would be 15:00. Must be in + 1 hour increments. - :return: Returns a dictionary containing the results. + Returns: + + dict: Returns a dictionary containing the results. CLI Examples: .. code-block:: bash salt '*' win_wua.set_wu_settings level=4 recommended=True featured=False - ''' - ret = {} - ret['Success'] = True + # The AutomaticUpdateSettings.Save() method used in this function does not + # work on Windows 10 / Server 2016. It is called in throughout this function + # like this: + # + # obj_au = win32com.client.Dispatch('Microsoft.Update.AutoUpdate') + # obj_au_settings = obj_au.Settings + # obj_au_settings.Save() + # + # The `Save()` method reports success but doesn't actually change anything. + # Windows Update settings are read-only in Windows 10 / Server 2016. There's + # a little blurb on MSDN that mentions this, but gives no alternative for + # changing these settings in Windows 10 / Server 2016. + # + # https://msdn.microsoft.com/en-us/library/aa385829(v=vs.85).aspx + # + # Apparently the Windows Update framework in Windows Vista - Windows 8.1 has + # been changed quite a bit in Windows 10 / Server 2016. It is now called the + # Unified Update Platform (UUP). I haven't found an API or a Powershell + # commandlet for working with the the UUP. Perhaps there will be something + # forthcoming. The `win_lgpo` module might be an option for changing the + # Windows Update settings using local group policy. + ret = {'Success': True} # Initialize the PyCom system pythoncom.CoInitialize() @@ -1076,30 +1218,31 @@ def get_wu_settings(): Boolean value that indicates whether to display notifications for featured updates. Group Policy Required (Read-only): - Boolean value that indicates whether Group Policy requires the Automatic - Updates service. + Boolean value that indicates whether Group Policy requires the + Automatic Updates service. Microsoft Update: Boolean value that indicates whether to turn on Microsoft Update for other Microsoft Products Needs Reboot: - Boolean value that indicates whether the machine is in a reboot pending - state. + Boolean value that indicates whether the machine is in a reboot + pending state. Non Admins Elevated: - Boolean value that indicates whether non-administrators can perform some - update-related actions without administrator approval. + Boolean value that indicates whether non-administrators can perform + some update-related actions without administrator approval. Notification Level: Number 1 to 4 indicating the update level: 1. Never check for updates - 2. Check for updates but let me choose whether to download and install them + 2. Check for updates but let me choose whether to download and + install them 3. Download updates but let me choose whether to install them 4. Install updates automatically Read Only (Read-only): Boolean value that indicates whether the Automatic Update settings are read-only. Recommended Updates: - Boolean value that indicates whether to include optional or recommended - updates when a search for updates and installation of updates is - performed. + Boolean value that indicates whether to include optional or + recommended updates when a search for updates and installation of + updates is performed. Scheduled Day: Days of the week on which Automatic Updates installs or uninstalls updates. @@ -1182,13 +1325,12 @@ def get_needs_reboot(): Returns: - bool: True if the system requires a reboot, False if not + bool: True if the system requires a reboot, otherwise False CLI Examples: .. code-block:: bash salt '*' win_wua.get_needs_reboot - ''' return salt.utils.win_update.needs_reboot() diff --git a/salt/modules/x509.py b/salt/modules/x509.py index abf3c4c9e4f..7344bc7d6ce 100644 --- a/salt/modules/x509.py +++ b/salt/modules/x509.py @@ -23,9 +23,10 @@ import datetime import ast # Import salt libs -import salt.utils +import salt.utils.files +import salt.utils.path import salt.exceptions -import salt.ext.six as six +from salt.ext import six from salt.utils.odict import OrderedDict # pylint: disable=import-error,redefined-builtin from salt.ext.six.moves import range @@ -165,7 +166,7 @@ def _parse_openssl_req(csr_filename): Parses openssl command line output, this is a workaround for M2Crypto's inability to get them from CSR objects. ''' - if not salt.utils.which('openssl'): + if not salt.utils.path.which('openssl'): raise salt.exceptions.SaltInvocationError( 'openssl binary not found in path' ) @@ -214,7 +215,7 @@ def _parse_openssl_crl(crl_filename): Parses openssl command line output, this is a workaround for M2Crypto's inability to get them from CSR objects. ''' - if not salt.utils.which('openssl'): + if not salt.utils.path.which('openssl'): raise salt.exceptions.SaltInvocationError( 'openssl binary not found in path' ) @@ -315,7 +316,7 @@ def _text_or_file(input_): content to be parsed. ''' if os.path.isfile(input_): - with salt.utils.fopen(input_) as fp_: + with salt.utils.files.fopen(input_) as fp_: return fp_.read() else: return input_ @@ -330,10 +331,14 @@ def _parse_subject(subject): for nid_name, nid_num in six.iteritems(subject.nid): if nid_num in nids: continue - val = getattr(subject, nid_name) - if val: - ret[nid_name] = val - nids.append(nid_num) + try: + val = getattr(subject, nid_name) + if val: + ret[nid_name] = val + nids.append(nid_num) + except TypeError as e: + if e.args and e.args[0] == 'No string argument provided': + pass return ret @@ -768,7 +773,7 @@ def write_pem(text, path, overwrite=True, pem_type=None): _private_key = get_pem_entry(_filecontents, '(?:RSA )?PRIVATE KEY') except salt.exceptions.SaltInvocationError: pass - with salt.utils.fopen(path, 'w') as _fp: + with salt.utils.files.fopen(path, 'w') as _fp: if pem_type and pem_type == 'CERTIFICATE' and _private_key: _fp.write(_private_key) _fp.write(text) @@ -1373,10 +1378,19 @@ def create_certificate( ['listen_in', 'preqrequired', '__prerequired__']: kwargs.pop(ignore, None) - cert_txt = __salt__['publish.publish']( + certs = __salt__['publish.publish']( tgt=ca_server, fun='x509.sign_remote_certificate', - arg=str(kwargs))[ca_server] + arg=str(kwargs)) + + if not any(certs): + raise salt.exceptions.SaltInvocationError( + 'ca_server did not respond' + ' salt master must permit peers to' + ' call the sign_remote_certificate function.') + + cert_txt = certs[ca_server] + if path: return write_pem( text=cert_txt, @@ -1736,7 +1750,7 @@ def verify_crl(crl, cert): salt '*' x509.verify_crl crl=/etc/pki/myca.crl cert=/etc/pki/myca.crt ''' - if not salt.utils.which('openssl'): + if not salt.utils.path.which('openssl'): raise salt.exceptions.SaltInvocationError( 'openssl binary not found in path' ) diff --git a/salt/modules/xapi.py b/salt/modules/xapi.py index c6c3c39d86e..aa031b2f4f8 100644 --- a/salt/modules/xapi.py +++ b/salt/modules/xapi.py @@ -33,9 +33,10 @@ except ImportError: HAS_IMPORTLIB = False # Import salt libs -from salt.exceptions import CommandExecutionError -import salt.utils +import salt.utils.files +import salt.utils.path import salt.modules.cmdmod +from salt.exceptions import CommandExecutionError # Define the module's virtual name __virtualname__ = 'virt' @@ -114,7 +115,7 @@ def _get_xtool(): Internal, returns xl or xm command line path ''' for xtool in ['xl', 'xm']: - path = salt.utils.which(xtool) + path = salt.utils.path.which(xtool) if path is not None: return path @@ -773,7 +774,7 @@ def is_hyper(): # virtual_subtype isn't set everywhere. return False try: - with salt.utils.fopen('/proc/modules') as fp_: + with salt.utils.files.fopen('/proc/modules') as fp_: if 'xen_' not in fp_.read(): return False except (OSError, IOError): diff --git a/salt/modules/xbpspkg.py b/salt/modules/xbpspkg.py index 6b4c9986ecd..732adcf3aab 100644 --- a/salt/modules/xbpspkg.py +++ b/salt/modules/xbpspkg.py @@ -16,7 +16,9 @@ import logging import glob # Import salt libs -import salt.utils +import salt.utils # Can be removed when is_true and compare_dicts are moved +import salt.utils.files +import salt.utils.path import salt.utils.pkg import salt.utils.decorators as decorators from salt.exceptions import CommandExecutionError, MinionError @@ -41,7 +43,7 @@ def _check_xbps(): ''' Looks to see if xbps-install is present on the system, return full path ''' - return salt.utils.which('xbps-install') + return salt.utils.path.which('xbps-install') @decorators.memoize @@ -552,7 +554,7 @@ def _locate_repo_files(repo, rewrite=False): for filename in files: write_buff = [] - with salt.utils.fopen(filename, 'r') as cur_file: + with salt.utils.files.fopen(filename, 'r') as cur_file: for line in cur_file: if regex.match(line): ret_val.append(filename) @@ -560,7 +562,7 @@ def _locate_repo_files(repo, rewrite=False): write_buff.append(line) if rewrite and filename in ret_val: if len(write_buff) > 0: - with salt.utils.fopen(filename, 'w') as rewrite_file: + with salt.utils.files.fopen(filename, 'w') as rewrite_file: rewrite_file.write("".join(write_buff)) else: # Prune empty files os.remove(filename) @@ -588,7 +590,7 @@ def add_repo(repo, conffile='/usr/share/xbps.d/15-saltstack.conf'): if len(_locate_repo_files(repo)) == 0: try: - with salt.utils.fopen(conffile, 'a+') as conf_file: + with salt.utils.files.fopen(conffile, 'a+') as conf_file: conf_file.write('repository='+repo+'\n') except IOError: return False diff --git a/salt/modules/xfs.py b/salt/modules/xfs.py index 87b0eac7caa..65e060b8989 100644 --- a/salt/modules/xfs.py +++ b/salt/modules/xfs.py @@ -33,11 +33,13 @@ import time import logging # Import Salt libs -import salt.utils +import salt.utils.files +import salt.utils.path +import salt.utils.platform from salt.exceptions import CommandExecutionError # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import range # pylint: disable=import-error,no-name-in-module,redefined-builtin log = logging.getLogger(__name__) @@ -47,7 +49,8 @@ def __virtual__(): ''' Only work on POSIX-like systems ''' - return not salt.utils.is_windows() and __grains__.get('kernel') == 'Linux' + return not salt.utils.platform.is_windows() \ + and __grains__.get('kernel') == 'Linux' def _verify_run(out, cmd=None): @@ -183,7 +186,7 @@ def dump(device, destination, level=0, label=None, noerase=None): salt '*' xfs.dump /dev/sda1 /detination/on/the/client label='Company accountancy' salt '*' xfs.dump /dev/sda1 /detination/on/the/client noerase=True ''' - if not salt.utils.which("xfsdump"): + if not salt.utils.path.which("xfsdump"): raise CommandExecutionError("Utility \"xfsdump\" has to be installed or missing.") label = label and label or time.strftime("XFS dump for \"{0}\" of %Y.%m.%d, %H:%M".format(device), @@ -507,7 +510,7 @@ def _get_mounts(): List mounted filesystems. ''' mounts = {} - with salt.utils.fopen("/proc/mounts") as fhr: + with salt.utils.files.fopen("/proc/mounts") as fhr: for line in fhr.readlines(): device, mntpnt, fstype, options, fs_freq, fs_passno = line.strip().split(" ") if fstype != 'xfs': diff --git a/salt/modules/yumpkg.py b/salt/modules/yumpkg.py index 61af4431fcc..e5ddc11e5ac 100644 --- a/salt/modules/yumpkg.py +++ b/salt/modules/yumpkg.py @@ -17,7 +17,6 @@ Support for YUM/DNF # Import python libs from __future__ import absolute_import import contextlib -import copy import datetime import fnmatch import itertools @@ -35,23 +34,31 @@ try: import yum HAS_YUM = True except ImportError: - from salt.ext.six.moves import configparser HAS_YUM = False + +from salt.ext.six.moves import configparser + # pylint: enable=import-error,redefined-builtin -# Import salt libs +# Import Salt libs import salt.utils -import salt.utils.pkg -import salt.ext.six as six +import salt.utils.args +import salt.utils.decorators.path +import salt.utils.files import salt.utils.itertools -import salt.utils.systemd -import salt.utils.decorators as decorators +import salt.utils.lazy +import salt.utils.pkg import salt.utils.pkg.rpm +import salt.utils.systemd +import salt.utils.versions from salt.utils.versions import LooseVersion as _LooseVersion from salt.exceptions import ( CommandExecutionError, MinionError, SaltInvocationError ) +# Import 3rd-party libs +from salt.ext import six + log = logging.getLogger(__name__) __HOLD_PATTERN = r'\w+(?:[.-][^-]+)*' @@ -181,7 +188,16 @@ def _check_versionlock(): Ensure that the appropriate versionlock plugin is present ''' if _yum() == 'dnf': - vl_plugin = 'python-dnf-plugins-extras-versionlock' + if int(__grains__.get('osmajorrelease')) >= 26: + if six.PY3: + vl_plugin = 'python3-dnf-plugin-versionlock' + else: + vl_plugin = 'python2-dnf-plugin-versionlock' + else: + if six.PY3: + vl_plugin = 'python3-dnf-plugins-extras-versionlock' + else: + vl_plugin = 'python-dnf-plugins-extras-versionlock' else: vl_plugin = 'yum-versionlock' \ if __grains__.get('osmajorrelease') == '5' \ @@ -261,7 +277,7 @@ def _get_extra_options(**kwargs): Returns list of extra options for yum ''' ret = [] - kwargs = salt.utils.clean_kwargs(**kwargs) + kwargs = salt.utils.args.clean_kwargs(**kwargs) for key, value in six.iteritems(kwargs): if isinstance(key, six.string_types): ret.append('--{0}=\'{1}\''.format(key, value)) @@ -475,7 +491,7 @@ def latest_version(*names, **kwargs): # If any installed version is greater than (or equal to) the # one found by yum/dnf list available, then it is not an # upgrade. - if salt.utils.compare_versions(ver1=installed_version, + if salt.utils.versions.compare(ver1=installed_version, oper='>=', ver2=pkg.version, cmp_func=version_cmp): @@ -584,15 +600,35 @@ def version_cmp(pkg1, pkg2, ignore_epoch=False): def list_pkgs(versions_as_list=False, **kwargs): ''' - List the packages currently installed in a dict:: + List the packages currently installed as a dict. By default, the dict + contains versions as a comma separated string:: - {'': ''} + {'': '[,...]'} + + versions_as_list: + If set to true, the versions are provided as a list + + {'': ['', '']} + + attr: + If a list of package attributes is specified, returned value will + contain them in addition to version, eg.:: + + {'': [{'version' : 'version', 'arch' : 'arch'}]} + + Valid attributes are: ``epoch``, ``version``, ``release``, ``arch``, + ``install_date``, ``install_date_time_t``. + + If ``all`` is specified, all valid attributes will be returned. + + .. versionadded:: Oxygen CLI Example: .. code-block:: bash salt '*' pkg.list_pkgs + salt '*' pkg.list_pkgs attr='["version", "arch"]' ''' versions_as_list = salt.utils.is_true(versions_as_list) # not yet implemented or not applicable @@ -600,17 +636,14 @@ def list_pkgs(versions_as_list=False, **kwargs): for x in ('removed', 'purge_desired')]): return {} + attr = kwargs.get("attr") if 'pkg.list_pkgs' in __context__: - if versions_as_list: - return __context__['pkg.list_pkgs'] - else: - ret = copy.deepcopy(__context__['pkg.list_pkgs']) - __salt__['pkg_resource.stringify'](ret) - return ret + cached = __context__['pkg.list_pkgs'] + return __salt__['pkg_resource.format_pkg_list'](cached, versions_as_list, attr) ret = {} cmd = ['rpm', '-qa', '--queryformat', - salt.utils.pkg.rpm.QUERYFORMAT.replace('%{REPOID}', '(none)\n')] + salt.utils.pkg.rpm.QUERYFORMAT.replace('%{REPOID}', '(none)') + '\n'] output = __salt__['cmd.run'](cmd, python_shell=False, output_loglevel='trace') @@ -620,15 +653,25 @@ def list_pkgs(versions_as_list=False, **kwargs): osarch=__grains__['osarch'] ) if pkginfo is not None: - __salt__['pkg_resource.add_pkg'](ret, - pkginfo.name, - pkginfo.version) + # see rpm version string rules available at https://goo.gl/UGKPNd + pkgver = pkginfo.version + epoch = '' + release = '' + if ':' in pkgver: + epoch, pkgver = pkgver.split(":", 1) + if '-' in pkgver: + pkgver, release = pkgver.split("-", 1) + all_attr = {'epoch': epoch, 'version': pkgver, 'release': release, + 'arch': pkginfo.arch, 'install_date': pkginfo.install_date, + 'install_date_time_t': pkginfo.install_date_time_t} + __salt__['pkg_resource.add_pkg'](ret, pkginfo.name, all_attr) - __salt__['pkg_resource.sort_pkglist'](ret) - __context__['pkg.list_pkgs'] = copy.deepcopy(ret) - if not versions_as_list: - __salt__['pkg_resource.stringify'](ret) - return ret + for pkgname in ret: + ret[pkgname] = sorted(ret[pkgname], key=lambda d: d['version']) + + __context__['pkg.list_pkgs'] = ret + + return __salt__['pkg_resource.format_pkg_list'](ret, versions_as_list, attr) def list_repo_pkgs(*args, **kwargs): @@ -846,8 +889,8 @@ def list_repo_pkgs(*args, **kwargs): _parse_output(out['stdout'], strict=True) else: for repo in repos: - cmd = [_yum(), '--quiet', 'repository-packages', repo, - 'list', '--showduplicates'] + cmd = [_yum(), '--quiet', '--showduplicates', + 'repository-packages', repo, 'list'] if cacheonly: cmd.append('-C') # Can't concatenate because args is a tuple, using list.extend() @@ -927,7 +970,7 @@ list_updates = salt.utils.alias_function(list_upgrades, 'list_updates') def list_downloaded(): ''' - .. versionadded:: Oxygen + .. versionadded:: 2017.7.0 List prefetched packages downloaded by Yum in the local disk. @@ -1034,6 +1077,11 @@ def refresh_db(**kwargs): clean_cmd = [_yum(), '--quiet', 'clean', 'expire-cache'] update_cmd = [_yum(), '--quiet', 'check-update'] + + if __grains__.get('os_family') == 'RedHat' and __grains__.get('osmajorrelease') == '7': + # This feature is disable because it is not used by Salt and lasts a lot with using large repo like EPEL + update_cmd.append('--setopt=autocheck_running_kernel=false') + for args in (repo_arg, exclude_arg, branch_arg): if args: clean_cmd.extend(args) @@ -1065,6 +1113,21 @@ def clean_metadata(**kwargs): return refresh_db(**kwargs) +class AvailablePackages(salt.utils.lazy.LazyDict): + def __init__(self, *args, **kwargs): + super(AvailablePackages, self).__init__() + self._args = args + self._kwargs = kwargs + + def _load(self, key): + self._load_all() + return True + + def _load_all(self): + self._dict = list_repo_pkgs(*self._args, **self._kwargs) + self.loaded = True + + def install(name=None, refresh=False, skip_verify=False, @@ -1218,11 +1281,42 @@ def install(name=None, .. versionadded:: 2014.7.0 + diff_attr: + If a list of package attributes is specified, returned value will + contain them, eg.:: + + {'': { + 'old': { + 'version': '', + 'arch': ''}, + + 'new': { + 'version': '', + 'arch': ''}}} + + Valid attributes are: ``epoch``, ``version``, ``release``, ``arch``, + ``install_date``, ``install_date_time_t``. + + If ``all`` is specified, all valid attributes will be returned. + + .. versionadded:: Oxygen Returns a dict containing the new package names and versions:: {'': {'old': '', 'new': ''}} + + If an attribute list in diff_attr is specified, the dict will also contain + any specified attribute, eg.:: + + {'': { + 'old': { + 'version': '', + 'arch': ''}, + + 'new': { + 'version': '', + 'arch': ''}}} ''' repo_arg = _get_repo_options(**kwargs) exclude_arg = _get_excludes_option(**kwargs) @@ -1244,10 +1338,11 @@ def install(name=None, version_num = kwargs.get('version') - old = list_pkgs(versions_as_list=False) if not downloadonly else list_downloaded() + diff_attr = kwargs.get("diff_attr") + old = list_pkgs(versions_as_list=False, attr=diff_attr) if not downloadonly else list_downloaded() # Use of __context__ means no duplicate work here, just accessing # information already in __context__ from the previous call to list_pkgs() - old_as_list = list_pkgs(versions_as_list=True) if not downloadonly else list_downloaded() + old_as_list = list_pkgs(versions_as_list=True, attr=diff_attr) if not downloadonly else list_downloaded() to_install = [] to_downgrade = [] @@ -1278,7 +1373,7 @@ def install(name=None, has_comparison.append(pkgname) except (TypeError, ValueError): continue - _available = list_repo_pkgs( + _available = AvailablePackages( *has_wildcards + has_comparison, byrepo=False, **kwargs) @@ -1413,7 +1508,7 @@ def install(name=None, if reinstall and cver: for ver in cver: ver = norm_epoch(ver, version_num) - if salt.utils.compare_versions(ver1=version_num, + if salt.utils.versions.compare(ver1=version_num, oper='==', ver2=ver, cmp_func=version_cmp): @@ -1427,7 +1522,7 @@ def install(name=None, else: for ver in cver: ver = norm_epoch(ver, version_num) - if salt.utils.compare_versions(ver1=version_num, + if salt.utils.versions.compare(ver1=version_num, oper='>=', ver2=ver, cmp_func=version_cmp): @@ -1582,7 +1677,7 @@ def install(name=None, errors.append(out['stdout']) __context__.pop('pkg.list_pkgs', None) - new = list_pkgs(versions_as_list=False) if not downloadonly else list_downloaded() + new = list_pkgs(versions_as_list=False, attr=diff_attr) if not downloadonly else list_downloaded() ret = salt.utils.compare_dicts(old, new) @@ -2572,15 +2667,16 @@ def del_repo(repo, basedir=None, **kwargs): # pylint: disable=W0613 if stanza == repo: continue comments = '' - if 'comments' in filerepos[stanza]: - comments = '\n'.join(filerepos[stanza]['comments']) + if 'comments' in six.iterkeys(filerepos[stanza]): + comments = salt.utils.pkg.rpm.combine_comments( + filerepos[stanza]['comments']) del filerepos[stanza]['comments'] content += '\n[{0}]'.format(stanza) for line in filerepos[stanza]: content += '\n{0}={1}'.format(line, filerepos[stanza][line]) content += '\n{0}\n'.format(comments) - with salt.utils.fopen(repofile, 'w') as fileout: + with salt.utils.files.fopen(repofile, 'w') as fileout: fileout.write(content) return 'Repo {0} has been removed from {1}'.format(repo, repofile) @@ -2708,7 +2804,8 @@ def mod_repo(repo, basedir=None, **kwargs): for stanza in six.iterkeys(filerepos): comments = '' if 'comments' in six.iterkeys(filerepos[stanza]): - comments = '\n'.join(filerepos[stanza]['comments']) + comments = salt.utils.pkg.rpm.combine_comments( + filerepos[stanza]['comments']) del filerepos[stanza]['comments'] content += '\n[{0}]'.format(stanza) for line in six.iterkeys(filerepos[stanza]): @@ -2720,7 +2817,7 @@ def mod_repo(repo, basedir=None, **kwargs): ) content += '\n{0}\n'.format(comments) - with salt.utils.fopen(repofile, 'w') as fileout: + with salt.utils.files.fopen(repofile, 'w') as fileout: fileout.write(content) return {repofile: filerepos} @@ -2730,41 +2827,32 @@ def _parse_repo_file(filename): ''' Turn a single repo file into a dict ''' - repos = {} - header = '' - repo = '' - with salt.utils.fopen(filename, 'r') as rfile: - for line in rfile: - if line.startswith('['): - repo = line.strip().replace('[', '').replace(']', '') - repos[repo] = {} + parsed = configparser.ConfigParser() + config = {} - # Even though these are essentially uselss, I want to allow the - # user to maintain their own comments, etc - if not line: - if not repo: - header += line - if line.startswith('#'): - if not repo: - header += line - else: - if 'comments' not in repos[repo]: - repos[repo]['comments'] = [] - repos[repo]['comments'].append(line.strip()) - continue + try: + parsed.read(filename) + except configparser.MissingSectionHeaderError as err: + log.error( + 'Failed to parse file {0}, error: {1}'.format(filename, err.message) + ) + return ('', {}) - # These are the actual configuration lines that matter - if '=' in line: - try: - comps = line.strip().split('=') - repos[repo][comps[0].strip()] = '='.join(comps[1:]) - except KeyError: - log.error( - 'Failed to parse line in %s, offending line was ' - '\'%s\'', filename, line.rstrip() - ) + for section in parsed._sections: + section_dict = dict(parsed._sections[section]) + section_dict.pop('__name__', None) + config[section] = section_dict - return (header, repos) + # Try to extract leading comments + headers = '' + with salt.utils.files.fopen(filename, 'r') as rawfile: + for line in rawfile: + if line.strip().startswith('#'): + headers += '{0}\n'.format(line.strip()) + else: + break + + return (headers, config) def file_list(*packages): @@ -2892,7 +2980,7 @@ def modified(*packages, **flags): return __salt__['lowpkg.modified'](*packages, **flags) -@decorators.which('yumdownloader') +@salt.utils.decorators.path.which('yumdownloader') def download(*packages): ''' .. versionadded:: 2015.5.0 @@ -3027,7 +3115,7 @@ def _get_patches(installed_only=False): def list_patches(refresh=False): ''' - .. versionadded:: Oxygen + .. versionadded:: 2017.7.0 List all known advisory patches from available repos. @@ -3050,7 +3138,7 @@ def list_patches(refresh=False): def list_installed_patches(): ''' - .. versionadded:: Oxygen + .. versionadded:: 2017.7.0 List installed advisory patches on the system. diff --git a/salt/modules/zabbix.py b/salt/modules/zabbix.py index 2403d2e2b9f..7fc76f4a098 100644 --- a/salt/modules/zabbix.py +++ b/salt/modules/zabbix.py @@ -31,6 +31,7 @@ import json # Import salt libs import salt.utils +import salt.utils.path from salt.utils.versions import LooseVersion as _LooseVersion from salt.ext.six.moves.urllib.error import HTTPError, URLError # pylint: disable=import-error,no-name-in-module @@ -46,7 +47,7 @@ def __virtual__(): ''' Only load the module if Zabbix server is installed ''' - if salt.utils.which('zabbix_server'): + if salt.utils.path.which('zabbix_server'): return __virtualname__ return (False, 'The zabbix execution module cannot be loaded: zabbix not installed.') diff --git a/salt/modules/zcbuildout.py b/salt/modules/zcbuildout.py index 9b7c495b980..66f111f0dfd 100644 --- a/salt/modules/zcbuildout.py +++ b/salt/modules/zcbuildout.py @@ -35,13 +35,13 @@ import copy # Import 3rd-party libs # pylint: disable=import-error,no-name-in-module,redefined-builtin -from salt.ext.six import string_types, text_type +from salt.ext import six from salt.ext.six.moves import range from salt.ext.six.moves.urllib.request import urlopen as _urlopen # pylint: enable=import-error,no-name-in-module,redefined-builtin # Import salt libs -import salt.utils +import salt.utils.files from salt.exceptions import CommandExecutionError @@ -124,7 +124,7 @@ def _salt_callback(func, **kwargs): LOG.clear() # before returning, trying to compact the log output for k in ['comment', 'out', 'outlog']: - if status[k] and isinstance(status[k], string_types): + if status[k] and isinstance(status[k], six.string_types): status[k] = '\n'.join([ log for log in status[k].split('\n') @@ -142,7 +142,7 @@ class _Logger(object): self._by_level = {} def _log(self, level, msg): - if not isinstance(msg, text_type): + if not isinstance(msg, six.text_type): msg = msg.decode('utf-8') if level not in self._by_level: self._by_level[level] = [] @@ -185,7 +185,7 @@ LOG = _Logger() def _encode_string(string): - if isinstance(string, text_type): + if isinstance(string, six.text_type): string = string.encode('utf-8') return string @@ -217,7 +217,7 @@ def _set_status(m, m['logs_by_level'] = LOG.by_level.copy() outlog, outlog_by_level = '', '' m['comment'] = comment - if out and isinstance(out, string_types): + if out and isinstance(out, six.string_types): outlog += HR outlog += 'OUTPUT:\n' outlog += '{0}\n'.format(_encode_string(out)) @@ -391,7 +391,7 @@ def _get_bootstrap_content(directory='.'): Get the current bootstrap.py script content ''' try: - with salt.utils.fopen(os.path.join( + with salt.utils.files.fopen(os.path.join( os.path.abspath(directory), 'bootstrap.py')) as fic: oldcontent = fic.read() @@ -416,7 +416,7 @@ def _get_buildout_ver(directory='.'): try: files = _find_cfgs(directory) for f in files: - with salt.utils.fopen(f) as fic: + with salt.utils.files.fopen(f) as fic: buildout1re = re.compile(r'^zc\.buildout\s*=\s*1', RE_F) dfic = fic.read() if ( @@ -518,7 +518,7 @@ def upgrade_bootstrap(directory='.', if not os.path.isdir(dbuild): os.makedirs(dbuild) # only try to download once per buildout checkout - with salt.utils.fopen(os.path.join( + with salt.utils.files.fopen(os.path.join( dbuild, '{0}.updated_bootstrap'.format(buildout_ver))): pass @@ -534,16 +534,16 @@ def upgrade_bootstrap(directory='.', data = '\n'.join(ldata) if updated: comment = 'Bootstrap updated' - with salt.utils.fopen(b_py, 'w') as fic: + with salt.utils.files.fopen(b_py, 'w') as fic: fic.write(data) if dled: - with salt.utils.fopen(os.path.join(dbuild, + with salt.utils.files.fopen(os.path.join(dbuild, '{0}.updated_bootstrap'.format( buildout_ver)), 'w') as afic: afic.write('foo') except (OSError, IOError): if oldcontent: - with salt.utils.fopen(b_py, 'w') as fic: + with salt.utils.files.fopen(b_py, 'w') as fic: fic.write(oldcontent) return {'comment': comment} @@ -734,7 +734,7 @@ def bootstrap(directory='.', buildout_ver=buildout_ver) # be sure which buildout bootstrap we have b_py = os.path.join(directory, 'bootstrap.py') - with salt.utils.fopen(b_py) as fic: + with salt.utils.files.fopen(b_py) as fic: content = fic.read() if ( (test_release is not False) @@ -1029,17 +1029,17 @@ def _check_onlyif_unless(onlyif, unless, directory, runas=None, env=()): status['status'] = False retcode = __salt__['cmd.retcode'] if onlyif is not None: - if not isinstance(onlyif, string_types): + if not isinstance(onlyif, six.string_types): if not onlyif: _valid(status, 'onlyif execution failed') - elif isinstance(onlyif, string_types): + elif isinstance(onlyif, six.string_types): if retcode(onlyif, cwd=directory, runas=runas, env=env) != 0: _valid(status, 'onlyif execution failed') if unless is not None: - if not isinstance(unless, string_types): + if not isinstance(unless, six.string_types): if unless: _valid(status, 'unless execution succeeded') - elif isinstance(unless, string_types): + elif isinstance(unless, six.string_types): if retcode(unless, cwd=directory, runas=runas, env=env, python_shell=False) == 0: _valid(status, 'unless execution succeeded') if status['status']: diff --git a/salt/modules/zfs.py b/salt/modules/zfs.py index 5053ca66055..dc42400796c 100644 --- a/salt/modules/zfs.py +++ b/salt/modules/zfs.py @@ -11,7 +11,8 @@ from __future__ import absolute_import import logging # Import Salt libs -import salt.utils +import salt.utils.args +import salt.utils.path import salt.modules.cmdmod import salt.utils.decorators as decorators from salt.utils.odict import OrderedDict @@ -30,7 +31,7 @@ def _check_zfs(): Looks to see if zfs is present on the system. ''' # Get the path to the zfs binary. - return salt.utils.which('zfs') + return salt.utils.path.which('zfs') @decorators.memoize @@ -39,7 +40,7 @@ def _check_features(): Looks to see if zpool-features is available ''' # get man location - man = salt.utils.which('man') + man = salt.utils.path.which('man') if not man: return False @@ -62,19 +63,24 @@ def __virtual__(): if on_freebsd: cmd = 'kldstat -q -m zfs' elif on_linux: - modinfo = salt.utils.which('modinfo') + modinfo = salt.utils.path.which('modinfo') if modinfo: cmd = '{0} zfs'.format(modinfo) else: cmd = 'ls /sys/module/zfs' elif on_solaris: - # not using salt.utils.which('zfs') to keep compatible with others + # not using salt.utils.path.which('zfs') to keep compatible with others cmd = 'which zfs' if cmd and salt.modules.cmdmod.retcode( cmd, output_loglevel='quiet', ignore_retcode=True ) == 0: return 'zfs' + + _zfs_fuse = lambda f: __salt__['service.' + f]('zfs-fuse') + if _zfs_fuse('available') and (_zfs_fuse('status') or _zfs_fuse('start')): + return 'zfs' + return (False, "The zfs module cannot be loaded: zfs not found") @@ -1142,7 +1148,7 @@ def set(*dataset, **kwargs): ret['error'] = 'one or more snapshots must be specified' # clean kwargs - properties = salt.utils.clean_kwargs(**kwargs) + properties = salt.utils.args.clean_kwargs(**kwargs) if len(properties) < 1: ret['error'] = '{0}one or more properties must be specified'.format( '{0},\n'.format(ret['error']) if 'error' in ret else '' diff --git a/salt/modules/zk_concurrency.py b/salt/modules/zk_concurrency.py index 4335a176d85..2dc0a8dbf58 100644 --- a/salt/modules/zk_concurrency.py +++ b/salt/modules/zk_concurrency.py @@ -185,7 +185,7 @@ def lock_holders(path, Example: - ... code-block: bash + .. code-block: bash salt minion zk_concurrency.lock_holders /lock/path host1:1234,host2:1234 ''' @@ -237,7 +237,7 @@ def lock(path, Example: - ... code-block: bash + .. code-block: bash salt minion zk_concurrency.lock /lock/path host1:1234,host2:1234 ''' @@ -298,7 +298,7 @@ def unlock(path, Example: - ... code-block: bash + .. code-block: bash salt minion zk_concurrency.unlock /lock/path host1:1234,host2:1234 ''' @@ -348,7 +348,7 @@ def party_members(path, Example: - ... code-block: bash + .. code-block: bash salt minion zk_concurrency.party_members /lock/path host1:1234,host2:1234 salt minion zk_concurrency.party_members /lock/path host1:1234,host2:1234 min_nodes=3 blocking=True diff --git a/salt/modules/znc.py b/salt/modules/znc.py index 23a2d59a5a3..8da86dfa85e 100644 --- a/salt/modules/znc.py +++ b/salt/modules/znc.py @@ -16,7 +16,7 @@ import random import signal # Import salt libs -import salt.utils +import salt.utils.path from salt.ext.six.moves import range log = logging.getLogger(__name__) @@ -26,7 +26,7 @@ def __virtual__(): ''' Only load the module if znc is installed ''' - if salt.utils.which('znc'): + if salt.utils.path.which('znc'): return 'znc' return (False, "Module znc: znc binary not found") diff --git a/salt/modules/zoneadm.py b/salt/modules/zoneadm.py index 4edbe156791..0ee2bead735 100644 --- a/salt/modules/zoneadm.py +++ b/salt/modules/zoneadm.py @@ -17,7 +17,7 @@ from __future__ import absolute_import import logging # Import Salt libs -import salt.utils +import salt.utils.path import salt.utils.decorators from salt.ext.six.moves import range @@ -61,11 +61,11 @@ def __virtual__(): We are available if we are have zoneadm and are the global zone on Solaris 10, OmniOS, OpenIndiana, OpenSolaris, or Smartos. ''' - ## note: we depend on PR#37472 to distinguish between Solaris and Oracle Solaris - if _is_globalzone() and salt.utils.which('zoneadm'): - if __grains__['os'] in ['Solaris', 'OpenSolaris', 'SmartOS', 'OmniOS', 'OpenIndiana']: + if _is_globalzone() and salt.utils.path.which('zoneadm'): + if __grains__['os'] in ['OpenSolaris', 'SmartOS', 'OmniOS', 'OpenIndiana']: + return __virtualname__ + elif __grains__['os'] == 'Oracle Solaris' and int(__grains__['osmajorrelease']) == 10: return __virtualname__ - return ( False, '{0} module can only be loaded in a solaris globalzone.'.format( diff --git a/salt/modules/zonecfg.py b/salt/modules/zonecfg.py index ff9178f0d6c..54e17473816 100644 --- a/salt/modules/zonecfg.py +++ b/salt/modules/zonecfg.py @@ -19,12 +19,15 @@ import logging import re # Import Salt libs -import salt.ext.six as six -import salt.utils -import salt.utils.files +import salt.utils.args import salt.utils.decorators +import salt.utils.files +import salt.utils.path from salt.utils.odict import OrderedDict +# Import 3rd-party libs +from salt.ext import six + log = logging.getLogger(__name__) # Define the module's virtual name @@ -97,11 +100,11 @@ def __virtual__(): We are available if we are have zonecfg and are the global zone on Solaris 10, OmniOS, OpenIndiana, OpenSolaris, or Smartos. ''' - ## note: we depend on PR#37472 to distinguish between Solaris and Oracle Solaris - if _is_globalzone() and salt.utils.which('zonecfg'): - if __grains__['os'] in ['Solaris', 'OpenSolaris', 'SmartOS', 'OmniOS', 'OpenIndiana']: + if _is_globalzone() and salt.utils.path.which('zonecfg'): + if __grains__['os'] in ['OpenSolaris', 'SmartOS', 'OmniOS', 'OpenIndiana']: + return __virtualname__ + elif __grains__['os'] == 'Oracle Solaris' and int(__grains__['osmajorrelease']) == 10: return __virtualname__ - return ( False, '{0} module can only be loaded in a solaris globalzone.'.format( @@ -124,7 +127,7 @@ def _parse_value(value): '''Internal helper for parsing configuration values into python values''' if isinstance(value, bool): return 'true' if value else 'false' - elif isinstance(value, str): + elif isinstance(value, six.string_types): # parse compacted notation to dict listparser = re.compile(r'''((?:[^,"']|"[^"]*"|'[^']*')+)''') @@ -190,14 +193,14 @@ def _sanitize_value(value): new_value.append(')') return "".join(str(v) for v in new_value).replace(',)', ')') else: - ## note: we can't use shelx or pipes quote here because it makes zonecfg barf + # note: we can't use shelx or pipes quote here because it makes zonecfg barf return '"{0}"'.format(value) if ' ' in value else value def _dump_cfg(cfg_file): '''Internal helper for debugging cfg files''' if __salt__['file.file_exists'](cfg_file): - with salt.utils.fopen(cfg_file, 'r') as fp_: + with salt.utils.files.fopen(cfg_file, 'r') as fp_: log.debug("zonecfg - configuration file:\n{0}".format("".join(fp_.readlines()))) @@ -222,14 +225,14 @@ def create(zone, brand, zonepath, force=False): ''' ret = {'status': True} - ## write config + # write config cfg_file = salt.utils.files.mkstemp() - with salt.utils.fpopen(cfg_file, 'w+', mode=0o600) as fp_: + with salt.utils.files.fpopen(cfg_file, 'w+', mode=0o600) as fp_: fp_.write("create -b -F\n" if force else "create -b\n") fp_.write("set brand={0}\n".format(_sanitize_value(brand))) fp_.write("set zonepath={0}\n".format(_sanitize_value(zonepath))) - ## create + # create if not __salt__['file.directory_exists'](zonepath): __salt__['file.makedirs_perms'](zonepath if zonepath[-1] == '/' else '{0}/'.format(zonepath), mode='0700') @@ -245,7 +248,7 @@ def create(zone, brand, zonepath, force=False): else: ret['message'] = _clean_message(ret['message']) - ## cleanup config file + # cleanup config file if __salt__['file.file_exists'](cfg_file): __salt__['file.remove'](cfg_file) @@ -272,7 +275,7 @@ def create_from_template(zone, template): ''' ret = {'status': True} - ## create from template + # create from template _dump_cfg(template) res = __salt__['cmd.run_all']('zonecfg -z {zone} create -t {tmpl} -F'.format( zone=zone, @@ -303,7 +306,7 @@ def delete(zone): ''' ret = {'status': True} - ## delete zone + # delete zone res = __salt__['cmd.run_all']('zonecfg -z {zone} delete -F'.format( zone=zone, )) @@ -335,7 +338,7 @@ def export(zone, path=None): ''' ret = {'status': True} - ## export zone + # export zone res = __salt__['cmd.run_all']('zonecfg -z {zone} export{path}'.format( zone=zone, path=' -f {0}'.format(path) if path else '', @@ -367,7 +370,7 @@ def import_(zone, path): ''' ret = {'status': True} - ## create from file + # create from file _dump_cfg(path) res = __salt__['cmd.run_all']('zonecfg -z {zone} -f {path}'.format( zone=zone, @@ -406,7 +409,7 @@ def _property(methode, zone, key, value): ret['message'] = 'unkown methode {0}!'.format(methode) else: cfg_file = salt.utils.files.mkstemp() - with salt.utils.fpopen(cfg_file, 'w+', mode=0o600) as fp_: + with salt.utils.files.fpopen(cfg_file, 'w+', mode=0o600) as fp_: if methode == 'set': if isinstance(value, dict) or isinstance(value, list): value = _sanitize_value(value) @@ -415,7 +418,7 @@ def _property(methode, zone, key, value): elif methode == 'clear': fp_.write("{0} {1}\n".format(methode, key)) - ## update property + # update property if cfg_file: _dump_cfg(cfg_file) res = __salt__['cmd.run_all']('zonecfg -z {zone} -f {path}'.format( @@ -429,7 +432,7 @@ def _property(methode, zone, key, value): else: ret['message'] = _clean_message(ret['message']) - ## cleanup config file + # cleanup config file if __salt__['file.file_exists'](cfg_file): __salt__['file.remove'](cfg_file) @@ -503,7 +506,7 @@ def _resource(methode, zone, resource_type, resource_selector, **kwargs): ret = {'status': True} # parse kwargs - kwargs = salt.utils.clean_kwargs(**kwargs) + kwargs = salt.utils.args.clean_kwargs(**kwargs) for k in kwargs: if isinstance(kwargs[k], dict) or isinstance(kwargs[k], list): kwargs[k] = _sanitize_value(kwargs[k]) @@ -518,7 +521,7 @@ def _resource(methode, zone, resource_type, resource_selector, **kwargs): # generate update script cfg_file = salt.utils.files.mkstemp() - with salt.utils.fpopen(cfg_file, 'w+', mode=0o600) as fp_: + with salt.utils.files.fpopen(cfg_file, 'w+', mode=0o600) as fp_: if methode in ['add']: fp_.write("add {0}\n".format(resource_type)) elif methode in ['update']: @@ -542,7 +545,7 @@ def _resource(methode, zone, resource_type, resource_selector, **kwargs): fp_.write("add {0} {1}\n".format(k, _sanitize_value(value))) fp_.write("end\n") - ## update property + # update property if cfg_file: _dump_cfg(cfg_file) res = __salt__['cmd.run_all']('zonecfg -z {zone} -f {path}'.format( @@ -556,7 +559,7 @@ def _resource(methode, zone, resource_type, resource_selector, **kwargs): else: ret['message'] = _clean_message(ret['message']) - ## cleanup config file + # cleanup config file if __salt__['file.file_exists'](cfg_file): __salt__['file.remove'](cfg_file) @@ -634,13 +637,13 @@ def remove_resource(zone, resource_type, resource_key, resource_value): # generate update script cfg_file = salt.utils.files.mkstemp() - with salt.utils.fpopen(cfg_file, 'w+', mode=0o600) as fp_: + with salt.utils.files.fpopen(cfg_file, 'w+', mode=0o600) as fp_: if resource_key: fp_.write("remove {0} {1}={2}\n".format(resource_type, resource_key, _sanitize_value(resource_value))) else: fp_.write("remove {0}\n".format(resource_type)) - ## update property + # update property if cfg_file: _dump_cfg(cfg_file) res = __salt__['cmd.run_all']('zonecfg -z {zone} -f {path}'.format( @@ -654,7 +657,7 @@ def remove_resource(zone, resource_type, resource_key, resource_value): else: ret['message'] = _clean_message(ret['message']) - ## cleanup config file + # cleanup config file if __salt__['file.file_exists'](cfg_file): __salt__['file.remove'](cfg_file) @@ -678,7 +681,7 @@ def info(zone, show_all=False): ''' ret = {} - ## dump zone + # dump zone res = __salt__['cmd.run_all']('zonecfg -z {zone} info'.format( zone=zone, )) diff --git a/salt/modules/zookeeper.py b/salt/modules/zookeeper.py index 6ed4a22c4fc..c14c539e382 100644 --- a/salt/modules/zookeeper.py +++ b/salt/modules/zookeeper.py @@ -75,6 +75,7 @@ except ImportError: # Import Salt libraries import salt.utils +import salt.utils.stringutils __virtualname__ = 'zookeeper' @@ -183,7 +184,7 @@ def create(path, value='', acls=None, ephemeral=False, sequence=False, makepath= acls = [make_digest_acl(**acl) for acl in acls] conn = _get_zk_conn(profile=profile, hosts=hosts, scheme=scheme, username=username, password=password, default_acl=default_acl) - return conn.create(path, salt.utils.to_bytes(value), acls, ephemeral, sequence, makepath) + return conn.create(path, salt.utils.stringutils.to_bytes(value), acls, ephemeral, sequence, makepath) def ensure_path(path, acls=None, profile=None, hosts=None, scheme=None, @@ -302,7 +303,7 @@ def get(path, profile=None, hosts=None, scheme=None, username=None, password=Non conn = _get_zk_conn(profile=profile, hosts=hosts, scheme=scheme, username=username, password=password, default_acl=default_acl) ret, _ = conn.get(path) - return salt.utils.to_str(ret) + return salt.utils.stringutils.to_str(ret) def get_children(path, profile=None, hosts=None, scheme=None, username=None, password=None, default_acl=None): @@ -384,7 +385,7 @@ def set(path, value, version=-1, profile=None, hosts=None, scheme=None, ''' conn = _get_zk_conn(profile=profile, hosts=hosts, scheme=scheme, username=username, password=password, default_acl=default_acl) - return conn.set(path, salt.utils.to_bytes(value), version=version) + return conn.set(path, salt.utils.stringutils.to_bytes(value), version=version) def get_acls(path, profile=None, hosts=None, scheme=None, username=None, password=None, default_acl=None): diff --git a/salt/modules/zpool.py b/salt/modules/zpool.py index 406530310e2..9433c87abd3 100644 --- a/salt/modules/zpool.py +++ b/salt/modules/zpool.py @@ -12,8 +12,9 @@ import stat import logging # Import Salt libs -import salt.utils -import salt.utils.decorators as decorators +import salt.utils.decorators +import salt.utils.decorators.path +import salt.utils.path from salt.utils.odict import OrderedDict log = logging.getLogger(__name__) @@ -24,21 +25,21 @@ __func_alias__ = { } -@decorators.memoize +@salt.utils.decorators.memoize def _check_zpool(): ''' Looks to see if zpool is present on the system ''' - return salt.utils.which('zpool') + return salt.utils.path.which('zpool') -@decorators.memoize +@salt.utils.decorators.memoize def _check_features(): ''' Looks to see if zpool-features is available ''' # get man location - man = salt.utils.which('man') + man = salt.utils.path.which('man') if not man: return False @@ -49,12 +50,12 @@ def _check_features(): return res['retcode'] == 0 -@decorators.memoize +@salt.utils.decorators.memoize def _check_mkfile(): ''' Looks to see if mkfile is present on the system ''' - return salt.utils.which('mkfile') + return salt.utils.path.which('mkfile') def __virtual__(): @@ -918,7 +919,7 @@ def replace(zpool, old_device, new_device=None, force=False): return ret -@salt.utils.decorators.which('mkfile') +@salt.utils.decorators.path.which('mkfile') def create_file_vdev(size, *vdevs): ''' .. versionchanged:: 2016.3.0 diff --git a/salt/modules/zypper.py b/salt/modules/zypper.py index 592f416b8c1..66658ba1f6a 100644 --- a/salt/modules/zypper.py +++ b/salt/modules/zypper.py @@ -14,20 +14,16 @@ Package support for openSUSE via the zypper package manager # Import python libs from __future__ import absolute_import -import copy import fnmatch import logging import re import os import time import datetime -from salt.utils.versions import LooseVersion # Import 3rd-party libs # pylint: disable=import-error,redefined-builtin,no-name-in-module -import salt.ext.six as six -from salt.exceptions import SaltInvocationError -import salt.utils.event +from salt.ext import six from salt.ext.six.moves import configparser from salt.ext.six.moves.urllib.parse import urlparse as _urlparse # pylint: enable=import-error,redefined-builtin,no-name-in-module @@ -37,10 +33,13 @@ from xml.parsers.expat import ExpatError # Import salt libs import salt.utils +import salt.utils.event +import salt.utils.files +import salt.utils.path import salt.utils.pkg import salt.utils.systemd -from salt.exceptions import ( - CommandExecutionError, MinionError) +from salt.utils.versions import LooseVersion +from salt.exceptions import CommandExecutionError, MinionError, SaltInvocationError log = logging.getLogger(__name__) @@ -61,7 +60,7 @@ def __virtual__(): if __grains__.get('os_family', '') != 'Suse': return (False, "Module zypper: non SUSE OS not suppored by zypper package manager") # Not all versions of SUSE use zypper, check that it is available - if not salt.utils.which('zypper'): + if not salt.utils.path.which('zypper'): return (False, "Module zypper: zypper package manager not found") return __virtualname__ @@ -276,7 +275,7 @@ class _Zypper(object): if os.path.exists(self.ZYPPER_LOCK): try: - with salt.utils.fopen(self.ZYPPER_LOCK) as rfh: + with salt.utils.files.fopen(self.ZYPPER_LOCK) as rfh: data = __salt__['ps.proc_info'](int(rfh.readline()), attrs=['pid', 'name', 'cmdline', 'create_time']) data['cmdline'] = ' '.join(data['cmdline']) @@ -450,9 +449,9 @@ def info_installed(*names, **kwargs): summary, description. :param errors: - Handle RPM field errors (true|false). By default, various mistakes in the textual fields are simply ignored and - omitted from the data. Otherwise a field with a mistake is not returned, instead a 'N/A (bad UTF-8)' - (not available, broken) text is returned. + Handle RPM field errors. If 'ignore' is chosen, then various mistakes are simply ignored and omitted + from the texts or strings. If 'report' is chonen, then a field with a mistake is not returned, instead + a 'N/A (broken)' (not available, broken) text is placed. Valid attributes are: ignore, report @@ -465,7 +464,8 @@ def info_installed(*names, **kwargs): salt '*' pkg.info_installed ... salt '*' pkg.info_installed attr=version,vendor salt '*' pkg.info_installed ... attr=version,vendor - salt '*' pkg.info_installed ... attr=version,vendor errors=true + salt '*' pkg.info_installed ... attr=version,vendor errors=ignore + salt '*' pkg.info_installed ... attr=version,vendor errors=report ''' ret = dict() for pkg_name, pkg_nfo in __salt__['lowpkg.info'](*names, **kwargs).items(): @@ -479,7 +479,7 @@ def info_installed(*names, **kwargs): else: value_ = value.decode('UTF-8', 'ignore').encode('UTF-8', 'ignore') if value != value_: - value = kwargs.get('errors') and value_ or 'N/A (invalid UTF-8)' + value = kwargs.get('errors', 'ignore') == 'ignore' and value_ or 'N/A (invalid UTF-8)' log.error('Package {0} has bad UTF-8 code in {1}: {2}'.format(pkg_name, key, value)) if key == 'source_rpm': t_nfo['source'] = value @@ -650,8 +650,8 @@ def version_cmp(ver1, ver2, ignore_epoch=False): def list_pkgs(versions_as_list=False, **kwargs): ''' - List the packages currently installed as a dict with versions - as a comma separated string:: + List the packages currently installed as a dict. By default, the dict + contains versions as a comma separated string:: {'': '[,...]'} @@ -660,6 +660,19 @@ def list_pkgs(versions_as_list=False, **kwargs): {'': ['', '']} + attr: + If a list of package attributes is specified, returned value will + contain them in addition to version, eg.:: + + {'': [{'version' : 'version', 'arch' : 'arch'}]} + + Valid attributes are: ``epoch``, ``version``, ``release``, ``arch``, + ``install_date``, ``install_date_time_t``. + + If ``all`` is specified, all valid attributes will be returned. + + .. versionadded:: Oxygen + removed: not supported @@ -671,6 +684,7 @@ def list_pkgs(versions_as_list=False, **kwargs): .. code-block:: bash salt '*' pkg.list_pkgs + salt '*' pkg.list_pkgs attr='["version", "arch"]' ''' versions_as_list = salt.utils.is_true(versions_as_list) # not yet implemented or not applicable @@ -678,30 +692,30 @@ def list_pkgs(versions_as_list=False, **kwargs): for x in ('removed', 'purge_desired')]): return {} + attr = kwargs.get("attr") if 'pkg.list_pkgs' in __context__: - if versions_as_list: - return __context__['pkg.list_pkgs'] - else: - ret = copy.deepcopy(__context__['pkg.list_pkgs']) - __salt__['pkg_resource.stringify'](ret) - return ret + cached = __context__['pkg.list_pkgs'] + return __salt__['pkg_resource.format_pkg_list'](cached, versions_as_list, attr) - cmd = ['rpm', '-qa', '--queryformat', '%{NAME}_|-%{VERSION}_|-%{RELEASE}_|-%|EPOCH?{%{EPOCH}}:{}|\\n'] + cmd = ['rpm', '-qa', '--queryformat', ( + "%{NAME}_|-%{VERSION}_|-%{RELEASE}_|-%{ARCH}_|-" + "%|EPOCH?{%{EPOCH}}:{}|_|-%{INSTALLTIME}\\n")] ret = {} for line in __salt__['cmd.run'](cmd, output_loglevel='trace', python_shell=False).splitlines(): - name, pkgver, rel, epoch = line.split('_|-') - if epoch: - pkgver = '{0}:{1}'.format(epoch, pkgver) - if rel: - pkgver += '-{0}'.format(rel) - __salt__['pkg_resource.add_pkg'](ret, name, pkgver) + name, pkgver, rel, arch, epoch, install_time = line.split('_|-') + install_date = datetime.datetime.utcfromtimestamp(int(install_time)).isoformat() + "Z" + install_date_time_t = int(install_time) - __salt__['pkg_resource.sort_pkglist'](ret) - __context__['pkg.list_pkgs'] = copy.deepcopy(ret) - if not versions_as_list: - __salt__['pkg_resource.stringify'](ret) + all_attr = {'epoch': epoch, 'version': pkgver, 'release': rel, 'arch': arch, + 'install_date': install_date, 'install_date_time_t': install_date_time_t} + __salt__['pkg_resource.add_pkg'](ret, name, all_attr) - return ret + for pkgname in ret: + ret[pkgname] = sorted(ret[pkgname], key=lambda d: d['version']) + + __context__['pkg.list_pkgs'] = ret + + return __salt__['pkg_resource.format_pkg_list'](ret, versions_as_list, attr) def _get_configured_repos(): @@ -1067,11 +1081,43 @@ def install(name=None, Zypper returns error code 106 if one of the repositories are not available for various reasons. In case to set strict check, this parameter needs to be set to True. Default: False. + diff_attr: + If a list of package attributes is specified, returned value will + contain them, eg.:: + + {'': { + 'old': { + 'version': '', + 'arch': ''}, + + 'new': { + 'version': '', + 'arch': ''}}} + + Valid attributes are: ``epoch``, ``version``, ``release``, ``arch``, + ``install_date``, ``install_date_time_t``. + + If ``all`` is specified, all valid attributes will be returned. + + .. versionadded:: Oxygen + Returns a dict containing the new package names and versions:: {'': {'old': '', 'new': ''}} + + If an attribute list is specified in ``diff_attr``, the dict will also contain + any specified attribute, eg.:: + + {'': { + 'old': { + 'version': '', + 'arch': ''}, + + 'new': { + 'version': '', + 'arch': ''}}} ''' if refresh: refresh_db() @@ -1109,7 +1155,8 @@ def install(name=None, else: targets = pkg_params - old = list_pkgs() if not downloadonly else list_downloaded() + diff_attr = kwargs.get("diff_attr") + old = list_pkgs(attr=diff_attr) if not downloadonly else list_downloaded() downgrades = [] if fromrepo: fromrepoopt = ['--force', '--force-resolution', '--from', fromrepo] @@ -1147,19 +1194,22 @@ def install(name=None, __zypper__(no_repo_failure=ignore_repo_failure).call(*cmd) __context__.pop('pkg.list_pkgs', None) - new = list_pkgs() if not downloadonly else list_downloaded() + new = list_pkgs(attr=diff_attr) if not downloadonly else list_downloaded() # Handle packages which report multiple new versions # (affects only kernel packages at this point) - for pkg in new: - if isinstance(new[pkg], six.string_types): - new[pkg] = new[pkg].split(',')[-1] + for pkg_name in new: + pkg_data = new[pkg_name] + if isinstance(pkg_data, six.string_types): + new[pkg_name] = pkg_data.split(',')[-1] ret = salt.utils.compare_dicts(old, new) if errors: raise CommandExecutionError( - 'Problem encountered installing package(s)', + 'Problem encountered {0} package(s)'.format( + 'downloading' if downloadonly else 'installing' + ), info={'errors': errors, 'changes': ret} ) @@ -1428,7 +1478,7 @@ def list_locks(): ''' locks = {} if os.path.exists(LOCKS): - with salt.utils.fopen(LOCKS) as fhr: + with salt.utils.files.fopen(LOCKS) as fhr: for meta in [item.split('\n') for item in fhr.read().split('\n\n')]: lock = {} for element in [el for el in meta if el]: @@ -1804,7 +1854,7 @@ def list_products(all=False, refresh=False): if 'productline' in p_nfo and p_nfo['productline']: oem_file = os.path.join(OEM_PATH, p_nfo['productline']) if os.path.isfile(oem_file): - with salt.utils.fopen(oem_file, 'r') as rfile: + with salt.utils.files.fopen(oem_file, 'r') as rfile: oem_release = rfile.readline().strip() if oem_release: p_nfo['release'] = oem_release @@ -1864,7 +1914,7 @@ def download(*packages, **kwargs): def list_downloaded(): ''' - .. versionadded:: Oxygen + .. versionadded:: 2017.7.0 List prefetched packages downloaded by Zypper in the local disk. @@ -1948,7 +1998,7 @@ def _get_patches(installed_only=False): def list_patches(refresh=False): ''' - .. versionadded:: Oxygen + .. versionadded:: 2017.7.0 List all known advisory patches from available repos. @@ -1971,7 +2021,7 @@ def list_patches(refresh=False): def list_installed_patches(): ''' - .. versionadded:: Oxygen + .. versionadded:: 2017.7.0 List installed advisory patches on the system. diff --git a/salt/netapi/__init__.py b/salt/netapi/__init__.py index eabb9981a0d..99a33697f2a 100644 --- a/salt/netapi/__init__.py +++ b/salt/netapi/__init__.py @@ -19,7 +19,7 @@ import salt.client.ssh.client import salt.exceptions # Import third party libs -import salt.ext.six as six +from salt.ext import six class NetapiClient(object): diff --git a/salt/netapi/rest_cherrypy/__init__.py b/salt/netapi/rest_cherrypy/__init__.py index 6285de289c6..d974cda6c1d 100644 --- a/salt/netapi/rest_cherrypy/__init__.py +++ b/salt/netapi/rest_cherrypy/__init__.py @@ -20,7 +20,7 @@ try: except ImportError as exc: cpy_error = exc -__virtualname__ = os.path.abspath(__file__).rsplit('/')[-2] or 'rest_cherrypy' +__virtualname__ = os.path.abspath(__file__).rsplit(os.sep)[-2] or 'rest_cherrypy' logger = logging.getLogger(__virtualname__) cpy_min = '3.2.2' diff --git a/salt/netapi/rest_cherrypy/app.py b/salt/netapi/rest_cherrypy/app.py index 57d92001059..16a97da3243 100644 --- a/salt/netapi/rest_cherrypy/app.py +++ b/salt/netapi/rest_cherrypy/app.py @@ -3,18 +3,14 @@ A REST API for Salt =================== -.. versionadded:: 2014.7.0 - .. py:currentmodule:: salt.netapi.rest_cherrypy.app :depends: - - CherryPy Python module. Version 3.2.3 is currently recommended when - SSL is enabled, since this version worked the best with SSL in - internal testing. Versions 3.2.3 - 4.x can be used if SSL is not enabled. - Be aware that there is a known - `SSL error `_ - introduced in version 3.2.5. The issue was reportedly resolved with - CherryPy milestone 3.3, but the patch was committed for version 3.6.1. + - CherryPy Python module. + + Note: there is a `known SSL traceback for CherryPy versions 3.2.5 through + 3.7.x `_. Please use + version 3.2.3 or the latest 10.x version instead. :optdepends: - ws4py Python module for websockets support. :client_libraries: - Java: https://github.com/SUSE/salt-netapi-client @@ -112,6 +108,10 @@ A REST API for Salt Collect and report statistics about the CherryPy server Reports are available via the :py:class:`Stats` URL. + stats_disable_auth : False + Do not require authentication to access the ``/stats`` endpoint. + + .. versionadded:: Oxygen static A filesystem path to static HTML/JavaScript/CSS/image assets. static_path : ``/static`` @@ -584,25 +584,33 @@ import signal import tarfile from multiprocessing import Process, Pipe -# Import third-party libs -# pylint: disable=import-error -import cherrypy # pylint: disable=3rd-party-module-not-gated -import yaml -import salt.ext.six as six -# pylint: enable=import-error +logger = logging.getLogger(__name__) +# Import third-party libs +# pylint: disable=import-error, 3rd-party-module-not-gated +import cherrypy +try: + from cherrypy.lib import cpstats +except ImportError: + cpstats = None + logger.warn('Import of cherrypy.cpstats failed. ' + 'Possible upstream bug: ' + 'https://github.com/cherrypy/cherrypy/issues/1444') + +import yaml +# pylint: enable=import-error, 3rd-party-module-not-gated # Import Salt libs import salt import salt.auth import salt.utils import salt.utils.event +import salt.utils.stringutils +from salt.ext import six # Import salt-api libs import salt.netapi -logger = logging.getLogger(__name__) - # Imports related to websocket try: from .tools import websockets @@ -872,7 +880,7 @@ def hypermedia_handler(*args, **kwargs): try: response = out(ret) if six.PY3: - response = salt.utils.to_bytes(response) + response = salt.utils.stringutils.to_bytes(response) return response except Exception: msg = 'Could not serialize the return data from Salt.' @@ -930,7 +938,7 @@ def urlencoded_processor(entity): entity.fp.read(fp_out=contents) contents.seek(0) body_str = contents.read() - body_bytes = salt.utils.to_bytes(body_str) + body_bytes = salt.utils.stringutils.to_bytes(body_str) body_bytes = six.BytesIO(body_bytes) body_bytes.seek(0) # Patch fp @@ -2704,6 +2712,10 @@ class Stats(object): 'tools.salt_auth.on': True, }) + def __init__(self): + if cherrypy.config['apiopts'].get('stats_disable_auth'): + self._cp_config['tools.salt_auth.on'] = False + def GET(self): ''' Return a dump of statistics collected from the CherryPy server @@ -2720,13 +2732,6 @@ class Stats(object): :status 406: |406| ''' if hasattr(logging, 'statistics'): - # Late import - try: - from cherrypy.lib import cpstats - except ImportError: - logger.error('Import of cherrypy.cpstats failed. Possible ' - 'upstream bug here: https://github.com/cherrypy/cherrypy/issues/1444') - return {} return cpstats.extrapolate_statistics(logging.statistics) return {} @@ -2846,13 +2851,14 @@ class API(object): 'tools.trailing_slash.on': True, 'tools.gzip.on': True, - 'tools.cpstats.on': self.apiopts.get('collect_stats', False), - 'tools.html_override.on': True, 'tools.cors_tool.on': True, }, } + if cpstats and self.apiopts.get('collect_stats', False): + conf['/']['tools.cpstats.on'] = True + if 'favicon' in self.apiopts: conf['/favicon.ico'] = { 'tools.staticfile.on': True, diff --git a/salt/netapi/rest_cherrypy/event_processor.py b/salt/netapi/rest_cherrypy/event_processor.py index 3301e41e925..58ec010b00c 100644 --- a/salt/netapi/rest_cherrypy/event_processor.py +++ b/salt/netapi/rest_cherrypy/event_processor.py @@ -6,7 +6,7 @@ import json import logging # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six # Import Salt libs import salt.netapi diff --git a/salt/netapi/rest_tornado/__init__.py b/salt/netapi/rest_tornado/__init__.py index 56a88e34d3d..24377140735 100644 --- a/salt/netapi/rest_tornado/__init__.py +++ b/salt/netapi/rest_tornado/__init__.py @@ -10,7 +10,7 @@ import os import salt.auth from salt.utils.versions import StrictVersion as _StrictVersion -__virtualname__ = os.path.abspath(__file__).rsplit('/')[-2] or 'rest_tornado' +__virtualname__ = os.path.abspath(__file__).rsplit(os.sep)[-2] or 'rest_tornado' logger = logging.getLogger(__virtualname__) @@ -115,6 +115,7 @@ def start(): ssl_opts.update({'keyfile': mod_opts['ssl_key']}) kwargs['ssl_options'] = ssl_opts + import tornado.httpserver http_server = tornado.httpserver.HTTPServer(get_application(__opts__), **kwargs) try: http_server.bind(mod_opts['port'], diff --git a/salt/netapi/rest_tornado/event_processor.py b/salt/netapi/rest_tornado/event_processor.py index b8e947a5912..c644f123617 100644 --- a/salt/netapi/rest_tornado/event_processor.py +++ b/salt/netapi/rest_tornado/event_processor.py @@ -3,7 +3,7 @@ from __future__ import absolute_import import json import logging import threading -import salt.ext.six as six +from salt.ext import six import salt.netapi diff --git a/salt/netapi/rest_tornado/saltnado.py b/salt/netapi/rest_tornado/saltnado.py index cf7bd652a9c..d40edf44e0c 100644 --- a/salt/netapi/rest_tornado/saltnado.py +++ b/salt/netapi/rest_tornado/saltnado.py @@ -206,7 +206,7 @@ import tornado.web import tornado.gen from tornado.concurrent import Future from zmq.eventloop import ioloop -import salt.ext.six as six +from salt.ext import six # pylint: enable=import-error # instantiate the zmq IOLoop (specialized poller) @@ -523,8 +523,7 @@ class BaseSaltAPIHandler(tornado.web.RequestHandler, SaltClientsMixIn): # pylin try: # Use cgi.parse_header to correctly separate parameters from value - header = cgi.parse_header(self.request.headers['Content-Type']) - value, parameters = header + value, parameters = cgi.parse_header(self.request.headers['Content-Type']) return ct_in_map[value](tornado.escape.native_str(data)) except KeyError: self.send_error(406) @@ -538,7 +537,7 @@ class BaseSaltAPIHandler(tornado.web.RequestHandler, SaltClientsMixIn): # pylin if not self.request.body: return data = self.deserialize(self.request.body) - self.raw_data = copy(data) + self.request_payload = copy(data) if data and 'arg' in data and not isinstance(data['arg'], list): data['arg'] = [data['arg']] @@ -696,15 +695,13 @@ class SaltAuthHandler(BaseSaltAPIHandler): # pylint: disable=W0223 }} ''' try: - request_payload = self.deserialize(self.request.body) - - if not isinstance(request_payload, dict): + if not isinstance(self.request_payload, dict): self.send_error(400) return - creds = {'username': request_payload['username'], - 'password': request_payload['password'], - 'eauth': request_payload['eauth'], + creds = {'username': self.request_payload['username'], + 'password': self.request_payload['password'], + 'eauth': self.request_payload['eauth'], } # if any of the args are missing, its a bad request except KeyError: @@ -1641,7 +1638,7 @@ class WebhookSaltAPIHandler(SaltAPIHandler): # pylint: disable=W0223 value = value[0] arguments[argname] = value ret = self.event.fire_event({ - 'post': self.raw_data, + 'post': self.request_payload, 'get': arguments, # In Tornado >= v4.0.3, the headers come # back as an HTTPHeaders instance, which diff --git a/salt/output/__init__.py b/salt/output/__init__.py index d9ce631b4e3..6cb90a47e12 100644 --- a/salt/output/__init__.py +++ b/salt/output/__init__.py @@ -4,21 +4,24 @@ Used to manage the outputter system. This package is the modular system used for managing outputters. ''' -# Import python libs -from __future__ import print_function +# Import Python libs from __future__ import absolute_import -import re -import os -import sys +from __future__ import print_function import errno import logging +import os +import re +import sys import traceback -# Import salt libs +# Import Salt libs import salt.loader import salt.utils -import salt.ext.six as six -from salt.utils import print_cli +import salt.utils.files +import salt.utils.platform + +# Import 3rd-party libs +from salt.ext import six # Are you really sure !!! # dealing with unicode is not as simple as setting defaultencoding @@ -97,7 +100,7 @@ def display_output(data, out=None, opts=None, **kwargs): # output filename can be either '' or None if output_filename: if not hasattr(output_filename, 'write'): - ofh = salt.utils.fopen(output_filename, 'a') # pylint: disable=resource-leakage + ofh = salt.utils.files.fopen(output_filename, 'a') # pylint: disable=resource-leakage fh_opened = True else: # Filehandle/file-like object @@ -124,7 +127,7 @@ def display_output(data, out=None, opts=None, **kwargs): ofh.close() return if display_data: - print_cli(display_data) + salt.utils.print_cli(display_data) except IOError as exc: # Only raise if it's NOT a broken pipe if exc.errno != errno.EPIPE: @@ -164,10 +167,17 @@ def get_printout(out, opts=None, **kwargs): if opts.get('force_color', False): opts['color'] = True - elif opts.get('no_color', False) or is_pipe() or salt.utils.is_windows(): + elif opts.get('no_color', False) or is_pipe() or salt.utils.platform.is_windows(): opts['color'] = False else: opts['color'] = True + else: + if opts.get('force_color', False): + opts['color'] = True + elif opts.get('no_color', False) or salt.utils.platform.is_windows(): + opts['color'] = False + else: + pass outputters = salt.loader.outputters(opts) if out not in outputters: diff --git a/salt/output/highstate.py b/salt/output/highstate.py index 0de81af1245..c003cfc32cf 100644 --- a/salt/output/highstate.py +++ b/salt/output/highstate.py @@ -108,12 +108,13 @@ import pprint import textwrap # Import salt libs -import salt.utils +import salt.utils.color +import salt.utils.stringutils import salt.output from salt.utils.locales import sdecode # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six import logging @@ -157,7 +158,7 @@ def output(data, **kwargs): # pylint: disable=unused-argument def _format_host(host, data): host = sdecode(host) - colors = salt.utils.get_colors( + colors = salt.utils.color.get_colors( __opts__.get('color'), __opts__.get('color_theme')) tabular = __opts__.get('state_tabular', False) @@ -168,7 +169,7 @@ def _format_host(host, data): nchanges = 0 strip_colors = __opts__.get('strip_colors', True) - if isinstance(data, int) or isinstance(data, str): + if isinstance(data, int) or isinstance(data, six.string_types): # Data in this format is from saltmod.function, # so it is always a 'change' nchanges = 1 @@ -241,8 +242,15 @@ def _format_host(host, data): if ret['result'] is None: hcolor = colors['LIGHT_YELLOW'] tcolor = colors['LIGHT_YELLOW'] + + state_output = __opts__.get('state_output', 'full').lower() comps = [sdecode(comp) for comp in tname.split('_|-')] - if __opts__.get('state_output', 'full').lower() == 'filter': + + if state_output == 'mixed_id': + # Swap in the ID for the name. Refs #35137 + comps[2] = comps[1] + + if state_output.startswith('filter'): # By default, full data is shown for all types. However, return # data may be excluded by setting state_output_exclude to a # comma-separated list of True, False or None, or including the @@ -275,28 +283,17 @@ def _format_host(host, data): continue if str(ret['result']) in exclude: continue - elif __opts__.get('state_output', 'full').lower() == 'terse': - # Print this chunk in a terse way and continue in the - # loop + + elif any(( + state_output.startswith('terse'), + state_output.startswith('mixed') and ret['result'] is not False, # only non-error'd + state_output.startswith('changes') and ret['result'] and not schanged # non-error'd non-changed + )): + # Print this chunk in a terse way and continue in the loop msg = _format_terse(tcolor, comps, ret, colors, tabular) hstrs.append(msg) continue - elif __opts__.get('state_output', 'full').lower().startswith('mixed'): - if __opts__['state_output'] == 'mixed_id': - # Swap in the ID for the name. Refs #35137 - comps[2] = comps[1] - # Print terse unless it failed - if ret['result'] is not False: - msg = _format_terse(tcolor, comps, ret, colors, tabular) - hstrs.append(msg) - continue - elif __opts__.get('state_output', 'full').lower() == 'changes': - # Print terse if no error and no changes, otherwise, be - # verbose - if ret['result'] and not schanged: - msg = _format_terse(tcolor, comps, ret, colors, tabular) - hstrs.append(msg) - continue + state_lines = [ u'{tcolor}----------{colors[ENDC]}', u' {tcolor} ID: {comps[1]}{colors[ENDC]}', @@ -319,7 +316,7 @@ def _format_host(host, data): if six.PY2: ret['comment'] = str(ret['comment']).decode('utf-8') else: - ret['comment'] = salt.utils.to_str(ret['comment']) + ret['comment'] = salt.utils.stringutils.to_str(ret['comment']) except UnicodeDecodeError: # but try to continue on errors pass @@ -547,7 +544,7 @@ def _format_terse(tcolor, comps, ret, colors, tabular): if __opts__.get('state_output_profile', True) and 'start_time' in ret: fmt_string += u'{6[start_time]!s} [{6[duration]!s:>7} ms] ' fmt_string += u'{2:>10}.{3:<10} {4:7} Name: {1}{5}' - elif isinstance(tabular, str): + elif isinstance(tabular, six.string_types): fmt_string = tabular else: fmt_string = '' diff --git a/salt/output/key.py b/salt/output/key.py index dfa42b2e067..afb504918d2 100644 --- a/salt/output/key.py +++ b/salt/output/key.py @@ -8,9 +8,9 @@ The ``salt-key`` command makes use of this outputter to format its output. from __future__ import absolute_import # Import salt libs -import salt.utils import salt.output from salt.utils.locales import sdecode +import salt.utils.color def output(data, **kwargs): # pylint: disable=unused-argument @@ -18,7 +18,7 @@ def output(data, **kwargs): # pylint: disable=unused-argument Read in the dict structure generated by the salt key API methods and print the structure. ''' - color = salt.utils.get_colors( + color = salt.utils.color.get_colors( __opts__.get('color'), __opts__.get('color_theme')) strip_colors = __opts__.get('strip_colors', True) diff --git a/salt/output/nested.py b/salt/output/nested.py index 7d21dba49f9..563bf5768fd 100644 --- a/salt/output/nested.py +++ b/salt/output/nested.py @@ -25,14 +25,14 @@ Example output:: ''' from __future__ import absolute_import # Import python libs -import salt.utils.odict from numbers import Number # Import salt libs import salt.output -from salt.ext.six import string_types -from salt.utils import get_colors +import salt.utils.color import salt.utils.locales +import salt.utils.odict +from salt.ext.six import string_types class NestDisplay(object): @@ -41,7 +41,7 @@ class NestDisplay(object): ''' def __init__(self): self.__dict__.update( - get_colors( + salt.utils.color.get_colors( __opts__.get('color'), __opts__.get('color_theme') ) diff --git a/salt/output/newline_values_only.py b/salt/output/newline_values_only.py index bd186a62085..b5bd440b3bc 100644 --- a/salt/output/newline_values_only.py +++ b/salt/output/newline_values_only.py @@ -73,7 +73,7 @@ Output from __future__ import absolute_import # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six def _get_values(data): diff --git a/salt/output/no_return.py b/salt/output/no_return.py index d0f3394fd82..00f601badec 100644 --- a/salt/output/no_return.py +++ b/salt/output/no_return.py @@ -15,7 +15,10 @@ Example output:: from __future__ import absolute_import # Import salt libs -import salt.utils +import salt.utils.color + +# Import 3rd-party libs +from salt.ext import six class NestDisplay(object): @@ -23,33 +26,33 @@ class NestDisplay(object): Create generator for nested output ''' def __init__(self): - self.colors = salt.utils.get_colors( - __opts__.get('color'), - __opts__.get('color_theme')) + self.colors = salt.utils.color.get_colors( + __opts__.get(u'color'), + __opts__.get(u'color_theme')) def display(self, ret, indent, prefix, out): ''' Recursively iterate down through data structures to determine output ''' - if isinstance(ret, str): - lines = ret.split('\n') + if isinstance(ret, six.string_types): + lines = ret.split(u'\n') for line in lines: - out += '{0}{1}{2}{3}{4}\n'.format( - self.colors['RED'], - ' ' * indent, + out += u'{0}{1}{2}{3}{4}\n'.format( + self.colors[u'RED'], + u' ' * indent, prefix, line, - self.colors['ENDC']) + self.colors[u'ENDC']) elif isinstance(ret, dict): for key in sorted(ret): val = ret[key] - out += '{0}{1}{2}{3}{4}:\n'.format( - self.colors['CYAN'], + out += u'{0}{1}{2}{3}{4}:\n'.format( + self.colors[u'CYAN'], ' ' * indent, prefix, key, - self.colors['ENDC']) - out = self.display(val, indent + 4, '', out) + self.colors[u'ENDC']) + out = self.display(val, indent + 4, u'', out) return out @@ -58,4 +61,4 @@ def output(ret, **kwargs): # pylint: disable=unused-argument Display ret data ''' nest = NestDisplay() - return nest.display(ret, 0, '', '') + return nest.display(ret, 0, u'', u'') diff --git a/salt/output/overstatestage.py b/salt/output/overstatestage.py index ef77e10dc69..ee101e76200 100644 --- a/salt/output/overstatestage.py +++ b/salt/output/overstatestage.py @@ -11,10 +11,10 @@ and should not be called directly. from __future__ import absolute_import # Import Salt libs -import salt.utils +import salt.utils.color # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six # [{'group2': {'match': ['fedora17-2', 'fedora17-3'], # 'require': ['group1'], @@ -27,7 +27,7 @@ def output(data, **kwargs): # pylint: disable=unused-argument ''' Format the data for printing stage information from the overstate system ''' - colors = salt.utils.get_colors( + colors = salt.utils.color.get_colors( __opts__.get('color'), __opts__.get('color_theme')) ostr = '' diff --git a/salt/output/pony.py b/salt/output/pony.py index 816df86ee9a..dbdddd396eb 100644 --- a/salt/output/pony.py +++ b/salt/output/pony.py @@ -46,15 +46,15 @@ from __future__ import absolute_import import subprocess # Import Salt libs -import salt.utils import salt.utils.locales +import salt.utils.path __virtualname__ = 'pony' def __virtual__(): - if salt.utils.which('ponysay'): + if salt.utils.path.which('ponysay'): return __virtualname__ return False diff --git a/salt/output/table_out.py b/salt/output/table_out.py index 59c4cd94862..531eb96c0fc 100644 --- a/salt/output/table_out.py +++ b/salt/output/table_out.py @@ -42,12 +42,10 @@ from functools import reduce # pylint: disable=redefined-builtin # Import salt libs import salt.output -import salt.utils.locales from salt.ext.six import string_types -from salt.utils import get_colors -from salt.ext.six.moves import map # pylint: disable=redefined-builtin -from salt.ext.six.moves import zip # pylint: disable=redefined-builtin - +from salt.ext.six.moves import map, zip # pylint: disable=redefined-builtin +import salt.utils.color +import salt.utils.locales __virtualname__ = 'table' @@ -78,7 +76,7 @@ class TableDisplay(object): width=50, # column max width wrapfunc=None): # function wrapper self.__dict__.update( - get_colors( + salt.utils.color.get_colors( __opts__.get('color'), __opts__.get('color_theme') ) diff --git a/salt/output/txt.py b/salt/output/txt.py index c0ae9eae2ca..62c55ed5968 100644 --- a/salt/output/txt.py +++ b/salt/output/txt.py @@ -17,18 +17,21 @@ def output(data, **kwargs): # pylint: disable=unused-argument ''' Output the data in lines, very nice for running commands ''' - ret = '' - if hasattr(data, 'keys'): + ret = u'' + if hasattr(data, u'keys'): for key in data: value = data[key] # Don't blow up on non-strings try: for line in value.splitlines(): - ret += '{0}: {1}\n'.format(key, line) + ret += u'{0}: {1}\n'.format(key, line) except AttributeError: - ret += '{0}: {1}\n'.format(key, value) + ret += u'{0}: {1}\n'.format(key, value) else: - # For non-dictionary data, just use print - ret += '{0}\n'.format(pprint.pformat(data)) + try: + ret += data + u'\n' + except TypeError: + # For non-dictionary, non-string data, just use print + ret += u'{0}\n'.format(pprint.pformat(data)) return ret diff --git a/salt/output/virt_query.py b/salt/output/virt_query.py index 491c96ef35f..6f988524cbe 100644 --- a/salt/output/virt_query.py +++ b/salt/output/virt_query.py @@ -11,7 +11,7 @@ runner. from __future__ import absolute_import # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six def output(data, **kwargs): # pylint: disable=unused-argument diff --git a/salt/payload.py b/salt/payload.py index 1235a2e8b5d..70492e4faa0 100644 --- a/salt/payload.py +++ b/salt/payload.py @@ -16,11 +16,11 @@ import datetime import salt.log import salt.crypt import salt.transport.frame +import salt.utils.immutabletypes as immutabletypes from salt.exceptions import SaltReqTimeoutError -from salt.utils import immutabletypes # Import third party libs -import salt.ext.six as six +from salt.ext import six try: import zmq except ImportError: @@ -46,15 +46,15 @@ except ImportError: except ImportError: # TODO: Come up with a sane way to get a configured logfile # and write to the logfile when this error is hit also - LOG_FORMAT = '[%(levelname)-8s] %(message)s' + LOG_FORMAT = u'[%(levelname)-8s] %(message)s' salt.log.setup_console_logger(log_format=LOG_FORMAT) - log.fatal('Unable to import msgpack or msgpack_pure python modules') + log.fatal(u'Unable to import msgpack or msgpack_pure python modules') # Don't exit if msgpack is not available, this is to make local mode # work without msgpack #sys.exit(salt.defaults.exitcodes.EX_GENERIC) -if HAS_MSGPACK and not hasattr(msgpack, 'exceptions'): +if HAS_MSGPACK and not hasattr(msgpack, u'exceptions'): class PackValueError(Exception): ''' older versions of msgpack do not have PackValueError @@ -89,11 +89,11 @@ def format_payload(enc, **kwargs): Pass in the required arguments for a payload, the enc type and the cmd, then a list of keyword args to generate the body of the load dict. ''' - payload = {'enc': enc} + payload = {u'enc': enc} load = {} for key in kwargs: load[key] = kwargs[key] - payload['load'] = load + payload[u'load'] = load return package(payload) @@ -104,11 +104,11 @@ class Serial(object): ''' def __init__(self, opts): if isinstance(opts, dict): - self.serial = opts.get('serial', 'msgpack') - elif isinstance(opts, str): + self.serial = opts.get(u'serial', u'msgpack') + elif isinstance(opts, six.string_types): self.serial = opts else: - self.serial = 'msgpack' + self.serial = u'msgpack' def loads(self, msg, encoding=None, raw=False): ''' @@ -140,10 +140,13 @@ class Serial(object): if six.PY3 and encoding is None and not raw: ret = salt.transport.frame.decode_embedded_strs(ret) except Exception as exc: - log.critical('Could not deserialize msgpack message.' - 'This often happens when trying to read a file not in binary mode' - 'To see message payload, enable debug logging and retry. Exception: {0}'.format(exc)) - log.debug('Msgpack deserialization failure on message: {0}'.format(msg)) + log.critical( + u'Could not deserialize msgpack message. This often happens ' + u'when trying to read a file not in binary mode. ' + u'To see message payload, enable debug logging and retry. ' + u'Exception: %s', exc + ) + log.debug(u'Msgpack deserialization failure on message: %s', msg) gc.collect() raise finally: @@ -158,7 +161,7 @@ class Serial(object): fn_.close() if data: if six.PY3: - return self.loads(data, encoding='utf-8') + return self.loads(data, encoding=u'utf-8') else: return self.loads(data) @@ -215,7 +218,7 @@ class Serial(object): return msgpack.ExtType(78, obj) def dt_encode(obj): - datetime_str = obj.strftime("%Y%m%dT%H:%M:%S.%f") + datetime_str = obj.strftime(u"%Y%m%dT%H:%M:%S.%f") if msgpack.version >= (0, 4, 0): return msgpack.packb(datetime_str, default=default, use_bin_type=use_bin_type) else: @@ -241,7 +244,7 @@ class Serial(object): return obj def immutable_encoder(obj): - log.debug('IMMUTABLE OBJ: {0}'.format(obj)) + log.debug(u'IMMUTABLE OBJ: %s', obj) if isinstance(obj, immutabletypes.ImmutableDict): return dict(obj) if isinstance(obj, immutabletypes.ImmutableList): @@ -249,12 +252,12 @@ class Serial(object): if isinstance(obj, immutabletypes.ImmutableSet): return set(obj) - if "datetime.datetime" in str(e): + if u"datetime.datetime" in str(e): if msgpack.version >= (0, 4, 0): return msgpack.dumps(datetime_encoder(msg), use_bin_type=use_bin_type) else: return msgpack.dumps(datetime_encoder(msg)) - elif "Immutable" in str(e): + elif u"Immutable" in str(e): if msgpack.version >= (0, 4, 0): return msgpack.dumps(msg, default=immutable_encoder, use_bin_type=use_bin_type) else: @@ -287,9 +290,10 @@ class Serial(object): else: return msgpack.dumps(odict_encoder(msg)) except (SystemError, TypeError) as exc: # pylint: disable=W0705 - log.critical('Unable to serialize message! Consider upgrading msgpack. ' - 'Message which failed was {failed_message} ' - 'with exception {exception_message}').format(msg, exc) + log.critical( + u'Unable to serialize message! Consider upgrading msgpack. ' + u'Message which failed was %s, with exception %s', msg, exc + ) def dump(self, msg, fn_): ''' @@ -309,7 +313,7 @@ class SREQ(object): ''' Create a generic interface to wrap salt zeromq req calls. ''' - def __init__(self, master, id_='', serial='msgpack', linger=0, opts=None): + def __init__(self, master, id_=u'', serial=u'msgpack', linger=0, opts=None): self.master = master self.id_ = id_ self.serial = Serial(serial) @@ -323,20 +327,20 @@ class SREQ(object): ''' Lazily create the socket. ''' - if not hasattr(self, '_socket'): + if not hasattr(self, u'_socket'): # create a new one self._socket = self.context.socket(zmq.REQ) - if hasattr(zmq, 'RECONNECT_IVL_MAX'): + if hasattr(zmq, u'RECONNECT_IVL_MAX'): self._socket.setsockopt( zmq.RECONNECT_IVL_MAX, 5000 ) self._set_tcp_keepalive() - if self.master.startswith('tcp://['): + if self.master.startswith(u'tcp://['): # Hint PF type if bracket enclosed IPv6 address - if hasattr(zmq, 'IPV6'): + if hasattr(zmq, u'IPV6'): self._socket.setsockopt(zmq.IPV6, 1) - elif hasattr(zmq, 'IPV4ONLY'): + elif hasattr(zmq, u'IPV4ONLY'): self._socket.setsockopt(zmq.IPV4ONLY, 0) self._socket.linger = self.linger if self.id_: @@ -345,37 +349,37 @@ class SREQ(object): return self._socket def _set_tcp_keepalive(self): - if hasattr(zmq, 'TCP_KEEPALIVE') and self.opts: - if 'tcp_keepalive' in self.opts: + if hasattr(zmq, u'TCP_KEEPALIVE') and self.opts: + if u'tcp_keepalive' in self.opts: self._socket.setsockopt( - zmq.TCP_KEEPALIVE, self.opts['tcp_keepalive'] + zmq.TCP_KEEPALIVE, self.opts[u'tcp_keepalive'] ) - if 'tcp_keepalive_idle' in self.opts: + if u'tcp_keepalive_idle' in self.opts: self._socket.setsockopt( - zmq.TCP_KEEPALIVE_IDLE, self.opts['tcp_keepalive_idle'] + zmq.TCP_KEEPALIVE_IDLE, self.opts[u'tcp_keepalive_idle'] ) - if 'tcp_keepalive_cnt' in self.opts: + if u'tcp_keepalive_cnt' in self.opts: self._socket.setsockopt( - zmq.TCP_KEEPALIVE_CNT, self.opts['tcp_keepalive_cnt'] + zmq.TCP_KEEPALIVE_CNT, self.opts[u'tcp_keepalive_cnt'] ) - if 'tcp_keepalive_intvl' in self.opts: + if u'tcp_keepalive_intvl' in self.opts: self._socket.setsockopt( - zmq.TCP_KEEPALIVE_INTVL, self.opts['tcp_keepalive_intvl'] + zmq.TCP_KEEPALIVE_INTVL, self.opts[u'tcp_keepalive_intvl'] ) def clear_socket(self): ''' delete socket if you have it ''' - if hasattr(self, '_socket'): + if hasattr(self, u'_socket'): if isinstance(self.poller.sockets, dict): sockets = list(self.poller.sockets.keys()) for socket in sockets: - log.trace('Unregistering socket: {0}'.format(socket)) + log.trace(u'Unregistering socket: %s', socket) self.poller.unregister(socket) else: for socket in self.poller.sockets: - log.trace('Unregistering socket: {0}'.format(socket)) + log.trace(u'Unregistering socket: %s', socket) self.poller.unregister(socket[0]) del self._socket @@ -383,8 +387,8 @@ class SREQ(object): ''' Takes two arguments, the encryption type and the base payload ''' - payload = {'enc': enc} - payload['load'] = load + payload = {u'enc': enc} + payload[u'load'] = load pkg = self.serial.dumps(payload) self.socket.send(pkg) self.poller.register(self.socket, zmq.POLLIN) @@ -395,12 +399,15 @@ class SREQ(object): if polled: break if tries > 1: - log.info('SaltReqTimeoutError: after {0} seconds. (Try {1} of {2})'.format( - timeout, tried, tries)) + log.info( + u'SaltReqTimeoutError: after %s seconds. (Try %s of %s)', + timeout, tried, tries + ) if tried >= tries: self.clear_socket() raise SaltReqTimeoutError( - 'SaltReqTimeoutError: after {0} seconds, ran {1} tries'.format(timeout * tried, tried) + u'SaltReqTimeoutError: after {0} seconds, ran {1} ' + u'tries'.format(timeout * tried, tried) ) return self.serial.loads(self.socket.recv()) @@ -408,8 +415,8 @@ class SREQ(object): ''' Detect the encryption type based on the payload ''' - enc = payload.get('enc', 'clear') - load = payload.get('load', {}) + enc = payload.get(u'enc', u'clear') + load = payload.get(u'load', {}) return self.send(enc, load, tries, timeout) def destroy(self): diff --git a/salt/pillar/__init__.py b/salt/pillar/__init__.py index f27eaa7c30a..8a0756251ca 100644 --- a/salt/pillar/__init__.py +++ b/salt/pillar/__init__.py @@ -13,6 +13,7 @@ import logging import tornado.gen import sys import traceback +import inspect # Import salt libs import salt.loader @@ -23,21 +24,22 @@ import salt.transport import salt.utils.url import salt.utils.cache import salt.utils.crypt +import salt.utils.dictupdate +import salt.utils.args from salt.exceptions import SaltClientError from salt.template import compile_template from salt.utils.dictupdate import merge from salt.utils.odict import OrderedDict from salt.version import __version__ -from salt.utils.locales import decode_recursively # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six log = logging.getLogger(__name__) def get_pillar(opts, grains, minion_id, saltenv=None, ext=None, funcs=None, - pillar=None, pillarenv=None): + pillar_override=None, pillarenv=None, extra_minion_data=None): ''' Return the correct pillar driver based on the file_client option ''' @@ -54,14 +56,16 @@ def get_pillar(opts, grains, minion_id, saltenv=None, ext=None, funcs=None, log.info('Compiling pillar from cache') log.debug('get_pillar using pillar cache with ext: {0}'.format(ext)) return PillarCache(opts, grains, minion_id, saltenv, ext=ext, functions=funcs, - pillar=pillar, pillarenv=pillarenv) + pillar_override=pillar_override, pillarenv=pillarenv) return ptype(opts, grains, minion_id, saltenv, ext, functions=funcs, - pillar=pillar, pillarenv=pillarenv) + pillar_override=pillar_override, pillarenv=pillarenv, + extra_minion_data=extra_minion_data) # TODO: migrate everyone to this one! def get_async_pillar(opts, grains, minion_id, saltenv=None, ext=None, funcs=None, - pillar=None, pillarenv=None): + pillar_override=None, pillarenv=None, + extra_minion_data=None): ''' Return the correct pillar driver based on the file_client option ''' @@ -73,15 +77,62 @@ def get_async_pillar(opts, grains, minion_id, saltenv=None, ext=None, funcs=None 'local': AsyncPillar, }.get(file_client, AsyncPillar) return ptype(opts, grains, minion_id, saltenv, ext, functions=funcs, - pillar=pillar, pillarenv=pillarenv) + pillar_override=pillar_override, pillarenv=pillarenv, + extra_minion_data=extra_minion_data) -class AsyncRemotePillar(object): +class RemotePillarMixin(object): + ''' + Common remote pillar functionality + ''' + def get_ext_pillar_extra_minion_data(self, opts): + ''' + Returns the extra data from the minion's opts dict (the config file). + + This data will be passed to external pillar functions. + ''' + def get_subconfig(opts_key): + ''' + Returns a dict containing the opts key subtree, while maintaining + the opts structure + ''' + ret_dict = aux_dict = {} + config_val = opts + subkeys = opts_key.split(':') + # Build an empty dict with the opts path + for subkey in subkeys[:-1]: + aux_dict[subkey] = {} + aux_dict = aux_dict[subkey] + if not config_val.get(subkey): + # The subkey is not in the config + return {} + config_val = config_val[subkey] + if subkeys[-1] not in config_val: + return {} + aux_dict[subkeys[-1]] = config_val[subkeys[-1]] + return ret_dict + + extra_data = {} + if 'pass_to_ext_pillars' in opts: + if not isinstance(opts['pass_to_ext_pillars'], list): + log.exception('\'pass_to_ext_pillars\' config is malformed.') + raise SaltClientError('\'pass_to_ext_pillars\' config is ' + 'malformed.') + for key in opts['pass_to_ext_pillars']: + salt.utils.dictupdate.update(extra_data, + get_subconfig(key), + recursive_update=True, + merge_lists=True) + log.trace('ext_pillar_extra_data = {0}'.format(extra_data)) + return extra_data + + +class AsyncRemotePillar(RemotePillarMixin): ''' Get the pillar from the master ''' def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, - pillar=None, pillarenv=None): + pillar_override=None, pillarenv=None, extra_minion_data=None): self.opts = opts self.opts['environment'] = saltenv self.ext = ext @@ -90,16 +141,18 @@ class AsyncRemotePillar(object): self.channel = salt.transport.client.AsyncReqChannel.factory(opts) if pillarenv is not None: self.opts['pillarenv'] = pillarenv - elif self.opts.get('pillarenv_from_saltenv', False): - self.opts['pillarenv'] = saltenv - elif 'pillarenv' not in self.opts: - self.opts['pillarenv'] = None - self.pillar_override = {} - if pillar is not None: - if isinstance(pillar, dict): - self.pillar_override = pillar - else: - log.error('Pillar data must be a dictionary') + self.pillar_override = pillar_override or {} + if not isinstance(self.pillar_override, dict): + self.pillar_override = {} + log.error('Pillar data must be a dictionary') + self.extra_minion_data = extra_minion_data or {} + if not isinstance(self.extra_minion_data, dict): + self.extra_minion_data = {} + log.error('Extra minion data must be a dictionary') + salt.utils.dictupdate.update(self.extra_minion_data, + self.get_ext_pillar_extra_minion_data(opts), + recursive_update=True, + merge_lists=True) @tornado.gen.coroutine def compile_pillar(self): @@ -111,6 +164,7 @@ class AsyncRemotePillar(object): 'saltenv': self.opts['environment'], 'pillarenv': self.opts['pillarenv'], 'pillar_override': self.pillar_override, + 'extra_minion_data': self.extra_minion_data, 'ver': '2', 'cmd': '_pillar'} if self.ext: @@ -133,12 +187,12 @@ class AsyncRemotePillar(object): raise tornado.gen.Return(ret_pillar) -class RemotePillar(object): +class RemotePillar(RemotePillarMixin): ''' Get the pillar from the master ''' def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, - pillar=None, pillarenv=None): + pillar_override=None, pillarenv=None, extra_minion_data=None): self.opts = opts self.opts['environment'] = saltenv self.ext = ext @@ -147,16 +201,18 @@ class RemotePillar(object): self.channel = salt.transport.Channel.factory(opts) if pillarenv is not None: self.opts['pillarenv'] = pillarenv - elif self.opts.get('pillarenv_from_saltenv', False): - self.opts['pillarenv'] = saltenv - elif 'pillarenv' not in self.opts: - self.opts['pillarenv'] = None - self.pillar_override = {} - if pillar is not None: - if isinstance(pillar, dict): - self.pillar_override = pillar - else: - log.error('Pillar data must be a dictionary') + self.pillar_override = pillar_override or {} + if not isinstance(self.pillar_override, dict): + self.pillar_override = {} + log.error('Pillar data must be a dictionary') + self.extra_minion_data = extra_minion_data or {} + if not isinstance(self.extra_minion_data, dict): + self.extra_minion_data = {} + log.error('Extra minion data must be a dictionary') + salt.utils.dictupdate.update(self.extra_minion_data, + self.get_ext_pillar_extra_minion_data(opts), + recursive_update=True, + merge_lists=True) def compile_pillar(self): ''' @@ -167,6 +223,7 @@ class RemotePillar(object): 'saltenv': self.opts['environment'], 'pillarenv': self.opts['pillarenv'], 'pillar_override': self.pillar_override, + 'extra_minion_data': self.extra_minion_data, 'ver': '2', 'cmd': '_pillar'} if self.ext: @@ -181,15 +238,15 @@ class RemotePillar(object): '{1}'.format(type(ret_pillar).__name__, ret_pillar) ) return {} - return decode_recursively(ret_pillar) + return ret_pillar class PillarCache(object): ''' Return a cached pillar if it exists, otherwise cache it. - Pillar caches are structed in two diminensions: minion_id with a dict of saltenvs. - Each saltenv contains a pillar dict + Pillar caches are structed in two diminensions: minion_id with a dict of + saltenvs. Each saltenv contains a pillar dict Example data structure: @@ -200,7 +257,7 @@ class PillarCache(object): ''' # TODO ABC? def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, - pillar=None, pillarenv=None): + pillar_override=None, pillarenv=None, extra_minion_data=None): # Yes, we need all of these because we need to route to the Pillar object # if we have no cache. This is another refactor target. @@ -210,7 +267,7 @@ class PillarCache(object): self.minion_id = minion_id self.ext = ext self.functions = functions - self.pillar = pillar + self.pillar_override = pillar_override self.pillarenv = pillarenv if saltenv is None: @@ -239,14 +296,14 @@ class PillarCache(object): ''' log.debug('Pillar cache getting external pillar with ext: {0}'.format(self.ext)) fresh_pillar = Pillar(self.opts, - self.grains, - self.minion_id, - self.saltenv, - ext=self.ext, - functions=self.functions, - pillar=self.pillar, - pillarenv=self.pillarenv) - return fresh_pillar.compile_pillar() # FIXME We are not yet passing pillar_dirs in here + self.grains, + self.minion_id, + self.saltenv, + ext=self.ext, + functions=self.functions, + pillar_override=self.pillar_override, + pillarenv=self.pillarenv) + return fresh_pillar.compile_pillar() def compile_pillar(self, *args, **kwargs): # Will likely just be pillar_dirs log.debug('Scanning pillar cache for information about minion {0} and saltenv {1}'.format(self.minion_id, self.saltenv)) @@ -278,7 +335,7 @@ class Pillar(object): Read over the pillar top files and render the pillar data ''' def __init__(self, opts, grains, minion_id, saltenv, ext=None, functions=None, - pillar=None, pillarenv=None): + pillar_override=None, pillarenv=None, extra_minion_data=None): self.minion_id = minion_id self.ext = ext if pillarenv is None: @@ -320,12 +377,14 @@ class Pillar(object): self.ext_pillars = salt.loader.pillars(ext_pillar_opts, self.functions) self.ignored_pillars = {} - self.pillar_override = {} - if pillar is not None: - if isinstance(pillar, dict): - self.pillar_override = pillar - else: - log.error('Pillar data must be a dictionary') + self.pillar_override = pillar_override or {} + if not isinstance(self.pillar_override, dict): + self.pillar_override = {} + log.error('Pillar data must be a dictionary') + self.extra_minion_data = extra_minion_data or {} + if not isinstance(self.extra_minion_data, dict): + self.extra_minion_data = {} + log.error('Extra minion data must be a dictionary') def __valid_on_demand_ext_pillar(self, opts): ''' @@ -431,20 +490,19 @@ class Pillar(object): self.opts['pillarenv'], ', '.join(self.opts['file_roots']) ) else: - tops[self.opts['pillarenv']] = [ - compile_template( - self.client.cache_file( - self.opts['state_top'], - self.opts['pillarenv'] - ), - self.rend, - self.opts['renderer'], - self.opts['renderer_blacklist'], - self.opts['renderer_whitelist'], - self.opts['pillarenv'], - _pillar_rend=True, - ) - ] + top = self.client.cache_file(self.opts['state_top'], self.opts['pillarenv']) + if top: + tops[self.opts['pillarenv']] = [ + compile_template( + top, + self.rend, + self.opts['renderer'], + self.opts['renderer_blacklist'], + self.opts['renderer_whitelist'], + self.opts['pillarenv'], + _pillar_rend=True, + ) + ] else: for saltenv in self._get_envs(): if self.opts.get('pillar_source_merging_strategy', None) == "none": @@ -582,7 +640,7 @@ class Pillar(object): merged_tops = self.merge_tops(tops) except TypeError as err: merged_tops = OrderedDict() - errors.append('Error encountered while render pillar top file.') + errors.append('Error encountered while rendering pillar top file.') return merged_tops, errors def top_matches(self, top): @@ -778,35 +836,43 @@ class Pillar(object): return pillar, errors - def _external_pillar_data(self, pillar, val, pillar_dirs, key): + def _external_pillar_data(self, pillar, val, key): ''' Builds actual pillar data structure and updates the ``pillar`` variable ''' ext = None + args = salt.utils.args.get_function_argspec(self.ext_pillars[key]).args if isinstance(val, dict): - ext = self.ext_pillars[key](self.minion_id, pillar, **val) + if ('extra_minion_data' in args) and self.extra_minion_data: + ext = self.ext_pillars[key]( + self.minion_id, pillar, + extra_minion_data=self.extra_minion_data, **val) + else: + ext = self.ext_pillars[key](self.minion_id, pillar, **val) elif isinstance(val, list): - if key == 'git': - ext = self.ext_pillars[key](self.minion_id, - val, - pillar_dirs) + if ('extra_minion_data' in args) and self.extra_minion_data: + ext = self.ext_pillars[key]( + self.minion_id, pillar, *val, + extra_minion_data=self.extra_minion_data) else: ext = self.ext_pillars[key](self.minion_id, pillar, *val) else: - if key == 'git': - ext = self.ext_pillars[key](self.minion_id, - val, - pillar_dirs) + if ('extra_minion_data' in args) and self.extra_minion_data: + ext = self.ext_pillars[key]( + self.minion_id, + pillar, + val, + extra_minion_data=self.extra_minion_data) else: ext = self.ext_pillars[key](self.minion_id, pillar, val) return ext - def ext_pillar(self, pillar, pillar_dirs, errors=None): + def ext_pillar(self, pillar, errors=None): ''' Render the external pillar data ''' @@ -835,7 +901,7 @@ class Pillar(object): return pillar, errors ext = None # Bring in CLI pillar data - if self.pillar_override and isinstance(self.pillar_override, dict): + if self.pillar_override: pillar = merge(pillar, self.pillar_override, self.merge_strategy, @@ -858,9 +924,8 @@ class Pillar(object): continue try: ext = self._external_pillar_data(pillar, - val, - pillar_dirs, - key) + val, + key) except Exception as exc: errors.append( 'Failed to load ext_pillar {0}: {1}'.format( @@ -882,16 +947,14 @@ class Pillar(object): ext = None return pillar, errors - def compile_pillar(self, ext=True, pillar_dirs=None): + def compile_pillar(self, ext=True): ''' Render the pillar data and return ''' top, top_errors = self.get_top() if ext: if self.opts.get('ext_pillar_first', False): - self.opts['pillar'], errors = self.ext_pillar( - self.pillar_override, - pillar_dirs) + self.opts['pillar'], errors = self.ext_pillar(self.pillar_override) self.rend = salt.loader.render(self.opts, self.functions) matches = self.top_matches(top) pillar, errors = self.render_pillar(matches, errors=errors) @@ -903,8 +966,7 @@ class Pillar(object): else: matches = self.top_matches(top) pillar, errors = self.render_pillar(matches) - pillar, errors = self.ext_pillar( - pillar, pillar_dirs, errors=errors) + pillar, errors = self.ext_pillar(pillar, errors=errors) else: matches = self.top_matches(top) pillar, errors = self.render_pillar(matches) @@ -922,7 +984,7 @@ class Pillar(object): log.critical('Pillar render error: {0}'.format(error)) pillar['_errors'] = errors - if self.pillar_override and isinstance(self.pillar_override, dict): + if self.pillar_override: pillar = merge(pillar, self.pillar_override, self.merge_strategy, @@ -999,6 +1061,6 @@ class Pillar(object): # ext_pillar etc. class AsyncPillar(Pillar): @tornado.gen.coroutine - def compile_pillar(self, ext=True, pillar_dirs=None): - ret = super(AsyncPillar, self).compile_pillar(ext=ext, pillar_dirs=pillar_dirs) + def compile_pillar(self, ext=True): + ret = super(AsyncPillar, self).compile_pillar(ext=ext) raise tornado.gen.Return(ret) diff --git a/salt/pillar/consul_pillar.py b/salt/pillar/consul_pillar.py index 0d10b80c363..d661649f4f7 100644 --- a/salt/pillar/consul_pillar.py +++ b/salt/pillar/consul_pillar.py @@ -167,7 +167,8 @@ def ext_pillar(minion_id, opts['target'] = match.group(1) temp = temp.replace(match.group(0), '') checker = salt.utils.minions.CkMinions(__opts__) - minions = checker.check_minions(opts['target'], 'compound') + _res = checker.check_minions(opts['target'], 'compound') + minions = _res['minions'] if minion_id not in minions: return {} diff --git a/salt/pillar/csvpillar.py b/salt/pillar/csvpillar.py index b5355501fe5..352fca95467 100644 --- a/salt/pillar/csvpillar.py +++ b/salt/pillar/csvpillar.py @@ -48,7 +48,7 @@ Will produce the following Pillar values for a minion named "jerry": from __future__ import absolute_import import csv -import salt.utils +import salt.utils.files __virtualname__ = 'csv' @@ -76,7 +76,7 @@ def ext_pillar( :param list fieldnames: (Optional) if the first row of the CSV is not column names they may be specified here instead. ''' - with salt.utils.fopen(path, 'rb') as f: + with salt.utils.files.fopen(path, 'rb') as f: sheet = csv.DictReader(f, fieldnames, restkey=restkey, restval=restval, dialect=dialect) diff --git a/salt/pillar/django_orm.py b/salt/pillar/django_orm.py index 833a13d1011..b87260110df 100644 --- a/salt/pillar/django_orm.py +++ b/salt/pillar/django_orm.py @@ -96,8 +96,9 @@ import os import sys import salt.exceptions -import salt.ext.six as six +from salt.ext import six import salt.utils +import salt.utils.stringutils HAS_VIRTUALENV = False @@ -178,14 +179,14 @@ def ext_pillar(minion_id, # pylint: disable=W0613 base_env = {} proc = subprocess.Popen(['bash', '-c', 'env'], stdout=subprocess.PIPE) for line in proc.stdout: - (key, _, value) = salt.utils.to_str(line).partition('=') + (key, _, value) = salt.utils.stringutils.to_str(line).partition('=') base_env[key] = value command = ['bash', '-c', 'source {0} && env'.format(env_file)] proc = subprocess.Popen(command, stdout=subprocess.PIPE) for line in proc.stdout: - (key, _, value) = salt.utils.to_str(line).partition('=') + (key, _, value) = salt.utils.stringutils.to_str(line).partition('=') # only add a key if it is different or doesn't already exist if key not in base_env or base_env[key] != value: os.environ[key] = value.rstrip('\n') diff --git a/salt/pillar/file_tree.py b/salt/pillar/file_tree.py index 1a825f80a80..323958e2f91 100644 --- a/salt/pillar/file_tree.py +++ b/salt/pillar/file_tree.py @@ -180,8 +180,8 @@ import os # Import salt libs import salt.loader -import salt.utils import salt.utils.dictupdate +import salt.utils.files import salt.utils.minions import salt.utils.stringio import salt.template @@ -261,7 +261,7 @@ def _construct_pillar(top_dir, contents = '' try: - with salt.utils.fopen(file_path, 'rb') as fhr: + with salt.utils.files.fopen(file_path, 'rb') as fhr: buf = fhr.read(__opts__['file_buffer_size']) while buf: contents += buf @@ -336,9 +336,10 @@ def ext_pillar(minion_id, if (os.path.isdir(nodegroups_dir) and nodegroup in master_ngroups): ckminions = salt.utils.minions.CkMinions(__opts__) - match = ckminions.check_minions( + _res = ckminions.check_minions( master_ngroups[nodegroup], 'compound') + match = _res['minions'] if minion_id in match: ngroup_dir = os.path.join( nodegroups_dir, str(nodegroup)) diff --git a/salt/pillar/git_pillar.py b/salt/pillar/git_pillar.py index 8b85d14ab32..53e58be0ac1 100644 --- a/salt/pillar/git_pillar.py +++ b/salt/pillar/git_pillar.py @@ -3,12 +3,6 @@ Use a git repository as a Pillar source --------------------------------------- -.. note:: - This external pillar has been rewritten for the :ref:`2015.8.0 - ` release. The old method of configuring this - external pillar will be maintained for a couple releases, allowing time for - configurations to be updated to reflect the new usage. - This external pillar allows for a Pillar top file and Pillar SLS files to be sourced from a git repository. @@ -41,8 +35,7 @@ the repo's URL. Configuration details can be found below. - bar Additionally, while git_pillar allows for the branch/tag to be overridden - (see :ref:`here `, or :ref:`here - ` for Salt releases before 2015.8.0), keep in + (see :ref:`here `), keep in mind that the top file must reference the actual environment name. It is common practice to make the environment in a git_pillar top file match the branch/tag name, but when remapping, the environment of course no longer @@ -51,113 +44,10 @@ the repo's URL. Configuration details can be found below. common misconfiguration that may be to blame, and is a good first step in troubleshooting. -.. _git-pillar-pre-2015-8-0: +.. _git-pillar-configuration: -Configuring git_pillar for Salt releases before 2015.8.0 -======================================================== - -.. note:: - This legacy configuration for git_pillar will no longer be supported as of - the **Oxygen** release of Salt. - -For Salt releases earlier than :ref:`2015.8.0 `, -GitPython is the only supported provider for git_pillar. Individual -repositories can be configured under the :conf_master:`ext_pillar` -configuration parameter like so: - -.. code-block:: yaml - - ext_pillar: - - git: master https://gitserver/git-pillar.git root=subdirectory - -The repository is specified in the format `` ``, with an -optional ``root`` parameter (added in the :ref:`2014.7.0 -` release) which allows the pillar SLS files to be -served up from a subdirectory (similar to :conf_master:`gitfs_root` in gitfs). - -To use more than one branch from the same repo, multiple lines must be -specified under :conf_master:`ext_pillar`: - -.. code-block:: yaml - - ext_pillar: - - git: master https://gitserver/git-pillar.git - - git: dev https://gitserver/git-pillar.git - -.. _git-pillar-env-remap-legacy: - -To remap a specific branch to a specific Pillar environment, use the format -``:``: - -.. code-block:: yaml - - ext_pillar: - - git: develop:dev https://gitserver/git-pillar.git - - git: master:prod https://gitserver/git-pillar.git - -In this case, the ``develop`` branch would need its own ``top.sls`` with a -``dev`` section in it, like this: - -.. code-block:: yaml - - dev: - '*': - - bar - -The ``master`` branch would need its own ``top.sls`` with a ``prod`` section in -it: - -.. code-block:: yaml - - prod: - '*': - - bar - -If ``__env__`` is specified as the branch name, then git_pillar will first look -at the minion's :conf_minion:`environment` option. If unset, it will fall back -to using branch specified by the master's :conf_master:`gitfs_base`: - -.. code-block:: yaml - - ext_pillar: - - git: __env__ https://gitserver/git-pillar.git root=pillar - -The corresponding Pillar top file would look like this: - -.. code-block:: yaml - - {{saltenv}}: - '*': - - bar - -.. note:: - This feature was unintentionally omitted when git_pillar was rewritten for - the 2015.8.0 release. It was added again in the 2016.3.4 release, but it - has changed slightly in that release. On Salt masters running 2015.8.0 - through 2016.3.3, this feature can only be accessed using the legacy config - described above. For 2016.3.4 and later, refer to explanation of the - ``__env__`` parameter in the below section. - - Versions 2016.3.0 through 2016.3.4 incorrectly check the *master's* - ``environment`` config option (instead of the minion's) before falling back - to :conf_master:`gitfs_base`. This has been fixed in the 2016.3.5 and - 2016.11.1 releases (2016.11.0 contains the incorrect behavior). - - Additionally, in releases before 2016.11.0, both ``{{env}}`` and - ``{{saltenv}}`` could be used as a placeholder for the environment. - Starting in 2016.11.0, ``{{env}}`` is no longer supported. - -.. _git-pillar-2015-8-0-and-later: - -Configuring git_pillar for Salt releases 2015.8.0 and later -=========================================================== - -.. note:: - In version 2015.8.0, the method of configuring git external pillars has - changed, and now more closely resembles that of the :ref:`Git Fileserver - Backend `. If Salt detects the old configuration schema, it - will use the pre-2015.8.0 code to compile the external pillar. A warning - will also be logged. +Configuring git_pillar for Salt +=============================== Beginning with Salt version 2015.8.0, pygit2_ is now supported in addition to GitPython_. The requirements for GitPython_ and pygit2_ are the same as for @@ -258,32 +148,6 @@ The corresponding Pillar top file would look like this: '*': - bar -.. note:: - This feature was unintentionally omitted when git_pillar was rewritten for - the 2015.8.0 release. It was added again in the 2016.3.4 release, but it - has changed slightly in that release. The fallback value replaced by - ``{{env}}`` is :conf_master: is :conf_master:`git_pillar_base`, while the - legacy config's version of this feature replaces ``{{env}}`` with - :conf_master:`gitfs_base`. - - On Salt masters running 2015.8.0 through 2016.3.3, this feature can only be - accessed using the legacy config in the previous section of this page. - - The same issue which affected the behavior of the minion's - :conf_minion:`environment` config value using the legacy configuration - syntax (see the documentation in the pre-2015.8.0 section above for the - legacy support of this feature) also affects the new-style git_pillar - syntax in version 2016.3.4. This has been corrected in version 2016.3.5 and - 2016.11.1 (2016.11.0 contains the incorrect behavior). - - 2016.3.4 incorrectly checks the *master's* ``environment`` config option - (instead of the minion's) before falling back to the master's - :conf_master:`git_pillar_base`. - - Additionally, in releases before 2016.11.0, both ``{{env}}`` and - ``{{saltenv}}`` could be used as a placeholder for the environment. - Starting in 2016.11.0, ``{{env}}`` is no longer supported. - With the addition of pygit2_ support, git_pillar can now interact with authenticated remotes. Authentication works just like in gitfs (as outlined in the :ref:`Git Fileserver Backend Walkthrough `), only @@ -469,25 +333,17 @@ from __future__ import absolute_import # Import python libs import copy import logging -import hashlib -import os # Import salt libs -import salt.utils import salt.utils.gitfs import salt.utils.dictupdate +import salt.utils.stringutils +import salt.utils.versions from salt.exceptions import FileserverConfigError from salt.pillar import Pillar # Import third party libs -import salt.ext.six as six -# pylint: disable=import-error -try: - import git - HAS_GITPYTHON = True -except ImportError: - HAS_GITPYTHON = False -# pylint: enable=import-error +from salt.ext import six PER_REMOTE_OVERRIDES = ('env', 'root', 'ssl_verify', 'refspecs') PER_REMOTE_ONLY = ('name', 'mountpoint') @@ -508,339 +364,89 @@ def __virtual__(): # No git external pillars were configured return False - for ext_pillar in git_ext_pillars: - if isinstance(ext_pillar['git'], six.string_types): - # Verification of legacy git pillar configuration - if not HAS_GITPYTHON: - log.error( - 'Git-based ext_pillar is enabled in configuration but ' - 'could not be loaded, is GitPython installed?' - ) - return False - if not git.__version__ > '0.3.0': - return False - return __virtualname__ - else: - # Verification of new git pillar configuration - try: - salt.utils.gitfs.GitPillar(__opts__) - # Initialization of the GitPillar object did not fail, so we - # know we have valid configuration syntax and that a valid - # provider was detected. - return __virtualname__ - except FileserverConfigError: - pass - return False + try: + salt.utils.gitfs.GitPillar(__opts__) + # Initialization of the GitPillar object did not fail, so we + # know we have valid configuration syntax and that a valid + # provider was detected. + return __virtualname__ + except FileserverConfigError: + return False -def ext_pillar(minion_id, repo, pillar_dirs): +def ext_pillar(minion_id, repo): ''' Checkout the ext_pillar sources and compile the resulting pillar SLS ''' - if isinstance(repo, six.string_types): - return _legacy_git_pillar(minion_id, repo, pillar_dirs) - else: - opts = copy.deepcopy(__opts__) - opts['pillar_roots'] = {} - opts['__git_pillar'] = True - pillar = salt.utils.gitfs.GitPillar(opts) - pillar.init_remotes(repo, PER_REMOTE_OVERRIDES, PER_REMOTE_ONLY) - if __opts__.get('__role') == 'minion': - # If masterless, fetch the remotes. We'll need to remove this once - # we make the minion daemon able to run standalone. - pillar.fetch_remotes() - pillar.checkout() - ret = {} - merge_strategy = __opts__.get( - 'pillar_source_merging_strategy', - 'smart' - ) - merge_lists = __opts__.get( - 'pillar_merge_lists', - False - ) - for pillar_dir, env in six.iteritems(pillar.pillar_dirs): - # If pillarenv is set, only grab pillars with that match pillarenv - if opts['pillarenv'] and env != opts['pillarenv']: - log.debug( - 'env \'%s\' for pillar dir \'%s\' does not match ' - 'pillarenv \'%s\', skipping', - env, pillar_dir, opts['pillarenv'] - ) - continue - if pillar_dir in pillar.pillar_linked_dirs: - log.debug( - 'git_pillar is skipping processing on %s as it is a ' - 'mounted repo', pillar_dir - ) - continue - else: - log.debug( - 'git_pillar is processing pillar SLS from %s for pillar ' - 'env \'%s\'', pillar_dir, env - ) - - if env == '__env__': - env = opts.get('pillarenv') \ - or opts.get('environment') \ - or opts.get('git_pillar_base') - log.debug('__env__ maps to %s', env) - - pillar_roots = [pillar_dir] - - if __opts__['git_pillar_includes']: - # Add the rest of the pillar_dirs in this environment to the - # list, excluding the current pillar_dir being processed. This - # is because it was already specified above as the first in the - # list, so that its top file is sourced from the correct - # location and not from another git_pillar remote. - pillar_roots.extend( - [d for (d, e) in six.iteritems(pillar.pillar_dirs) - if env == e and d != pillar_dir] - ) - - opts['pillar_roots'] = {env: pillar_roots} - - local_pillar = Pillar(opts, __grains__, minion_id, env) - ret = salt.utils.dictupdate.merge( - ret, - local_pillar.compile_pillar(ext=False), - strategy=merge_strategy, - merge_lists=merge_lists - ) - return ret - - -# Legacy git_pillar code -class _LegacyGitPillar(object): - ''' - Deal with the remote git repository for Pillar - ''' - - def __init__(self, branch, repo_location, opts): - ''' - Try to initialize the Git repo object - ''' - self.branch = self.map_branch(branch, opts) - self.rp_location = repo_location - self.opts = opts - self._envs = set() - self.working_dir = '' - self.repo = None - - hash_type = getattr(hashlib, opts['hash_type']) - hash_str = '{0} {1}'.format(self.branch, self.rp_location) - repo_hash = hash_type(salt.utils.to_bytes(hash_str)).hexdigest() - rp_ = os.path.join(self.opts['cachedir'], 'pillar_gitfs', repo_hash) - - if not os.path.isdir(rp_): - os.makedirs(rp_) - try: - self.repo = git.Repo.init(rp_) - except (git.exc.NoSuchPathError, - git.exc.InvalidGitRepositoryError) as exc: - log.error( - 'GitPython exception caught while initializing the repo: %s. ' - 'Maybe the git CLI program is not available.', exc - ) - except Exception as exc: - log.exception('Undefined exception in git pillar. ' - 'This may be a bug should be reported to the ' - 'SaltStack developers.') - - # Git directory we are working on - # Should be the same as self.repo.working_dir - self.working_dir = rp_ - - if isinstance(self.repo, git.Repo): - if not self.repo.remotes: - try: - self.repo.create_remote('origin', self.rp_location) - # ignore git ssl verification if requested - if self.opts.get('pillar_gitfs_ssl_verify', True): - self.repo.git.config('http.sslVerify', 'true') - else: - self.repo.git.config('http.sslVerify', 'false') - except os.error: - # This exception occurs when two processes are - # trying to write to the git config at once, go - # ahead and pass over it since this is the only - # write. - # This should place a lock down. - pass - else: - if self.repo.remotes.origin.url != self.rp_location: - self.repo.remotes.origin.config_writer.set( - 'url', self.rp_location) - - def map_branch(self, branch, opts=None): - opts = __opts__ if opts is None else opts - if branch == '__env__': - branch = opts.get('environment') or 'base' - if branch == 'base': - branch = opts.get('gitfs_base') or 'master' - elif ':' in branch: - branch = branch.split(':', 1)[0] - return branch - - def update(self): - ''' - Ensure you are following the latest changes on the remote - - Return boolean whether it worked - ''' - try: - log.debug('Legacy git_pillar: Updating \'%s\'', self.rp_location) - self.repo.git.fetch() - except git.exc.GitCommandError as exc: - log.error( - 'Unable to fetch the latest changes from remote %s: %s', - self.rp_location, exc - ) - return False - - try: - checkout_ref = 'origin/{0}'.format(self.branch) - log.debug('Legacy git_pillar: Checking out %s for \'%s\'', - checkout_ref, self.rp_location) - self.repo.git.checkout(checkout_ref) - except git.exc.GitCommandError as exc: - log.error( - 'Legacy git_pillar: Failed to checkout %s for \'%s\': %s', - checkout_ref, self.rp_location, exc - ) - return False - - return True - - def envs(self): - ''' - Return a list of refs that can be used as environments - ''' - if isinstance(self.repo, git.Repo): - remote = self.repo.remote() - for ref in self.repo.refs: - parted = ref.name.partition('/') - short = parted[2] if parted[2] else parted[0] - if isinstance(ref, git.Head): - if short == 'master': - short = 'base' - if ref not in remote.stale_refs: - self._envs.add(short) - elif isinstance(ref, git.Tag): - self._envs.add(short) - - return list(self._envs) - - -def _legacy_git_pillar(minion_id, repo_string, pillar_dirs): - ''' - Support pre-Beryllium config schema - ''' - salt.utils.warn_until( - 'Oxygen', - 'The git ext_pillar configuration is deprecated. Please refer to the ' - 'documentation at ' - 'https://docs.saltstack.com/en/latest/ref/pillar/all/salt.pillar.git_pillar.html ' - 'for more information. This configuration will no longer be supported ' - 'as of the Oxygen release of Salt.' - ) - if pillar_dirs is None: - return - # split the branch, repo name and optional extra (key=val) parameters. - options = repo_string.strip().split() - branch_env = options[0] - repo_location = options[1] - root = '' - - for extraopt in options[2:]: - # Support multiple key=val attributes as custom parameters. - DELIM = '=' - if DELIM not in extraopt: - log.error( - 'Legacy git_pillar: Incorrectly formatted extra parameter ' - '\'%s\' within \'%s\' missing \'%s\')', - extraopt, repo_string, DELIM - ) - key, val = _extract_key_val(extraopt, DELIM) - if key == 'root': - root = val - else: - log.error( - 'Legacy git_pillar: Unrecognized extra parameter \'%s\' ' - 'in \'%s\'', - key, repo_string - ) - - # environment is "different" from the branch - cfg_branch, _, environment = branch_env.partition(':') - - gitpil = _LegacyGitPillar(cfg_branch, repo_location, __opts__) - branch = gitpil.branch - - if environment == '': - if branch == 'master': - environment = 'base' - else: - environment = branch - - # normpath is needed to remove appended '/' if root is empty string. - pillar_dir = os.path.normpath(os.path.join(gitpil.working_dir, root)) - log.debug( - 'Legacy git_pillar: pillar_dir for \'%s\' is \'%s\'', - repo_string, pillar_dir - ) - log.debug( - 'Legacy git_pillar: branch for \'%s\' is \'%s\'', - repo_string, branch - ) - - pillar_dirs.setdefault(pillar_dir, {}) - - if cfg_branch == '__env__' and branch not in ['master', 'base']: - gitpil.update() - elif pillar_dirs[pillar_dir].get(branch, False): - log.debug( - 'Already processed pillar_dir \'%s\' for \'%s\'', - pillar_dir, repo_string - ) - return {} # we've already seen this combo - - pillar_dirs[pillar_dir].setdefault(branch, True) - - # Don't recurse forever-- the Pillar object will re-call the ext_pillar - # function - if __opts__['pillar_roots'].get(branch, []) == [pillar_dir]: - return {} - opts = copy.deepcopy(__opts__) - - opts['pillar_roots'][environment] = [pillar_dir] + opts['pillar_roots'] = {} opts['__git_pillar'] = True + pillar = salt.utils.gitfs.GitPillar(opts) + pillar.init_remotes(repo, PER_REMOTE_OVERRIDES, PER_REMOTE_ONLY) + if __opts__.get('__role') == 'minion': + # If masterless, fetch the remotes. We'll need to remove this once + # we make the minion daemon able to run standalone. + pillar.fetch_remotes() + pillar.checkout() + ret = {} + merge_strategy = __opts__.get( + 'pillar_source_merging_strategy', + 'smart' + ) + merge_lists = __opts__.get( + 'pillar_merge_lists', + False + ) + for pillar_dir, env in six.iteritems(pillar.pillar_dirs): + # If pillarenv is set, only grab pillars with that match pillarenv + if opts['pillarenv'] and env != opts['pillarenv']: + log.debug( + 'env \'%s\' for pillar dir \'%s\' does not match ' + 'pillarenv \'%s\', skipping', + env, pillar_dir, opts['pillarenv'] + ) + continue + if pillar_dir in pillar.pillar_linked_dirs: + log.debug( + 'git_pillar is skipping processing on %s as it is a ' + 'mounted repo', pillar_dir + ) + continue + else: + log.debug( + 'git_pillar is processing pillar SLS from %s for pillar ' + 'env \'%s\'', pillar_dir, env + ) - pil = Pillar(opts, __grains__, minion_id, branch) + if env == '__env__': + env = opts.get('pillarenv') \ + or opts.get('environment') \ + or opts.get('git_pillar_base') + log.debug('__env__ maps to %s', env) - return pil.compile_pillar(ext=False) + pillar_roots = [pillar_dir] + if __opts__['git_pillar_includes']: + # Add the rest of the pillar_dirs in this environment to the + # list, excluding the current pillar_dir being processed. This + # is because it was already specified above as the first in the + # list, so that its top file is sourced from the correct + # location and not from another git_pillar remote. + pillar_roots.extend( + [d for (d, e) in six.iteritems(pillar.pillar_dirs) + if env == e and d != pillar_dir] + ) -def _update(branch, repo_location): - ''' - Ensure you are following the latest changes on the remote + opts['pillar_roots'] = {env: pillar_roots} - return boolean whether it worked - ''' - gitpil = _LegacyGitPillar(branch, repo_location, __opts__) - - return gitpil.update() - - -def _envs(branch, repo_location): - ''' - Return a list of refs that can be used as environments - ''' - gitpil = _LegacyGitPillar(branch, repo_location, __opts__) - - return gitpil.envs() + local_pillar = Pillar(opts, __grains__, minion_id, env) + ret = salt.utils.dictupdate.merge( + ret, + local_pillar.compile_pillar(ext=False), + strategy=merge_strategy, + merge_lists=merge_lists + ) + return ret def _extract_key_val(kv, delimiter='='): diff --git a/salt/pillar/hg_pillar.py b/salt/pillar/hg_pillar.py index f43d83ddd1b..32c5bd9a041 100644 --- a/salt/pillar/hg_pillar.py +++ b/salt/pillar/hg_pillar.py @@ -28,9 +28,10 @@ import os # Import Salt Libs import salt.pillar import salt.utils +import salt.utils.stringutils # Import Third Party Libs -import salt.ext.six as six +from salt.ext import six try: import hglib except ImportError: @@ -105,7 +106,7 @@ class Repo(object): if six.PY2: repo_hash = hash_type(repo_uri).hexdigest() else: - repo_hash = hash_type(salt.utils.to_bytes(repo_uri)).hexdigest() + repo_hash = hash_type(salt.utils.stringutils.to_bytes(repo_uri)).hexdigest() self.working_dir = os.path.join(cachedir, repo_hash) if not os.path.isdir(self.working_dir): self.repo = hglib.clone(repo_uri, self.working_dir) diff --git a/salt/pillar/hiera.py b/salt/pillar/hiera.py index 5ad5a52395f..5a1a8d4031d 100644 --- a/salt/pillar/hiera.py +++ b/salt/pillar/hiera.py @@ -3,16 +3,16 @@ Use hiera data as a Pillar source ''' -# Import python libs +# Import Python libs from __future__ import absolute_import import logging +import yaml # Import salt libs -import salt.utils +import salt.utils.path -# Import third party libs -import yaml -import salt.ext.six as six +# Import 3rd-party libs +from salt.ext import six # Set up logging @@ -23,7 +23,7 @@ def __virtual__(): ''' Only return if hiera is installed ''' - return 'hiera' if salt.utils.which('hiera') else False + return 'hiera' if salt.utils.path.which('hiera') else False def ext_pillar(minion_id, # pylint: disable=W0613 diff --git a/salt/pillar/http_json.py b/salt/pillar/http_json.py index d931d4a3d0f..57c276e3851 100644 --- a/salt/pillar/http_json.py +++ b/salt/pillar/http_json.py @@ -27,6 +27,16 @@ in <> brackets) in the url in order to populate pillar data based on the grain v url: http://example.com/api/ with_grains: True +.. versionchanged:: Oxygen + + If %s is present in the url, it will be automaticaly replaced by the minion_id: + + .. code-block:: yaml + + ext_pillar: + - http_json: + url: http://example.com/api/%s + Module Documentation ==================== ''' @@ -64,6 +74,9 @@ def ext_pillar(minion_id, :return: A dictionary of the pillar data to add. :rtype: dict ''' + + url = url.replace('%s', _quote(minion_id)) + grain_pattern = r'<(?P.*?)>' if with_grains: diff --git a/salt/pillar/http_yaml.py b/salt/pillar/http_yaml.py index f62d298ea26..68e005c10e7 100644 --- a/salt/pillar/http_yaml.py +++ b/salt/pillar/http_yaml.py @@ -27,6 +27,16 @@ in <> brackets) in the url in order to populate pillar data based on the grain v url: http://example.com/api/ with_grains: True +.. versionchanged:: Oxygen + + If %s is present in the url, it will be automaticaly replaced by the minion_id: + + .. code-block:: yaml + + ext_pillar: + - http_json: + url: http://example.com/api/%s + Module Documentation ==================== ''' @@ -65,6 +75,9 @@ def ext_pillar(minion_id, :return: A dictionary of the pillar data to add. :rtype: dict ''' + + url = url.replace('%s', _quote(minion_id)) + grain_pattern = r'<(?P.*?)>' if with_grains: diff --git a/salt/pillar/libvirt.py b/salt/pillar/libvirt.py index 182c4aae6a1..4ff98d42e0b 100644 --- a/salt/pillar/libvirt.py +++ b/salt/pillar/libvirt.py @@ -15,11 +15,12 @@ import os import subprocess # Import salt libs -import salt.utils +import salt.utils.files +import salt.utils.path def __virtual__(): - return salt.utils.which('certtool') is not None + return salt.utils.path.which('certtool') is not None def ext_pillar(minion_id, @@ -50,9 +51,9 @@ def ext_pillar(minion_id, if not key.endswith('.pem'): continue fn_ = os.path.join(key_dir, key) - with salt.utils.fopen(fn_, 'r') as fp_: + with salt.utils.files.fopen(fn_, 'r') as fp_: ret['libvirt.{0}'.format(key)] = fp_.read() - with salt.utils.fopen(cacert, 'r') as fp_: + with salt.utils.files.fopen(cacert, 'r') as fp_: ret['libvirt.cacert.pem'] = fp_.read() return ret @@ -76,7 +77,7 @@ def gen_hyper_keys(minion_id, cacert = os.path.join(key_dir, 'cacert.pem') cainfo = os.path.join(key_dir, 'ca.info') if not os.path.isfile(cainfo): - with salt.utils.fopen(cainfo, 'w+') as fp_: + with salt.utils.files.fopen(cainfo, 'w+') as fp_: fp_.write('cn = salted\nca\ncert_signing_key') if not os.path.isfile(cakey): subprocess.call( @@ -96,7 +97,7 @@ def gen_hyper_keys(minion_id, ccert = os.path.join(sub_dir, 'clientcert.pem') clientinfo = os.path.join(sub_dir, 'client.info') if not os.path.isfile(srvinfo): - with salt.utils.fopen(srvinfo, 'w+') as fp_: + with salt.utils.files.fopen(srvinfo, 'w+') as fp_: infodat = ('organization = salted\ncn = {0}\ntls_www_server' '\nencryption_key\nsigning_key' '\ndigitalSignature\nexpiration_days = {1}' @@ -114,7 +115,7 @@ def gen_hyper_keys(minion_id, ).format(priv, cacert, cakey, srvinfo, cert) subprocess.call(cmd, shell=True) if not os.path.isfile(clientinfo): - with salt.utils.fopen(clientinfo, 'w+') as fp_: + with salt.utils.files.fopen(clientinfo, 'w+') as fp_: infodat = ('country = {0}\nstate = {1}\nlocality = ' '{2}\norganization = {3}\ncn = {4}\n' 'tls_www_client\nencryption_key\nsigning_key\n' diff --git a/salt/pillar/makostack.py b/salt/pillar/makostack.py index f233dcfd215..83494e4492b 100644 --- a/salt/pillar/makostack.py +++ b/salt/pillar/makostack.py @@ -381,7 +381,7 @@ from functools import partial import yaml # Import Salt libs -import salt.ext.six as six +from salt.ext import six try: from mako.lookup import TemplateLookup diff --git a/salt/pillar/nacl.py b/salt/pillar/nacl.py new file mode 100644 index 00000000000..912becfc2b0 --- /dev/null +++ b/salt/pillar/nacl.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- + +''' +Decrypt pillar data through the builtin NACL renderer + +In most cases, you'll want to make this the last external pillar used. For +example, to pair with the builtin stack pillar you could do something like +this: + +.. code:: yaml + + nacl.config: + keyfile: /root/.nacl + + ext_pillar: + - stack: /path/to/stack.cfg + - nacl: {} + +Set ``nacl.config`` in your config. + +''' + +from __future__ import absolute_import +import salt + + +def ext_pillar(minion_id, pillar, *args, **kwargs): + render_function = salt.loader.render(__opts__, __salt__).get("nacl") + return render_function(pillar) diff --git a/salt/pillar/nodegroups.py b/salt/pillar/nodegroups.py index 7ebe5e6d406..540213436c5 100644 --- a/salt/pillar/nodegroups.py +++ b/salt/pillar/nodegroups.py @@ -43,7 +43,7 @@ from __future__ import absolute_import from salt.utils.minions import CkMinions # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six __version__ = '0.0.2' @@ -64,9 +64,10 @@ def ext_pillar(minion_id, pillar, pillar_name=None): ckminions = None for nodegroup_name in six.iterkeys(all_nodegroups): ckminions = ckminions or CkMinions(__opts__) - match = ckminions.check_minions( + _res = ckminions.check_minions( all_nodegroups[nodegroup_name], 'compound') + match = _res['minions'] if minion_id in match: nodegroups_minion_is_in.append(nodegroup_name) diff --git a/salt/pillar/pepa.py b/salt/pillar/pepa.py index 320ef9d73ca..01fd90a559e 100644 --- a/salt/pillar/pepa.py +++ b/salt/pillar/pepa.py @@ -280,9 +280,9 @@ import re from os.path import isfile, join # Import Salt libs -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import input # pylint: disable=import-error,redefined-builtin -import salt.utils +import salt.utils.files from salt.utils.yamlloader import SaltYamlSafeLoader # Import 3rd-party libs @@ -428,7 +428,7 @@ def ext_pillar(minion_id, pillar, resource, sequence, subkey=False, subkey_only= fn = join(templdir, re.sub(r'\W', '_', entry.lower()) + '.yaml') if isfile(fn): log.info("Loading template: {0}".format(fn)) - with salt.utils.fopen(fn) as fhr: + with salt.utils.files.fopen(fn) as fhr: template = jinja2.Template(fhr.read()) output['pepa_templates'].append(fn) @@ -528,7 +528,7 @@ def validate(output, resource): pepa_schemas = [] for fn in glob.glob(valdir + '/*.yaml'): log.info("Loading schema: {0}".format(fn)) - with salt.utils.fopen(fn) as fhr: + with salt.utils.files.fopen(fn) as fhr: template = jinja2.Template(fhr.read()) data = output data['grains'] = __grains__.copy() @@ -557,7 +557,7 @@ if __name__ == '__main__': sys.exit(1) # Get configuration - with salt.utils.fopen(args.config) as fh_: + with salt.utils.files.fopen(args.config) as fh_: __opts__.update( yaml.load( fh_.read(), diff --git a/salt/pillar/reclass_adapter.py b/salt/pillar/reclass_adapter.py index 7f3c1c20e2c..dd83b3e0f52 100644 --- a/salt/pillar/reclass_adapter.py +++ b/salt/pillar/reclass_adapter.py @@ -63,7 +63,7 @@ from salt.utils.reclass import ( ) # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six # Define the module's virtual name __virtualname__ = 'reclass' diff --git a/salt/pillar/s3.py b/salt/pillar/s3.py index 807d4fe321d..e65dfd8b36c 100644 --- a/salt/pillar/s3.py +++ b/salt/pillar/s3.py @@ -98,7 +98,7 @@ from copy import deepcopy # Import 3rd-party libs # pylint: disable=import-error,no-name-in-module,redefined-builtin -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import filter from salt.ext.six.moves.urllib.parse import quote as _quote # pylint: enable=import-error,no-name-in-module,redefined-builtin @@ -106,6 +106,7 @@ from salt.ext.six.moves.urllib.parse import quote as _quote # Import salt libs from salt.pillar import Pillar import salt.utils +import salt.utils.files # Set up logging log = logging.getLogger(__name__) @@ -337,7 +338,7 @@ def _refresh_buckets_cache_file(creds, cache_file, multiple_env, environment, pr log.debug('Writing S3 buckets pillar cache file') - with salt.utils.fopen(cache_file, 'w') as fp_: + with salt.utils.files.fopen(cache_file, 'w') as fp_: pickle.dump(metadata, fp_) return metadata @@ -350,7 +351,7 @@ def _read_buckets_cache_file(cache_file): log.debug('Reading buckets cache file') - with salt.utils.fopen(cache_file, 'rb') as fp_: + with salt.utils.files.fopen(cache_file, 'rb') as fp_: data = pickle.load(fp_) return data diff --git a/salt/pillar/sql_base.py b/salt/pillar/sql_base.py index e7abceb134e..ca6c9d54523 100644 --- a/salt/pillar/sql_base.py +++ b/salt/pillar/sql_base.py @@ -251,7 +251,7 @@ class SqlBaseExtPillar(six.with_metaclass(abc.ABCMeta, object)): # Filter out values that don't have queries. qbuffer = [x for x in qbuffer if ( - (isinstance(x[1], str) and len(x[1])) + (isinstance(x[1], six.string_types) and len(x[1])) or (isinstance(x[1], (list, tuple)) and (len(x[1]) > 0) and x[1][0]) or @@ -266,7 +266,7 @@ class SqlBaseExtPillar(six.with_metaclass(abc.ABCMeta, object)): 'with_lists': None, 'ignore_null': False } - if isinstance(qb[1], str): + if isinstance(qb[1], six.string_types): defaults['query'] = qb[1] elif isinstance(qb[1], (list, tuple)): defaults['query'] = qb[1][0] diff --git a/salt/pillar/stack.py b/salt/pillar/stack.py index 42fae41b7ca..6165f90d362 100644 --- a/salt/pillar/stack.py +++ b/salt/pillar/stack.py @@ -386,7 +386,7 @@ import yaml from jinja2 import FileSystemLoader, Environment # Import Salt libs -import salt.ext.six as six +from salt.ext import six import salt.utils diff --git a/salt/pillar/vault.py b/salt/pillar/vault.py index c57ed337078..96e25df5866 100644 --- a/salt/pillar/vault.py +++ b/salt/pillar/vault.py @@ -48,10 +48,12 @@ Multiple Vault sources may also be used: - vault: path=secret/minions/{minion}/pass ''' -# import python libs +# Import Python libs from __future__ import absolute_import import logging -import salt.utils + +# Import Salt libs +import salt.utils.versions log = logging.getLogger(__name__) @@ -76,7 +78,7 @@ def ext_pillar(minion_id, # pylint: disable=W0613 comps = conf.split() if not comps[0].startswith('path='): - salt.utils.warn_until( + salt.utils.versions.warn_until( 'Fluorine', 'The \'profile\' argument has been deprecated. Any parts up until ' 'and following the first "path=" are discarded' diff --git a/salt/pillar/vmware_pillar.py b/salt/pillar/vmware_pillar.py index 314f7dc55c9..04dd5a64b86 100644 --- a/salt/pillar/vmware_pillar.py +++ b/salt/pillar/vmware_pillar.py @@ -147,11 +147,11 @@ from __future__ import absolute_import import logging # Import salt libs -from salt.utils import dictupdate +import salt.utils.dictupdate as dictupdate import salt.utils.vmware # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six try: from pyVmomi import vim from pyVim.connect import Disconnect diff --git a/salt/proxy/cimc.py b/salt/proxy/cimc.py new file mode 100644 index 00000000000..4692a8ef319 --- /dev/null +++ b/salt/proxy/cimc.py @@ -0,0 +1,290 @@ +# -*- coding: utf-8 -*- +''' + +Proxy Minion interface module for managing Cisco Integrated Management Controller devices. + +:codeauthor: :email:`Spencer Ervin ` +:maturity: new +:depends: none +:platform: unix + +This proxy minion enables Cisco Integrated Management Controller devices (hereafter referred to +as simply 'cimc' devices to be treated individually like a Salt Minion. + +The cimc proxy leverages the XML API functionality on the Cisco Integrated Management Controller. +The Salt proxy must have access to the cimc on HTTPS (tcp/443). + +More in-depth conceptual reading on Proxy Minions can be found in the +:ref:`Proxy Minion ` section of Salt's +documentation. + + +Configuration +============= +To use this integration proxy module, please configure the following: + +Pillar +------ + +Proxy minions get their configuration from Salt's Pillar. Every proxy must +have a stanza in Pillar and a reference in the Pillar top-file that matches +the ID. + +.. code-block:: yaml + + proxy: + proxytype: cimc + host: + username: + password: + +proxytype +^^^^^^^^^ +The ``proxytype`` key and value pair is critical, as it tells Salt which +interface to load from the ``proxy`` directory in Salt's install hierarchy, +or from ``/srv/salt/_proxy`` on the Salt Master (if you have created your +own proxy module, for example). To use this cimc Proxy Module, set this to +``cimc``. + +host +^^^^ +The location, or ip/dns, of the cimc host. Required. + +username +^^^^^^^^ +The username used to login to the cimc host. Required. + +password +^^^^^^^^ +The password used to login to the cimc host. Required. + +''' + +from __future__ import absolute_import + +# Import Python Libs +import logging +import re + +# Import Salt Libs +import salt.exceptions +from salt._compat import ElementTree as ET + +# This must be present or the Salt loader won't load this module. +__proxyenabled__ = ['cimc'] + +# Variables are scoped to this module so we can have persistent data. +GRAINS_CACHE = {'vendor': 'Cisco'} +DETAILS = {} + +# Set up logging +log = logging.getLogger(__file__) + +# Define the module's virtual name +__virtualname__ = 'cimc' + + +def __virtual__(): + ''' + Only return if all the modules are available. + ''' + return __virtualname__ + + +def init(opts): + ''' + This function gets called when the proxy starts up. + ''' + if 'host' not in opts['proxy']: + log.critical('No \'host\' key found in pillar for this proxy.') + return False + if 'username' not in opts['proxy']: + log.critical('No \'username\' key found in pillar for this proxy.') + return False + if 'password' not in opts['proxy']: + log.critical('No \'passwords\' key found in pillar for this proxy.') + return False + + DETAILS['url'] = 'https://{0}/nuova'.format(opts['proxy']['host']) + DETAILS['headers'] = {'Content-Type': 'application/x-www-form-urlencoded', + 'Content-Length': 62, + 'USER-Agent': 'lwp-request/2.06'} + + # Set configuration details + DETAILS['host'] = opts['proxy']['host'] + DETAILS['username'] = opts['proxy'].get('username') + DETAILS['password'] = opts['proxy'].get('password') + + # Ensure connectivity to the device + log.debug("Attempting to connect to cimc proxy host.") + get_config_resolver_class("computeRackUnit") + log.debug("Successfully connected to cimc proxy host.") + + DETAILS['initialized'] = True + + +def set_config_modify(dn=None, inconfig=None, hierarchical=False): + ''' + The configConfMo method configures the specified managed object in a single subtree (for example, DN). + ''' + ret = {} + cookie = logon() + + # Declare if the search contains hierarchical results. + h = "false" + if hierarchical is True: + h = "true" + + payload = '' \ + '{3}'.format(cookie, h, dn, inconfig) + r = __utils__['http.query'](DETAILS['url'], + data=payload, + method='POST', + decode_type='plain', + decode=True, + verify_ssl=False, + raise_error=True, + headers=DETAILS['headers']) + answer = re.findall(r'(<[\s\S.]*>)', r['text'])[0] + items = ET.fromstring(answer) + logout(cookie) + for item in items: + ret[item.tag] = prepare_return(item) + return ret + + +def get_config_resolver_class(cid=None, hierarchical=False): + ''' + The configResolveClass method returns requested managed object in a given class. + ''' + ret = {} + cookie = logon() + + # Declare if the search contains hierarchical results. + h = "false" + if hierarchical is True: + h = "true" + + payload = ''.format(cookie, h, cid) + r = __utils__['http.query'](DETAILS['url'], + data=payload, + method='POST', + decode_type='plain', + decode=True, + verify_ssl=False, + raise_error=True, + headers=DETAILS['headers']) + + answer = re.findall(r'(<[\s\S.]*>)', r['text'])[0] + items = ET.fromstring(answer) + logout(cookie) + for item in items: + ret[item.tag] = prepare_return(item) + return ret + + +def logon(): + ''' + Logs into the cimc device and returns the session cookie. + ''' + content = {} + payload = "".format(DETAILS['username'], DETAILS['password']) + r = __utils__['http.query'](DETAILS['url'], + data=payload, + method='POST', + decode_type='plain', + decode=True, + verify_ssl=False, + raise_error=False, + headers=DETAILS['headers']) + answer = re.findall(r'(<[\s\S.]*>)', r['text'])[0] + items = ET.fromstring(answer) + for item in items.attrib: + content[item] = items.attrib[item] + + if 'outCookie' not in content: + raise salt.exceptions.CommandExecutionError("Unable to log into proxy device.") + + return content['outCookie'] + + +def logout(cookie=None): + ''' + Closes the session with the device. + ''' + payload = ''.format(cookie) + __utils__['http.query'](DETAILS['url'], + data=payload, + method='POST', + decode_type='plain', + decode=True, + verify_ssl=False, + raise_error=True, + headers=DETAILS['headers']) + return + + +def prepare_return(x): + ''' + Converts the etree to dict + ''' + ret = {} + for a in list(x): + if a.tag not in ret: + ret[a.tag] = [] + ret[a.tag].append(prepare_return(a)) + for a in x.attrib: + ret[a] = x.attrib[a] + return ret + + +def initialized(): + ''' + Since grains are loaded in many different places and some of those + places occur before the proxy can be initialized, return whether + our init() function has been called + ''' + return DETAILS.get('initialized', False) + + +def grains(): + ''' + Get the grains from the proxied device + ''' + if not DETAILS.get('grains_cache', {}): + DETAILS['grains_cache'] = GRAINS_CACHE + try: + compute_rack = get_config_resolver_class('computeRackUnit', False) + DETAILS['grains_cache'] = compute_rack['outConfigs']['computeRackUnit'] + except Exception as err: + log.error(err) + return DETAILS['grains_cache'] + + +def grains_refresh(): + ''' + Refresh the grains from the proxied device + ''' + DETAILS['grains_cache'] = None + return grains() + + +def ping(): + ''' + Returns true if the device is reachable, else false. + ''' + try: + cookie = logon() + logout(cookie) + except Exception as err: + log.debug(err) + return False + return True + + +def shutdown(): + ''' + Shutdown the connection to the proxy device. For this proxy, + shutdown is a no-op. + ''' + log.debug('CIMC proxy shutdown() called.') diff --git a/salt/proxy/esxcluster.py b/salt/proxy/esxcluster.py new file mode 100644 index 00000000000..af3740d8d59 --- /dev/null +++ b/salt/proxy/esxcluster.py @@ -0,0 +1,310 @@ +# -*- coding: utf-8 -*- +''' +Proxy Minion interface module for managing VMWare ESXi clusters. + +Dependencies +============ + +- pyVmomi +- jsonschema + +Configuration +============= +To use this integration proxy module, please configure the following: + +Pillar +------ + +Proxy minions get their configuration from Salt's Pillar. This can now happen +from the proxy's configuration file. + +Example pillars: + +``userpass`` mechanism: + +.. code-block:: yaml + + proxy: + proxytype: esxcluster + cluster: + datacenter: + vcenter: + mechanism: userpass + username: + passwords: (required if userpass is used) + - first_password + - second_password + - third_password + +``sspi`` mechanism: + +.. code-block:: yaml + + proxy: + proxytype: esxcluster + cluster: + datacenter: + vcenter: + mechanism: sspi + domain: + principal: + +proxytype +^^^^^^^^^ +To use this Proxy Module, set this to ``esxdatacenter``. + +cluster +^^^^^^^ +Name of the managed cluster. Required. + +datacenter +^^^^^^^^^^ +Name of the datacenter the managed cluster is in. Required. + +vcenter +^^^^^^^ +The location of the VMware vCenter server (host of ip) where the datacenter +should be managed. Required. + +mechanism +^^^^^^^^ +The mechanism used to connect to the vCenter server. Supported values are +``userpass`` and ``sspi``. Required. + +Note: + Connections are attempted using all (``username``, ``password``) + combinations on proxy startup. + +username +^^^^^^^^ +The username used to login to the host, such as ``root``. Required if mechanism +is ``userpass``. + +passwords +^^^^^^^^^ +A list of passwords to be used to try and login to the vCenter server. At least +one password in this list is required if mechanism is ``userpass``. When the +proxy comes up, it will try the passwords listed in order. + +domain +^^^^^^ +User domain. Required if mechanism is ``sspi``. + +principal +^^^^^^^^ +Kerberos principal. Rquired if mechanism is ``sspi``. + +protocol +^^^^^^^^ +If the ESXi host is not using the default protocol, set this value to an +alternate protocol. Default is ``https``. + +port +^^^^ +If the ESXi host is not using the default port, set this value to an +alternate port. Default is ``443``. + +Salt Proxy +---------- + +After your pillar is in place, you can test the proxy. The proxy can run on +any machine that has network connectivity to your Salt Master and to the +vCenter server in the pillar. SaltStack recommends that the machine running the +salt-proxy process also run a regular minion, though it is not strictly +necessary. + +To start a proxy minion one needs to establish its identity : + +.. code-block:: bash + + salt-proxy --proxyid + +On the machine that will run the proxy, make sure there is a configuration file +present. By default this is ``/etc/salt/proxy``. If in a different location, the +```` has to be specified when running the proxy: +file with at least the following in it: + +.. code-block:: bash + + salt-proxy --proxyid -c + +Commands +-------- + +Once the proxy is running it will connect back to the specified master and +individual commands can be runs against it: + +.. code-block:: bash + + # Master - minion communication + salt test.ping + + # Test vcenter connection + salt vsphere.test_vcenter_connection + +States +------ + +Associated states are documented in +:mod:`salt.states.esxcluster `. +Look there to find an example structure for Pillar as well as an example +``.sls`` file for configuring an ESX cluster from scratch. +''' + + +# Import Python Libs +from __future__ import absolute_import +import logging +import os + +# Import Salt Libs +import salt.exceptions +from salt.config.schemas.esxcluster import EsxclusterProxySchema +from salt.utils.dictupdate import merge + +# This must be present or the Salt loader won't load this module. +__proxyenabled__ = ['esxcluster'] + +# External libraries +try: + import jsonschema + HAS_JSONSCHEMA = True +except ImportError: + HAS_JSONSCHEMA = False + +# Variables are scoped to this module so we can have persistent data +# across calls to fns in here. +GRAINS_CACHE = {} +DETAILS = {} + + +# Set up logging +log = logging.getLogger(__name__) +# Define the module's virtual name +__virtualname__ = 'esxcluster' + + +def __virtual__(): + ''' + Only load if the vsphere execution module is available. + ''' + if HAS_JSONSCHEMA: + return __virtualname__ + + return False, 'The esxcluster proxy module did not load.' + + +def init(opts): + ''' + This function gets called when the proxy starts up. For + login + the protocol and port are cached. + ''' + log.debug('Initting esxcluster proxy module in process ' + '{}'.format(os.getpid())) + log.debug('Validating esxcluster proxy input') + schema = EsxclusterProxySchema.serialize() + log.trace('schema = {}'.format(schema)) + proxy_conf = merge(opts.get('proxy', {}), __pillar__.get('proxy', {})) + log.trace('proxy_conf = {0}'.format(proxy_conf)) + try: + jsonschema.validate(proxy_conf, schema) + except jsonschema.exceptions.ValidationError as exc: + raise salt.exceptions.InvalidConfigError(exc) + + # Save mandatory fields in cache + for key in ('vcenter', 'datacenter', 'cluster', 'mechanism'): + DETAILS[key] = proxy_conf[key] + + # Additional validation + if DETAILS['mechanism'] == 'userpass': + if 'username' not in proxy_conf: + raise salt.exceptions.InvalidConfigError( + 'Mechanism is set to \'userpass\', but no ' + '\'username\' key found in proxy config.') + if 'passwords' not in proxy_conf: + raise salt.exceptions.InvalidConfigError( + 'Mechanism is set to \'userpass\', but no ' + '\'passwords\' key found in proxy config.') + for key in ('username', 'passwords'): + DETAILS[key] = proxy_conf[key] + else: + if 'domain' not in proxy_conf: + raise salt.exceptions.InvalidConfigError( + 'Mechanism is set to \'sspi\', but no ' + '\'domain\' key found in proxy config.') + if 'principal' not in proxy_conf: + raise salt.exceptions.InvalidConfigError( + 'Mechanism is set to \'sspi\', but no ' + '\'principal\' key found in proxy config.') + for key in ('domain', 'principal'): + DETAILS[key] = proxy_conf[key] + + # Save optional + DETAILS['protocol'] = proxy_conf.get('protocol') + DETAILS['port'] = proxy_conf.get('port') + + # Test connection + if DETAILS['mechanism'] == 'userpass': + # Get the correct login details + log.debug('Retrieving credentials and testing vCenter connection for ' + 'mehchanism \'userpass\'') + try: + username, password = find_credentials() + DETAILS['password'] = password + except salt.exceptions.SaltSystemExit as err: + log.critical('Error: {0}'.format(err)) + return False + return True + + +def ping(): + ''' + Returns True. + + CLI Example: + + .. code-block:: bash + + salt esx-cluster test.ping + ''' + return True + + +def shutdown(): + ''' + Shutdown the connection to the proxy device. For this proxy, + shutdown is a no-op. + ''' + log.debug('esxcluster proxy shutdown() called...') + + +def find_credentials(): + ''' + Cycle through all the possible credentials and return the first one that + works. + ''' + + # if the username and password were already found don't fo though the + # connection process again + if 'username' in DETAILS and 'password' in DETAILS: + return DETAILS['username'], DETAILS['password'] + + passwords = DETAILS['passwords'] + for password in passwords: + DETAILS['password'] = password + if not __salt__['vsphere.test_vcenter_connection'](): + # We are unable to authenticate + continue + # If we have data returned from above, we've successfully authenticated. + return DETAILS['username'], password + # We've reached the end of the list without successfully authenticating. + raise salt.exceptions.VMwareConnectionError('Cannot complete login due to ' + 'incorrect credentials.') + + +def get_details(): + ''' + Function that returns the cached details + ''' + return DETAILS diff --git a/salt/proxy/esxdatacenter.py b/salt/proxy/esxdatacenter.py new file mode 100644 index 00000000000..186b880c2cc --- /dev/null +++ b/salt/proxy/esxdatacenter.py @@ -0,0 +1,298 @@ +# -*- coding: utf-8 -*- +''' +Proxy Minion interface module for managing VMWare ESXi clusters. + +Dependencies +============ + +- pyVmomi +- jsonschema + +Configuration +============= +To use this integration proxy module, please configure the following: + +Pillar +------ + +Proxy minions get their configuration from Salt's Pillar. This can now happen +from the proxy's configuration file. + +Example pillars: + +``userpass`` mechanism: + +.. code-block:: yaml + + proxy: + proxytype: esxdatacenter + datacenter: + vcenter: + mechanism: userpass + username: + passwords: (required if userpass is used) + - first_password + - second_password + - third_password + +``sspi`` mechanism: + +.. code-block:: yaml + + proxy: + proxytype: esxdatacenter + datacenter: + vcenter: + mechanism: sspi + domain: + principal: + +proxytype +^^^^^^^^^ +To use this Proxy Module, set this to ``esxdatacenter``. + +datacenter +^^^^^^^^^^ +Name of the managed datacenter. Required. + +vcenter +^^^^^^^ +The location of the VMware vCenter server (host of ip) where the datacenter +should be managed. Required. + +mechanism +^^^^^^^^ +The mechanism used to connect to the vCenter server. Supported values are +``userpass`` and ``sspi``. Required. + +Note: + Connections are attempted using all (``username``, ``password``) + combinations on proxy startup. + +username +^^^^^^^^ +The username used to login to the host, such as ``root``. Required if mechanism +is ``userpass``. + +passwords +^^^^^^^^^ +A list of passwords to be used to try and login to the vCenter server. At least +one password in this list is required if mechanism is ``userpass``. When the +proxy comes up, it will try the passwords listed in order. + +domain +^^^^^^ +User domain. Required if mechanism is ``sspi``. + +principal +^^^^^^^^ +Kerberos principal. Rquired if mechanism is ``sspi``. + +protocol +^^^^^^^^ +If the ESXi host is not using the default protocol, set this value to an +alternate protocol. Default is ``https``. + +port +^^^^ +If the ESXi host is not using the default port, set this value to an +alternate port. Default is ``443``. + +Salt Proxy +---------- + +After your pillar is in place, you can test the proxy. The proxy can run on +any machine that has network connectivity to your Salt Master and to the +vCenter server in the pillar. SaltStack recommends that the machine running the +salt-proxy process also run a regular minion, though it is not strictly +necessary. + +To start a proxy minion one needs to establish its identity : + +.. code-block:: bash + + salt-proxy --proxyid + +On the machine that will run the proxy, make sure there is a configuration file +present. By default this is ``/etc/salt/proxy``. If in a different location, the +```` has to be specified when running the proxy: +file with at least the following in it: + +.. code-block:: bash + + salt-proxy --proxyid -c + +Commands +-------- + +Once the proxy is running it will connect back to the specified master and +individual commands can be runs against it: + +.. code-block:: bash + + # Master - minion communication + salt test.ping + + # Test vcenter connection + salt vsphere.test_vcenter_connection + +States +------ + +Associated states are documented in +:mod:`salt.states.esxdatacenter `. +Look there to find an example structure for Pillar as well as an example +``.sls`` file for configuring an ESX datacenter from scratch. +''' + +# Import Python Libs +from __future__ import absolute_import +import logging +import os + +# Import Salt Libs +import salt.exceptions +from salt.config.schemas.esxdatacenter import EsxdatacenterProxySchema + +# This must be present or the Salt loader won't load this module. +__proxyenabled__ = ['esxdatacenter'] + +# External libraries +try: + import jsonschema + HAS_JSONSCHEMA = True +except ImportError: + HAS_JSONSCHEMA = False + +# Variables are scoped to this module so we can have persistent data +# across calls to fns in here. +DETAILS = {} + + +# Set up logging +log = logging.getLogger(__name__) +# Define the module's virtual name +__virtualname__ = 'esxdatacenter' + + +def __virtual__(): + ''' + Only load if the vsphere execution module is available. + ''' + if HAS_JSONSCHEMA: + return __virtualname__ + + return False, 'The esxdatacenter proxy module did not load.' + + +def init(opts): + ''' + This function gets called when the proxy starts up. + All login details are cached. + ''' + log.debug('Initting esxdatacenter proxy module in process ' + '{}'.format(os.getpid())) + log.trace('Validating esxdatacenter proxy input') + schema = EsxdatacenterProxySchema.serialize() + log.trace('schema = {}'.format(schema)) + try: + jsonschema.validate(opts['proxy'], schema) + except jsonschema.exceptions.ValidationError as exc: + raise salt.exceptions.InvalidConfigError(exc) + + # Save mandatory fields in cache + for key in ('vcenter', 'datacenter', 'mechanism'): + DETAILS[key] = opts['proxy'][key] + + # Additional validation + if DETAILS['mechanism'] == 'userpass': + if 'username' not in opts['proxy']: + raise salt.exceptions.InvalidConfigError( + 'Mechanism is set to \'userpass\', but no ' + '\'username\' key found in proxy config.') + if 'passwords' not in opts['proxy']: + raise salt.exceptions.InvalidConfigError( + 'Mechanism is set to \'userpass\', but no ' + '\'passwords\' key found in proxy config.') + for key in ('username', 'passwords'): + DETAILS[key] = opts['proxy'][key] + else: + if 'domain' not in opts['proxy']: + raise salt.exceptions.InvalidConfigError( + 'Mechanism is set to \'sspi\', but no ' + '\'domain\' key found in proxy config.') + if 'principal' not in opts['proxy']: + raise salt.exceptions.InvalidConfigError( + 'Mechanism is set to \'sspi\', but no ' + '\'principal\' key found in proxy config.') + for key in ('domain', 'principal'): + DETAILS[key] = opts['proxy'][key] + + # Save optional + DETAILS['protocol'] = opts['proxy'].get('protocol') + DETAILS['port'] = opts['proxy'].get('port') + + # Test connection + if DETAILS['mechanism'] == 'userpass': + # Get the correct login details + log.debug('Retrieving credentials and testing vCenter connection for ' + 'mehchanism \'userpass\'') + try: + username, password = find_credentials() + DETAILS['password'] = password + except salt.exceptions.SaltSystemExit as err: + log.critical('Error: {0}'.format(err)) + return False + return True + + +def ping(): + ''' + Returns True. + + CLI Example: + + .. code-block:: bash + + salt dc_id test.ping + ''' + return True + + +def shutdown(): + ''' + Shutdown the connection to the proxy device. For this proxy, + shutdown is a no-op. + ''' + log.debug('esxdatacenter proxy shutdown() called...') + + +def find_credentials(): + ''' + Cycle through all the possible credentials and return the first one that + works. + ''' + + # if the username and password were already found don't fo though the + # connection process again + if 'username' in DETAILS and 'password' in DETAILS: + return DETAILS['username'], DETAILS['password'] + + passwords = DETAILS['passwords'] + for password in passwords: + DETAILS['password'] = password + if not __salt__['vsphere.test_vcenter_connection'](): + # We are unable to authenticate + continue + # If we have data returned from above, we've successfully authenticated. + return DETAILS['username'], password + # We've reached the end of the list without successfully authenticating. + raise salt.exceptions.VMwareConnectionError('Cannot complete login due to ' + 'incorrect credentials.') + + +def get_details(): + ''' + Function that returns the cached details + ''' + return DETAILS diff --git a/salt/proxy/fx2.py b/salt/proxy/fx2.py index e90c345a45f..df03bb1ce2b 100644 --- a/salt/proxy/fx2.py +++ b/salt/proxy/fx2.py @@ -175,8 +175,8 @@ from __future__ import absolute_import # Import python libs import logging -import salt.utils import salt.utils.http +import salt.utils.path # This must be present or the Salt loader won't load this module __proxyenabled__ = ['fx2'] @@ -195,7 +195,7 @@ def __virtual__(): ''' Only return if all the modules are available ''' - if not salt.utils.which('racadm'): + if not salt.utils.path.which('racadm'): log.critical('fx2 proxy minion needs "racadm" to be installed.') return False diff --git a/salt/proxy/junos.py b/salt/proxy/junos.py index c8278fdc715..e3227bb4aec 100644 --- a/salt/proxy/junos.py +++ b/salt/proxy/junos.py @@ -37,7 +37,6 @@ Run the salt proxy via the following command: ''' from __future__ import absolute_import -# Import python libs import logging # Import 3rd-party libs @@ -47,6 +46,11 @@ try: import jnpr.junos.utils import jnpr.junos.utils.config import jnpr.junos.utils.sw + from jnpr.junos.exception import RpcTimeoutError + from jnpr.junos.exception import ConnectClosedError + from jnpr.junos.exception import RpcError + from jnpr.junos.exception import ConnectError + from ncclient.operations.errors import TimeoutExpiredError except ImportError: HAS_JUNOS = False @@ -118,11 +122,31 @@ def conn(): def alive(opts): ''' - Return the connection status with the remote device. + Validate and return the connection status with the remote device. .. versionadded:: Oxygen ''' - return thisproxy['conn'].connected + + dev = conn() + + # Check that the underlying netconf connection still exists. + if dev._conn is None: + return False + + # call rpc only if ncclient queue is empty. If not empty that means other + # rpc call is going on. + if hasattr(dev._conn, '_session'): + if dev._conn._session._transport.is_active(): + # there is no on going rpc call. + if dev._conn._session._q.empty(): + thisproxy['conn'].connected = ping() + else: + # ssh connection is lost + dev.connected = False + else: + # other connection modes, like telnet + thisproxy['conn'].connected = ping() + return dev.connected def proxytype(): @@ -150,7 +174,16 @@ def ping(): ''' Ping? Pong! ''' - return thisproxy['conn'].connected + + dev = conn() + try: + dev.rpc.file_list(path='/dev/null', dev_timeout=2) + return True + except (RpcTimeoutError, ConnectClosedError): + try: + dev.close() + except (RpcError, ConnectError, TimeoutExpiredError): + return False def shutdown(opts): diff --git a/salt/proxy/napalm.py b/salt/proxy/napalm.py index b0cd8d6546a..1acee331486 100644 --- a/salt/proxy/napalm.py +++ b/salt/proxy/napalm.py @@ -23,37 +23,88 @@ Please check Installation_ for complete details. Pillar ------ -The napalm proxy configuration requires four mandatory parameters in order to connect to the network device: +The napalm proxy configuration requires the following parameters in order to connect to the network device: -* driver: specifies the network device operating system. For a complete list of the supported operating systems \ -please refer to the `NAPALM Read the Docs page`_. -* host: hostname -* username: username to be used when connecting to the device -* passwd: the password needed to establish the connection -* optional_args: dictionary with the optional arguments. Check the complete list of supported `optional arguments`_ +driver + Specifies the network device operating system. + For a complete list of the supported operating systems please refer to the + `NAPALM Read the Docs page`_. -.. _`NAPALM Read the Docs page`: https://napalm.readthedocs.io/en/latest/#supported-network-operating-systems -.. _`optional arguments`: http://napalm.readthedocs.io/en/latest/support/index.html#list-of-supported-optional-arguments +host + The IP Address or FQDN to use when connecting to the device. Alternatively, + the following field names can be used instead: ``hostname``, ``fqdn``, ``ip``. -.. versionadded:: 2017.7.0 +username + The username to be used when connecting to the device. -* always_alive: in certain less dynamic environments, maintaining the remote connection permanently +passwd + The password needed to establish the connection. + + .. note:: + + This field may not be mandatory when working with SSH-based drivers, and + the username has a SSH key properly configured on the device targeted to + be managed. + +optional_args + Dictionary with the optional arguments. + Check the complete list of supported `optional arguments`_. + +always_alive: ``True`` + In certain less dynamic environments, maintaining the remote connection permanently open with the network device is not always beneficial. In that case, the user can select to initialize the connection only when needed, by specifying this field to ``false``. Default: ``true`` (maintains the connection with the remote network device). -Example: + .. versionadded:: 2017.7.0 + +provider: ``napalm_base`` + The module that provides the ``get_network_device`` function. + This option is useful when the user has more specific needs and requires + to extend the NAPALM capabilities using a private library implementation. + The only constraint is that the alternative library needs to have the + ``get_network_device`` function available. + + .. versionadded:: 2017.7.1 + +multiprocessing: ``False`` + Overrides the :conf_minion:`multiprocessing` option, per proxy minion. + The ``multiprocessing`` option must be turned off for SSH-based proxies. + However, some NAPALM drivers (e.g. Arista, NX-OS) are not SSH-based. + As multiple proxy minions may share the same configuration file, + this option permits the configuration of the ``multiprocessing`` option + more specifically, for some proxy minions. + + .. versionadded:: 2017.7.2 + + +.. _`NAPALM Read the Docs page`: https://napalm.readthedocs.io/en/latest/#supported-network-operating-systems +.. _`optional arguments`: http://napalm.readthedocs.io/en/latest/support/index.html#list-of-supported-optional-arguments + +Proxy pillar file example: .. code-block:: yaml proxy: - proxytype: napalm - driver: junos - host: core05.nrt02 - username: my_username - passwd: my_password - optional_args: - port: 12201 + proxytype: napalm + driver: junos + host: core05.nrt02 + username: my_username + passwd: my_password + optional_args: + port: 12201 + +Example using a user-specific library, extending NAPALM's capabilities, e.g. ``custom_napalm_base``: + +.. code-block:: yaml + + proxy: + proxytype: napalm + driver: ios + fqdn: cr1.th2.par.as1234.net + username: salt + password: '' + provider: custom_napalm_base .. seealso:: diff --git a/salt/proxy/panos.py b/salt/proxy/panos.py new file mode 100644 index 00000000000..f7fb8f574ca --- /dev/null +++ b/salt/proxy/panos.py @@ -0,0 +1,418 @@ +# -*- coding: utf-8 -*- +''' + +Proxy Minion interface module for managing Palo Alto firewall devices. + +:codeauthor: :email:`Spencer Ervin ` +:maturity: new +:depends: none +:platform: unix + +This proxy minion enables Palo Alto firewalls (hereafter referred to +as simply 'panos') to be treated individually like a Salt Minion. + +The panos proxy leverages the XML API functionality on the Palo Alto +firewall. The Salt proxy must have access to the Palo Alto firewall on +HTTPS (tcp/443). + +More in-depth conceptual reading on Proxy Minions can be found in the +:ref:`Proxy Minion ` section of Salt's +documentation. + + +Configuration +============= +To use this integration proxy module, please configure the following: + +Pillar +------ + +Proxy minions get their configuration from Salt's Pillar. Every proxy must +have a stanza in Pillar and a reference in the Pillar top-file that matches +the ID. There are four connection options available for the panos proxy module. + +- Direct Device (Password) +- Direct Device (API Key) +- Panorama Pass-Through (Password) +- Panorama Pass-Through (API Key) + + +Direct Device (Password) +------------------------ + +The direct device configuration configures the proxy to connect directly to +the device with username and password. + +.. code-block:: yaml + + proxy: + proxytype: panos + host: + username: + password: + +proxytype +^^^^^^^^^ +The ``proxytype`` key and value pair is critical, as it tells Salt which +interface to load from the ``proxy`` directory in Salt's install hierarchy, +or from ``/srv/salt/_proxy`` on the Salt Master (if you have created your +own proxy module, for example). To use this panos Proxy Module, set this to +``panos``. + +host +^^^^ +The location, or ip/dns, of the panos host. Required. + +username +^^^^^^^^ +The username used to login to the panos host. Required. + +password +^^^^^^^^ +The password used to login to the panos host. Required. + +Direct Device (API Key) +------------------------ + +Palo Alto devices allow for access to the XML API with a generated 'API key'_ +instead of username and password. + +.. _API key: https://www.paloaltonetworks.com/documentation/71/pan-os/xml-api/get-started-with-the-pan-os-xml-api/get-your-api-key + +.. code-block:: yaml + + proxy: + proxytype: panos + host: + apikey: + +proxytype +^^^^^^^^^ +The ``proxytype`` key and value pair is critical, as it tells Salt which +interface to load from the ``proxy`` directory in Salt's install hierarchy, +or from ``/srv/salt/_proxy`` on the Salt Master (if you have created your +own proxy module, for example). To use this panos Proxy Module, set this to +``panos``. + +host +^^^^ +The location, or ip/dns, of the panos host. Required. + +apikey +^^^^^^^^ +The generated XML API key for the panos host. Required. + +Panorama Pass-Through (Password) +------------------------ + +The Panorama pass-through method sends all connections through the Panorama +management system. It passes the connections to the appropriate device using +the serial number of the Palo Alto firewall. + +This option will reduce the number of connections that must be present for the +proxy server. It will only require a connection to the Panorama server. + +The username and password will be for authentication to the Panorama server, +not the panos device. + +.. code-block:: yaml + + proxy: + proxytype: panos + serial: + host: + username: + password: + +proxytype +^^^^^^^^^ +The ``proxytype`` key and value pair is critical, as it tells Salt which +interface to load from the ``proxy`` directory in Salt's install hierarchy, +or from ``/srv/salt/_proxy`` on the Salt Master (if you have created your +own proxy module, for example). To use this panos Proxy Module, set this to +``panos``. + +serial +^^^^^^ +The serial number of the panos host. Required. + +host +^^^^ +The location, or ip/dns, of the Panorama server. Required. + +username +^^^^^^^^ +The username used to login to the Panorama server. Required. + +password +^^^^^^^^ +The password used to login to the Panorama server. Required. + +Panorama Pass-Through (API Key) +------------------------ + +The Panorama server can also utilize a generated 'API key'_ for authentication. + +.. _API key: https://www.paloaltonetworks.com/documentation/71/pan-os/xml-api/get-started-with-the-pan-os-xml-api/get-your-api-key + +.. code-block:: yaml + + proxy: + proxytype: panos + serial: + host: + apikey: + +proxytype +^^^^^^^^^ +The ``proxytype`` key and value pair is critical, as it tells Salt which +interface to load from the ``proxy`` directory in Salt's install hierarchy, +or from ``/srv/salt/_proxy`` on the Salt Master (if you have created your +own proxy module, for example). To use this panos Proxy Module, set this to +``panos``. + +serial +^^^^^^ +The serial number of the panos host. Required. + +host +^^^^ +The location, or ip/dns, of the Panorama server. Required. + +apikey +^^^^^^^^ +The generated XML API key for the Panorama server. Required. + +''' + +from __future__ import absolute_import + +# Import Python Libs +import logging + +# Import Salt Libs +import salt.exceptions + +# This must be present or the Salt loader won't load this module. +__proxyenabled__ = ['panos'] + +# Variables are scoped to this module so we can have persistent data. +GRAINS_CACHE = {'vendor': 'Palo Alto'} +DETAILS = {} + +# Set up logging +log = logging.getLogger(__file__) + +# Define the module's virtual name +__virtualname__ = 'panos' + + +def __virtual__(): + ''' + Only return if all the modules are available. + ''' + return __virtualname__ + + +def init(opts): + ''' + This function gets called when the proxy starts up. For + panos devices, a determination is made on the connection type + and the appropriate connection details that must be cached. + ''' + if 'host' not in opts['proxy']: + log.critical('No \'host\' key found in pillar for this proxy.') + return False + if 'apikey' not in opts['proxy']: + # If we do not have an apikey, we must have both a username and password + if 'username' not in opts['proxy']: + log.critical('No \'username\' key found in pillar for this proxy.') + return False + if 'password' not in opts['proxy']: + log.critical('No \'passwords\' key found in pillar for this proxy.') + return False + + DETAILS['url'] = 'https://{0}/api/'.format(opts['proxy']['host']) + + # Set configuration details + DETAILS['host'] = opts['proxy']['host'] + if 'serial' in opts['proxy']: + DETAILS['serial'] = opts['proxy'].get('serial') + if 'apikey' in opts['proxy']: + log.debug("Selected pan_key method for panos proxy module.") + DETAILS['method'] = 'pan_key' + DETAILS['apikey'] = opts['proxy'].get('apikey') + else: + log.debug("Selected pan_pass method for panos proxy module.") + DETAILS['method'] = 'pan_pass' + DETAILS['username'] = opts['proxy'].get('username') + DETAILS['password'] = opts['proxy'].get('password') + else: + if 'apikey' in opts['proxy']: + log.debug("Selected dev_key method for panos proxy module.") + DETAILS['method'] = 'dev_key' + DETAILS['apikey'] = opts['proxy'].get('apikey') + else: + log.debug("Selected dev_pass method for panos proxy module.") + DETAILS['method'] = 'dev_pass' + DETAILS['username'] = opts['proxy'].get('username') + DETAILS['password'] = opts['proxy'].get('password') + + # Ensure connectivity to the device + log.debug("Attempting to connect to panos proxy host.") + query = {'type': 'op', 'cmd': ''} + call(query) + log.debug("Successfully connected to panos proxy host.") + + DETAILS['initialized'] = True + + +def call(payload=None): + ''' + This function captures the query string and sends it to the Palo Alto device. + ''' + ret = {} + try: + if DETAILS['method'] == 'dev_key': + # Pass the api key without the target declaration + conditional_payload = {'key': DETAILS['apikey']} + payload.update(conditional_payload) + r = __utils__['http.query'](DETAILS['url'], + data=payload, + method='POST', + decode_type='xml', + decode=True, + verify_ssl=False, + raise_error=True) + ret = r['dict'][0] + elif DETAILS['method'] == 'dev_pass': + # Pass credentials without the target declaration + r = __utils__['http.query'](DETAILS['url'], + username=DETAILS['username'], + password=DETAILS['password'], + data=payload, + method='POST', + decode_type='xml', + decode=True, + verify_ssl=False, + raise_error=True) + ret = r['dict'][0] + elif DETAILS['method'] == 'pan_key': + # Pass the api key with the target declaration + conditional_payload = {'key': DETAILS['apikey'], + 'target': DETAILS['serial']} + payload.update(conditional_payload) + r = __utils__['http.query'](DETAILS['url'], + data=payload, + method='POST', + decode_type='xml', + decode=True, + verify_ssl=False, + raise_error=True) + ret = r['dict'][0] + elif DETAILS['method'] == 'pan_pass': + # Pass credentials with the target declaration + conditional_payload = {'target': DETAILS['serial']} + payload.update(conditional_payload) + r = __utils__['http.query'](DETAILS['url'], + username=DETAILS['username'], + password=DETAILS['password'], + data=payload, + method='POST', + decode_type='xml', + decode=True, + verify_ssl=False, + raise_error=True) + ret = r['dict'][0] + except KeyError as err: + raise salt.exceptions.CommandExecutionError("Did not receive a valid response from host.") + return ret + + +def is_required_version(required_version='0.0.0'): + ''' + Because different versions of Palo Alto support different command sets, this function + will return true if the current version of Palo Alto supports the required command. + ''' + if 'sw-version' in DETAILS['grains_cache']: + current_version = DETAILS['grains_cache']['sw-version'] + else: + # If we do not have the current sw-version cached, we cannot check version requirements. + return False + + required_version_split = required_version.split(".") + current_version_split = current_version.split(".") + + try: + if int(current_version_split[0]) > int(required_version_split[0]): + return True + elif int(current_version_split[0]) < int(required_version_split[0]): + return False + + if int(current_version_split[1]) > int(required_version_split[1]): + return True + elif int(current_version_split[1]) < int(required_version_split[1]): + return False + + if int(current_version_split[2]) > int(required_version_split[2]): + return True + elif int(current_version_split[2]) < int(required_version_split[2]): + return False + + # We have an exact match + return True + except Exception as err: + return False + + +def initialized(): + ''' + Since grains are loaded in many different places and some of those + places occur before the proxy can be initialized, return whether + our init() function has been called + ''' + return DETAILS.get('initialized', False) + + +def grains(): + ''' + Get the grains from the proxied device + ''' + if not DETAILS.get('grains_cache', {}): + DETAILS['grains_cache'] = GRAINS_CACHE + try: + query = {'type': 'op', 'cmd': ''} + DETAILS['grains_cache'] = call(query)['system'] + except Exception as err: + pass + return DETAILS['grains_cache'] + + +def grains_refresh(): + ''' + Refresh the grains from the proxied device + ''' + DETAILS['grains_cache'] = None + return grains() + + +def ping(): + ''' + Returns true if the device is reachable, else false. + ''' + try: + query = {'type': 'op', 'cmd': ''} + if 'system' in call(query): + return True + else: + return False + except Exception as err: + return False + + +def shutdown(): + ''' + Shutdown the connection to the proxy device. For this proxy, + shutdown is a no-op. + ''' + log.debug('Panos proxy shutdown() called.') diff --git a/salt/queues/pgjsonb_queue.py b/salt/queues/pgjsonb_queue.py index 93ea8ca3146..479c463b703 100644 --- a/salt/queues/pgjsonb_queue.py +++ b/salt/queues/pgjsonb_queue.py @@ -46,7 +46,7 @@ import json import sys # import salt libs -import salt.ext.six as six +from salt.ext import six from salt.exceptions import SaltInvocationError, SaltMasterError try: diff --git a/salt/queues/sqlite_queue.py b/salt/queues/sqlite_queue.py index 768fe8f2288..6232361b9b0 100644 --- a/salt/queues/sqlite_queue.py +++ b/salt/queues/sqlite_queue.py @@ -23,6 +23,9 @@ import re import sqlite3 as lite from salt.exceptions import SaltInvocationError +# Import 3rd-party libs +from salt.ext import six + log = logging.getLogger(__name__) # Define the module's virtual name @@ -139,7 +142,7 @@ def insert(queue, items): con = _conn(queue) with con: cur = con.cursor() - if isinstance(items, str): + if isinstance(items, six.string_types): items = _quote_escape(items) cmd = '''INSERT INTO {0}(name) VALUES('{1}')'''.format(queue, items) log.debug('SQL Query: {0}'.format(cmd)) @@ -171,7 +174,7 @@ def delete(queue, items): con = _conn(queue) with con: cur = con.cursor() - if isinstance(items, str): + if isinstance(items, six.string_types): items = _quote_escape(items) cmd = """DELETE FROM {0} WHERE name = '{1}'""".format(queue, items) log.debug('SQL Query: {0}'.format(cmd)) diff --git a/salt/renderers/gpg.py b/salt/renderers/gpg.py index 9b8e80b6954..7f9b65721c7 100644 --- a/salt/renderers/gpg.py +++ b/salt/renderers/gpg.py @@ -216,13 +216,13 @@ import logging from subprocess import Popen, PIPE # Import salt libs -import salt.utils +import salt.utils.path import salt.utils.stringio import salt.syspaths from salt.exceptions import SaltRenderError # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six log = logging.getLogger(__name__) @@ -233,7 +233,7 @@ def _get_gpg_exec(): ''' return the GPG executable or raise an error ''' - gpg_exec = salt.utils.which('gpg') + gpg_exec = salt.utils.path.which('gpg') if gpg_exec: return gpg_exec else: diff --git a/salt/renderers/jinja.py b/salt/renderers/jinja.py index f110c6c49f6..a78619ac23c 100644 --- a/salt/renderers/jinja.py +++ b/salt/renderers/jinja.py @@ -14,7 +14,7 @@ from salt.exceptions import SaltRenderError import salt.utils.templates # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import StringIO # pylint: disable=import-error log = logging.getLogger(__name__) @@ -66,11 +66,12 @@ def render(template_file, saltenv='base', sls='', argline='', sls=sls, context=context, tmplpath=tmplpath, + proxy=__proxy__, **kws) if not tmp_data.get('result', False): raise SaltRenderError( tmp_data.get('data', 'Unknown render error in jinja renderer') ) - if six.PY3 and isinstance(tmp_data['data'], bytes): + if isinstance(tmp_data['data'], bytes): tmp_data['data'] = tmp_data['data'].decode(__salt_system_encoding__) return StringIO(tmp_data['data']) diff --git a/salt/renderers/mako.py b/salt/renderers/mako.py index 4704b4d0540..9f8b43077d6 100644 --- a/salt/renderers/mako.py +++ b/salt/renderers/mako.py @@ -7,7 +7,7 @@ Mako Renderer for Salt from __future__ import absolute_import # Import salt libs -import salt.ext.six as six +from salt.ext import six import salt.utils.templates from salt.exceptions import SaltRenderError diff --git a/salt/renderers/nacl.py b/salt/renderers/nacl.py new file mode 100644 index 00000000000..91ba558e9d7 --- /dev/null +++ b/salt/renderers/nacl.py @@ -0,0 +1,102 @@ +# -*- coding: utf-8 -*- +r''' +Renderer that will decrypt NACL ciphers + +Any key in the SLS file can be an NACL cipher, and this renderer will decrypt it +before passing it off to Salt. This allows you to safely store secrets in +source control, in such a way that only your Salt master can decrypt them and +distribute them only to the minions that need them. + +The typical use-case would be to use ciphers in your pillar data, and keep a +secret key on your master. You can put the public key in source control so that +developers can add new secrets quickly and easily. + +This renderer requires the libsodium library binary and libnacl >= 1.5.1 +python package (support for sealed boxes came in 1.5.1 version). + + +Setup +----- + +To set things up, first generate a keypair. On the master, run the following: + +.. code-block:: bash + + # salt-call --local nacl.keygen sk_file=/root/.nacl + + +Using encrypted pillar +--------------------- + +To encrypt secrets, copy the public key to your local machine and run: + +.. code-block:: bash + + $ salt-call --local nacl.enc datatoenc pk_file=/root/.nacl.pub + + +To apply the renderer on a file-by-file basis add the following line to the +top of any pillar with nacl encrypted data in it: + +.. code-block:: yaml + + #!yaml|nacl + +Now with your renderer configured, you can include your ciphers in your pillar +data like so: + +.. code-block:: yaml + + #!yaml|nacl + + a-secret: "NACL[MRN3cc+fmdxyQbz6WMF+jq1hKdU5X5BBI7OjK+atvHo1ll+w1gZ7XyWtZVfq9gK9rQaMfkDxmidJKwE0Mw==]" +''' + + +from __future__ import absolute_import +import re +import logging + +# Import salt libs +import salt.utils +import salt.utils.stringio +import salt.syspaths + +# Import 3rd-party libs +import salt.ext.six as six + +log = logging.getLogger(__name__) +NACL_REGEX = r'^NACL\[(.*)\]$' + + +def _decrypt_object(obj, **kwargs): + ''' + Recursively try to decrypt any object. If the object is a six.string_types + (string or unicode), and it contains a valid NACLENC pretext, decrypt it, + otherwise keep going until a string is found. + ''' + if salt.utils.stringio.is_readable(obj): + return _decrypt_object(obj.getvalue(), **kwargs) + if isinstance(obj, six.string_types): + if re.search(NACL_REGEX, obj) is not None: + return __salt__['nacl.dec'](re.search(NACL_REGEX, obj).group(1), **kwargs) + else: + return obj + elif isinstance(obj, dict): + for key, value in six.iteritems(obj): + obj[key] = _decrypt_object(value, **kwargs) + return obj + elif isinstance(obj, list): + for key, value in enumerate(obj): + obj[key] = _decrypt_object(value, **kwargs) + return obj + else: + return obj + + +def render(nacl_data, saltenv='base', sls='', argline='', **kwargs): + ''' + Decrypt the data to be rendered using the given nacl key or the one given + in config + ''' + return _decrypt_object(nacl_data, **kwargs) diff --git a/salt/renderers/pass.py b/salt/renderers/pass.py index 05607e94751..381ca194489 100644 --- a/salt/renderers/pass.py +++ b/salt/renderers/pass.py @@ -48,11 +48,11 @@ from os.path import expanduser from subprocess import Popen, PIPE # Import salt libs -import salt.utils +import salt.utils.path from salt.exceptions import SaltRenderError # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six log = logging.getLogger(__name__) @@ -61,7 +61,7 @@ def _get_pass_exec(): """ Return the pass executable or raise an error """ - pass_exec = salt.utils.which('pass') + pass_exec = salt.utils.path.which('pass') if pass_exec: return pass_exec else: diff --git a/salt/renderers/pydsl.py b/salt/renderers/pydsl.py index 59a32314561..14874b5c55e 100644 --- a/salt/renderers/pydsl.py +++ b/salt/renderers/pydsl.py @@ -336,9 +336,10 @@ For example: ''' from __future__ import absolute_import -import imp +import types +import salt.utils.pydsl as pydsl +import salt.utils.stringutils from salt.ext.six import exec_ -from salt.utils import pydsl from salt.utils.pydsl import PyDslError from salt.exceptions import SaltRenderError @@ -346,12 +347,14 @@ __all__ = ['render'] def render(template, saltenv='base', sls='', tmplpath=None, rendered_sls=None, **kws): - mod = imp.new_module(sls) + sls = salt.utils.stringutils.to_str(sls) + mod = types.ModuleType(sls) # Note: mod object is transient. It's existence only lasts as long as # the lowstate data structure that the highstate in the sls file # is compiled to. - mod.__name__ = sls + # __name__ can't be assigned a unicode + mod.__name__ = str(sls) # future lint: disable=non-unicode-string # to workaround state.py's use of copy.deepcopy(chunk) mod.__deepcopy__ = lambda x: mod diff --git a/salt/renderers/pyobjects.py b/salt/renderers/pyobjects.py index 86f5b06201d..9f0b8430fda 100644 --- a/salt/renderers/pyobjects.py +++ b/salt/renderers/pyobjects.py @@ -223,21 +223,57 @@ different grain matches. class Samba(Map): merge = 'samba:lookup' + # NOTE: priority is new to 2017.7.0 + priority = ('os_family', 'os') + + class Ubuntu: + __grain__ = 'os' + service = 'smbd' class Debian: server = 'samba' client = 'samba-client' service = 'samba' - class Ubuntu: - __grain__ = 'os' - service = 'smbd' - - class RedHat: + class RHEL: + __match__ = 'RedHat' server = 'samba' client = 'samba' service = 'smb' +.. note:: + By default, the ``os_family`` grain will be used as the target for + matching. This can be overridden by specifying a ``__grain__`` attribute. + + If a ``__match__`` attribute is defined for a given class, then that value + will be matched against the targeted grain, otherwise the class name's + value will be be matched. + + Given the above example, the following is true: + + 1. Minions with an ``os_family`` of **Debian** will be assigned the + attributes defined in the **Debian** class. + 2. Minions with an ``os`` grain of **Ubuntu** will be assigned the + attributes defined in the **Ubuntu** class. + 3. Minions with an ``os_family`` grain of **RedHat** will be assigned the + attributes defined in the **RHEL** class. + + That said, sometimes a minion may match more than one class. For instance, + in the above example, Ubuntu minions will match both the **Debian** and + **Ubuntu** classes, since Ubuntu has an ``os_family`` grain of **Debian** + an an ``os`` grain of **Ubuntu**. As of the 2017.7.0 release, the order is + dictated by the order of declaration, with classes defined later overriding + earlier ones. Addtionally, 2017.7.0 adds support for explicitly defining + the ordering using an optional attribute called ``priority``. + + Given the above example, ``os_family`` matches will be processed first, + with ``os`` matches processed after. This would have the effect of + assigning ``smbd`` as the ``service`` attribute on Ubuntu minions. If the + ``priority`` item was not defined, or if the order of the items in the + ``priority`` tuple were reversed, Ubuntu minions would have a ``service`` + attribute of ``samba``, since ``os_family`` matches would have been + processed second. + To use this new data you can import it into your state file and then access your attributes. To access the data in the map you simply access the attribute name on the base class that is extending Map. Assuming the above Map was in the @@ -266,11 +302,11 @@ import re # Import Salt Libs from salt.ext.six import exec_ -import salt.utils +import salt.utils.files import salt.loader from salt.fileclient import get_file_client from salt.utils.pyobjects import Registry, StateFactory, SaltObject, Map -import salt.ext.six as six +from salt.ext import six # our import regexes FROM_RE = re.compile(r'^\s*from\s+(salt:\/\/.*)\s+import (.*)$') @@ -421,7 +457,7 @@ def render(template, saltenv='base', sls='', salt_data=True, **kwargs): 'Could not find the file \'{0}\''.format(import_file) ) - with salt.utils.fopen(state_file) as state_fh: + with salt.utils.files.fopen(state_file) as state_fh: state_contents, state_globals = process_template(state_fh) exec_(state_contents, state_globals) diff --git a/salt/renderers/stateconf.py b/salt/renderers/stateconf.py index fe41692e3a5..7165b0cfbda 100644 --- a/salt/renderers/stateconf.py +++ b/salt/renderers/stateconf.py @@ -35,11 +35,11 @@ import copy from os import path as ospath # Import salt libs -import salt.utils +import salt.utils.files from salt.exceptions import SaltRenderError # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import StringIO # pylint: disable=import-error __all__ = ['render'] @@ -206,7 +206,7 @@ def render(input, saltenv='base', sls='', argline='', **kws): raise INVALID_USAGE_ERROR if isinstance(input, six.string_types): - with salt.utils.fopen(input, 'r') as ifile: + with salt.utils.files.fopen(input, 'r') as ifile: sls_templ = ifile.read() else: # assume file-like sls_templ = input.read() diff --git a/salt/renderers/wempy.py b/salt/renderers/wempy.py index a6777e76d69..9c27e9c9526 100644 --- a/salt/renderers/wempy.py +++ b/salt/renderers/wempy.py @@ -4,7 +4,7 @@ from __future__ import absolute_import # Import salt libs -import salt.ext.six as six +from salt.ext import six from salt.exceptions import SaltRenderError import salt.utils.templates diff --git a/salt/renderers/yaml.py b/salt/renderers/yaml.py index 08d1514ddd3..2adb6679922 100644 --- a/salt/renderers/yaml.py +++ b/salt/renderers/yaml.py @@ -19,9 +19,8 @@ import salt.utils.url from salt.utils.yamlloader import SaltYamlSafeLoader, load from salt.utils.odict import OrderedDict from salt.exceptions import SaltRenderError -import salt.ext.six as six +from salt.ext import six from salt.ext.six import string_types -from salt.ext.six.moves import range log = logging.getLogger(__name__) @@ -67,12 +66,6 @@ def render(yaml_data, saltenv='base', sls='', argline='', **kws): ) if not data: data = {} - else: - if 'config.get' in __salt__: - if __salt__['config.get']('yaml_utf8', False): - data = _yaml_result_unicode_to_utf8(data) - elif __opts__.get('yaml_utf8'): - data = _yaml_result_unicode_to_utf8(data) log.debug('Results of YAML rendering: \n{0}'.format(data)) def _validate_data(data): @@ -94,23 +87,3 @@ def render(yaml_data, saltenv='base', sls='', argline='', **kws): _validate_data(data) return data - - -def _yaml_result_unicode_to_utf8(data): - '''' - Replace `unicode` strings by utf-8 `str` in final yaml result - - This is a recursive function - ''' - if six.PY3: - return data - if isinstance(data, OrderedDict): - for key, elt in six.iteritems(data): - data[key] = _yaml_result_unicode_to_utf8(elt) - elif isinstance(data, list): - for i in range(len(data)): - data[i] = _yaml_result_unicode_to_utf8(data[i]) - elif isinstance(data, six.text_type): - # here also - data = data.encode('utf-8') - return data diff --git a/salt/returners/carbon_return.py b/salt/returners/carbon_return.py index 077236bd88a..ba06ad29b6f 100644 --- a/salt/returners/carbon_return.py +++ b/salt/returners/carbon_return.py @@ -95,7 +95,7 @@ import salt.utils.jid import salt.returners # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import cPickle, map # pylint: disable=import-error,no-name-in-module,redefined-builtin log = logging.getLogger(__name__) @@ -303,4 +303,4 @@ def prep_jid(nocache=False, passed_jid=None): # pylint: disable=unused-argument ''' Do any work necessary to prepare a JID, including sending a custom id ''' - return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid() + return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid(__opts__) diff --git a/salt/returners/cassandra_cql_return.py b/salt/returners/cassandra_cql_return.py index 2a3b3255959..5e7e7c0ff97 100644 --- a/salt/returners/cassandra_cql_return.py +++ b/salt/returners/cassandra_cql_return.py @@ -454,4 +454,4 @@ def prep_jid(nocache, passed_jid=None): # pylint: disable=unused-argument ''' Do any work necessary to prepare a JID, including sending a custom id ''' - return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid() + return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid(__opts__) diff --git a/salt/returners/cassandra_return.py b/salt/returners/cassandra_return.py index 47b42578283..d2c0e2b06df 100644 --- a/salt/returners/cassandra_return.py +++ b/salt/returners/cassandra_return.py @@ -27,7 +27,7 @@ import logging import salt.utils.jid # Import third party libs -import salt.ext.six as six +from salt.ext import six try: import pycassa # pylint: disable=import-error HAS_PYCASSA = True @@ -80,4 +80,4 @@ def prep_jid(nocache=False, passed_jid=None): # pylint: disable=unused-argument ''' Do any work necessary to prepare a JID, including sending a custom id ''' - return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid() + return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid(__opts__) diff --git a/salt/returners/couchbase_return.py b/salt/returners/couchbase_return.py index 24c3a9105ae..972d7afb6b3 100644 --- a/salt/returners/couchbase_return.py +++ b/salt/returners/couchbase_return.py @@ -160,7 +160,7 @@ def prep_jid(nocache=False, passed_jid=None): So do what you have to do to make sure that stays the case ''' if passed_jid is None: - jid = salt.utils.jid.gen_jid() + jid = salt.utils.jid.gen_jid(__opts__) else: jid = passed_jid @@ -213,8 +213,8 @@ def save_load(jid, clear_load, minion=None): try: jid_doc = cb_.get(str(jid)) except couchbase.exceptions.NotFoundError: - log.warning('Could not write job cache file for jid: {0}'.format(jid)) - return False + cb_.add(str(jid), {}, ttl=_get_ttl()) + jid_doc = cb_.get(str(jid)) jid_doc.value['load'] = clear_load cb_.replace(str(jid), jid_doc.value, cas=jid_doc.cas, ttl=_get_ttl()) @@ -223,10 +223,11 @@ def save_load(jid, clear_load, minion=None): if 'tgt' in clear_load and clear_load['tgt'] != '': ckminions = salt.utils.minions.CkMinions(__opts__) # Retrieve the minions list - minions = ckminions.check_minions( + _res = ckminions.check_minions( clear_load['tgt'], clear_load.get('tgt_type', 'glob') ) + minions = _res['minions'] save_minions(jid, minions) diff --git a/salt/returners/couchdb_return.py b/salt/returners/couchdb_return.py index d24020db4e6..117b2802f40 100644 --- a/salt/returners/couchdb_return.py +++ b/salt/returners/couchdb_return.py @@ -364,7 +364,7 @@ def prep_jid(nocache=False, passed_jid=None): # pylint: disable=unused-argument ''' Do any work necessary to prepare a JID, including sending a custom id ''' - return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid() + return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid(__opts__) def save_minions(jid, minions, syndic_id=None): # pylint: disable=unused-argument diff --git a/salt/returners/django_return.py b/salt/returners/django_return.py index 5d756e61117..8a4517a5ce8 100644 --- a/salt/returners/django_return.py +++ b/salt/returners/django_return.py @@ -82,4 +82,4 @@ def prep_jid(nocache=False, passed_jid=None): ''' Do any work necessary to prepare a JID, including sending a custom ID ''' - return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid() + return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid(__opts__) diff --git a/salt/returners/elasticsearch_return.py b/salt/returners/elasticsearch_return.py index 2ddb4a26ebe..e4ffb20f1ef 100644 --- a/salt/returners/elasticsearch_return.py +++ b/salt/returners/elasticsearch_return.py @@ -107,7 +107,7 @@ import salt.returners import salt.utils.jid # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six __virtualname__ = 'elasticsearch' @@ -362,7 +362,7 @@ def prep_jid(nocache=False, passed_jid=None): # pylint: disable=unused-argument ''' Do any work necessary to prepare a JID, including sending a custom id ''' - return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid() + return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid(__opts__) def save_load(jid, load, minions=None): diff --git a/salt/returners/etcd_return.py b/salt/returners/etcd_return.py index 6582e957e28..5f3d8a2baea 100644 --- a/salt/returners/etcd_return.py +++ b/salt/returners/etcd_return.py @@ -223,4 +223,4 @@ def prep_jid(nocache=False, passed_jid=None): # pylint: disable=unused-argument ''' Do any work necessary to prepare a JID, including sending a custom id ''' - return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid() + return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid(__opts__) diff --git a/salt/returners/highstate_return.py b/salt/returners/highstate_return.py index 385abc06cc5..72286b86489 100644 --- a/salt/returners/highstate_return.py +++ b/salt/returners/highstate_return.py @@ -89,7 +89,7 @@ import yaml from salt.ext.six.moves import range from salt.ext.six.moves import StringIO -import salt.utils +import salt.utils.files import salt.returners log = logging.getLogger(__name__) @@ -440,7 +440,7 @@ def _produce_output(report, failed, setup): if report_delivery == 'file': output_file = _sprinkle(setup.get('file_output', '/tmp/test.rpt')) - with salt.utils.fopen(output_file, 'w') as out: + with salt.utils.files.fopen(output_file, 'w') as out: out.write(report_text) else: msg = MIMEText(report_text, report_format) @@ -490,7 +490,7 @@ def __test_html(): report_delivery: file file_output: '/srv/salt/_returners/test.rpt' ''' - with salt.utils.fopen('test.rpt', 'r') as input_file: + with salt.utils.files.fopen('test.rpt', 'r') as input_file: data_text = input_file.read() data = yaml.safe_load(data_text) @@ -499,7 +499,7 @@ def __test_html(): string_file.seek(0) result = string_file.read() - with salt.utils.fopen('test.html', 'w') as output: + with salt.utils.files.fopen('test.html', 'w') as output: output.write(result) diff --git a/salt/returners/influxdb_return.py b/salt/returners/influxdb_return.py index d37958b0e8d..e6cafb7cc53 100644 --- a/salt/returners/influxdb_return.py +++ b/salt/returners/influxdb_return.py @@ -328,4 +328,4 @@ def prep_jid(nocache=False, passed_jid=None): # pylint: disable=unused-argument ''' Do any work necessary to prepare a JID, including sending a custom id ''' - return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid() + return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid(__opts__) diff --git a/salt/returners/local_cache.py b/salt/returners/local_cache.py index 83b39efbc8e..c0b135c3efa 100644 --- a/salt/returners/local_cache.py +++ b/salt/returners/local_cache.py @@ -16,14 +16,15 @@ import bisect # Import salt libs import salt.payload -import salt.utils +import salt.utils.atomicfile import salt.utils.files import salt.utils.jid +import salt.utils.minions import salt.exceptions # Import 3rd-party libs import msgpack -import salt.ext.six as six +from salt.ext import six log = logging.getLogger(__name__) @@ -69,7 +70,7 @@ def _walk_through(job_dir): if not os.path.isfile(load_path): continue - with salt.utils.fopen(load_path, 'rb') as rfh: + with salt.utils.files.fopen(load_path, 'rb') as rfh: job = serial.load(rfh) jid = job['jid'] yield jid, job, t_path, final @@ -89,7 +90,7 @@ def prep_jid(nocache=False, passed_jid=None, recurse_count=0): log.error(err) raise salt.exceptions.SaltCacheError(err) if passed_jid is None: # this can be a None or an empty string. - jid = salt.utils.jid.gen_jid() + jid = salt.utils.jid.gen_jid(__opts__) else: jid = passed_jid @@ -106,13 +107,13 @@ def prep_jid(nocache=False, passed_jid=None, recurse_count=0): return prep_jid(nocache=nocache, recurse_count=recurse_count+1) try: - with salt.utils.fopen(os.path.join(jid_dir, 'jid'), 'wb+') as fn_: + with salt.utils.files.fopen(os.path.join(jid_dir, 'jid'), 'wb+') as fn_: if six.PY2: fn_.write(jid) else: fn_.write(bytes(jid, 'utf-8')) if nocache: - with salt.utils.fopen(os.path.join(jid_dir, 'nocache'), 'wb+') as fn_: + with salt.utils.files.fopen(os.path.join(jid_dir, 'nocache'), 'wb+') as fn_: fn_.write(b'') except IOError: log.warning('Could not write out jid file for job {0}. Retrying.'.format(jid)) @@ -209,7 +210,7 @@ def save_load(jid, clear_load, minions=None, recurse_count=0): else: raise try: - with salt.utils.fopen(os.path.join(jid_dir, LOAD_P), 'w+b') as wfh: + with salt.utils.files.fopen(os.path.join(jid_dir, LOAD_P), 'w+b') as wfh: serial.dump(clear_load, wfh) except IOError as exc: log.warning( @@ -224,10 +225,11 @@ def save_load(jid, clear_load, minions=None, recurse_count=0): if minions is None: ckminions = salt.utils.minions.CkMinions(__opts__) # Retrieve the minions list - minions = ckminions.check_minions( + _res = ckminions.check_minions( clear_load['tgt'], clear_load.get('tgt_type', 'glob') ) + minions = _res['minions'] # save the minions to a cache so we can see in the UI save_minions(jid, minions) @@ -274,7 +276,7 @@ def save_minions(jid, minions, syndic_id=None): os.makedirs(jid_dir) except OSError: pass - with salt.utils.fopen(minions_path, 'w+b') as wfh: + with salt.utils.files.fopen(minions_path, 'w+b') as wfh: serial.dump(minions, wfh) except IOError as exc: log.error( @@ -292,7 +294,7 @@ def get_load(jid): if not os.path.exists(jid_dir) or not os.path.exists(load_fn): return {} serial = salt.payload.Serial(__opts__) - with salt.utils.fopen(os.path.join(jid_dir, LOAD_P), 'rb') as rfh: + with salt.utils.files.fopen(os.path.join(jid_dir, LOAD_P), 'rb') as rfh: ret = serial.load(rfh) minions_cache = [os.path.join(jid_dir, MINIONS_P)] @@ -303,7 +305,7 @@ def get_load(jid): for minions_path in minions_cache: log.debug('Reading minion list from %s', minions_path) try: - with salt.utils.fopen(minions_path, 'rb') as rfh: + with salt.utils.files.fopen(minions_path, 'rb') as rfh: all_minions.update(serial.load(rfh)) except IOError as exc: salt.utils.files.process_read_exception(exc, minions_path) @@ -335,7 +337,7 @@ def get_jid(jid): continue while fn_ not in ret: try: - with salt.utils.fopen(retp, 'rb') as rfh: + with salt.utils.files.fopen(retp, 'rb') as rfh: ret_data = serial.load(rfh) if not isinstance(ret_data, dict) or 'return' not in ret_data: # Convert the old format in which return.p contains the only return data to @@ -344,7 +346,7 @@ def get_jid(jid): ret_data = {'return': ret_data} ret[fn_] = ret_data if os.path.isfile(outp): - with salt.utils.fopen(outp, 'rb') as rfh: + with salt.utils.files.fopen(outp, 'rb') as rfh: ret[fn_]['out'] = serial.load(rfh) except Exception as exc: if 'Permission denied:' in str(exc): @@ -396,7 +398,6 @@ def clean_old_jobs(): Clean out the old jobs from the job cache ''' if __opts__['keep_jobs'] != 0: - cur = time.time() jid_root = _job_dir() if not os.path.exists(jid_root): @@ -426,7 +427,7 @@ def clean_old_jobs(): shutil.rmtree(t_path) elif os.path.isfile(jid_file): jid_ctime = os.stat(jid_file).st_ctime - hours_difference = (cur - jid_ctime) / 3600.0 + hours_difference = (time.time()- jid_ctime) / 3600.0 if hours_difference > __opts__['keep_jobs'] and os.path.exists(t_path): # Remove the entire t_path from the original JID dir shutil.rmtree(t_path) @@ -440,7 +441,7 @@ def clean_old_jobs(): # Checking the time again prevents a possible race condition where # t_path JID dirs were created, but not yet populated by a jid file. t_path_ctime = os.stat(t_path).st_ctime - hours_difference = (cur - t_path_ctime) / 3600.0 + hours_difference = (time.time() - t_path_ctime) / 3600.0 if hours_difference > __opts__['keep_jobs']: shutil.rmtree(t_path) @@ -455,7 +456,7 @@ def update_endtime(jid, time): try: if not os.path.exists(jid_dir): os.makedirs(jid_dir) - with salt.utils.fopen(os.path.join(jid_dir, ENDTIME), 'w') as etfile: + with salt.utils.files.fopen(os.path.join(jid_dir, ENDTIME), 'w') as etfile: etfile.write(time) except IOError as exc: log.warning('Could not write job invocation cache file: {0}'.format(exc)) @@ -471,7 +472,7 @@ def get_endtime(jid): etpath = os.path.join(jid_dir, ENDTIME) if not os.path.exists(etpath): return False - with salt.utils.fopen(etpath, 'r') as etfile: + with salt.utils.files.fopen(etpath, 'r') as etfile: endtime = etfile.read().strip('\n') return endtime @@ -498,7 +499,7 @@ def save_reg(data): else: raise try: - with salt.utils.fopen(regfile, 'a') as fh_: + with salt.utils.files.fopen(regfile, 'a') as fh_: msgpack.dump(data, fh_) except: log.error('Could not write to msgpack file {0}'.format(__opts__['outdir'])) @@ -512,7 +513,7 @@ def load_reg(): reg_dir = _reg_dir() regfile = os.path.join(reg_dir, 'register') try: - with salt.utils.fopen(regfile, 'r') as fh_: + with salt.utils.files.fopen(regfile, 'r') as fh_: return msgpack.load(fh_) except: log.error('Could not write to msgpack file {0}'.format(__opts__['outdir'])) diff --git a/salt/returners/mattermost_returner.py b/salt/returners/mattermost_returner.py index 3d022297dec..c682c65be5d 100644 --- a/salt/returners/mattermost_returner.py +++ b/salt/returners/mattermost_returner.py @@ -36,7 +36,7 @@ import logging import json # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six # pylint: disable=import-error,no-name-in-module,redefined-builtin import salt.ext.six.moves.http_client # pylint: enable=import-error,no-name-in-module,redefined-builtin diff --git a/salt/returners/memcache_return.py b/salt/returners/memcache_return.py index dd3657da1f6..c00dcbdf9bc 100644 --- a/salt/returners/memcache_return.py +++ b/salt/returners/memcache_return.py @@ -134,7 +134,7 @@ def prep_jid(nocache=False, passed_jid=None): # pylint: disable=unused-argument ''' Do any work necessary to prepare a JID, including sending a custom id ''' - return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid() + return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid(__opts__) def returner(ret): diff --git a/salt/returners/mongo_future_return.py b/salt/returners/mongo_future_return.py index b36d03e02a5..0d9c7328b13 100644 --- a/salt/returners/mongo_future_return.py +++ b/salt/returners/mongo_future_return.py @@ -71,7 +71,7 @@ import logging # Import Salt libs import salt.utils.jid import salt.returners -import salt.ext.six as six +from salt.ext import six # Import third party libs try: @@ -322,7 +322,7 @@ def prep_jid(nocache=False, passed_jid=None): # pylint: disable=unused-argument ''' Do any work necessary to prepare a JID, including sending a custom id ''' - return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid() + return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid(__opts__) def event_return(events): diff --git a/salt/returners/mongo_return.py b/salt/returners/mongo_return.py index 07177addd57..f59c25f9f55 100644 --- a/salt/returners/mongo_return.py +++ b/salt/returners/mongo_return.py @@ -68,7 +68,7 @@ import logging # import Salt libs import salt.utils.jid import salt.returners -import salt.ext.six as six +from salt.ext import six # Import third party libs try: @@ -231,7 +231,7 @@ def prep_jid(nocache=False, passed_jid=None): # pylint: disable=unused-argument ''' Do any work necessary to prepare a JID, including sending a custom id ''' - return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid() + return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid(__opts__) def save_minions(jid, minions, syndic_id=None): # pylint: disable=unused-argument diff --git a/salt/returners/mysql.py b/salt/returners/mysql.py index ad5f11f35cc..a7bfbed243b 100644 --- a/salt/returners/mysql.py +++ b/salt/returners/mysql.py @@ -154,7 +154,7 @@ import salt.utils.jid import salt.exceptions # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six try: import MySQLdb HAS_MYSQL = True @@ -460,7 +460,7 @@ def prep_jid(nocache=False, passed_jid=None): # pylint: disable=unused-argument ''' Do any work necessary to prepare a JID, including sending a custom id ''' - return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid() + return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid(__opts__) def _purge_jobs(timestamp): diff --git a/salt/returners/odbc.py b/salt/returners/odbc.py index 7cc9ada0d9d..03c114cb100 100644 --- a/salt/returners/odbc.py +++ b/salt/returners/odbc.py @@ -329,4 +329,4 @@ def prep_jid(nocache=False, passed_jid=None): # pylint: disable=unused-argument ''' Do any work necessary to prepare a JID, including sending a custom id ''' - return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid() + return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid(__opts__) diff --git a/salt/returners/pgjsonb.py b/salt/returners/pgjsonb.py index f6af142ac03..dd09d31d784 100644 --- a/salt/returners/pgjsonb.py +++ b/salt/returners/pgjsonb.py @@ -416,4 +416,4 @@ def prep_jid(nocache=False, passed_jid=None): # pylint: disable=unused-argument ''' Do any work necessary to prepare a JID, including sending a custom id ''' - return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid() + return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid(__opts__) diff --git a/salt/returners/postgres.py b/salt/returners/postgres.py index 7fb45fe8376..02cbde3ce79 100644 --- a/salt/returners/postgres.py +++ b/salt/returners/postgres.py @@ -381,4 +381,4 @@ def prep_jid(nocache=False, passed_jid=None): # pylint: disable=unused-argument ''' Do any work necessary to prepare a JID, including sending a custom id ''' - return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid() + return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid(__opts__) diff --git a/salt/returners/postgres_local_cache.py b/salt/returners/postgres_local_cache.py index a7d77efe4f2..0485fa4fe9f 100644 --- a/salt/returners/postgres_local_cache.py +++ b/salt/returners/postgres_local_cache.py @@ -117,7 +117,7 @@ import sys # Import salt libs import salt.utils import salt.utils.jid -import salt.ext.six as six +from salt.ext import six # Import third party libs try: @@ -189,7 +189,7 @@ def _gen_jid(cur): ''' Generate an unique job id ''' - jid = salt.utils.jid.gen_jid() + jid = salt.utils.jid.gen_jid(__opts__) sql = '''SELECT jid FROM jids WHERE jid = %s''' cur.execute(sql, (jid,)) data = cur.fetchall() diff --git a/salt/returners/rawfile_json.py b/salt/returners/rawfile_json.py index 6d2a0730bd0..0341a7bce4a 100644 --- a/salt/returners/rawfile_json.py +++ b/salt/returners/rawfile_json.py @@ -24,6 +24,7 @@ import json import salt.returners import salt.utils +import salt.utils.files log = logging.getLogger(__name__) @@ -57,7 +58,7 @@ def returner(ret): ''' opts = _get_options({}) # Pass in empty ret, since this is a list of events try: - with salt.utils.flopen(opts['filename'], 'a') as logfile: + with salt.utils.files.flopen(opts['filename'], 'a') as logfile: logfile.write(json.dumps(ret)+'\n') except: log.error('Could not write to rawdata_json file {0}'.format(opts['filename'])) @@ -74,7 +75,7 @@ def event_return(events): return opts = _get_options({}) # Pass in empty ret, since this is a list of events try: - with salt.utils.flopen(opts['filename'], 'a') as logfile: + with salt.utils.files.flopen(opts['filename'], 'a') as logfile: for event in events: json.dump(event, logfile) logfile.write('\n') diff --git a/salt/returners/redis_return.py b/salt/returners/redis_return.py index 8ebf273f53c..5eea5bf782b 100644 --- a/salt/returners/redis_return.py +++ b/salt/returners/redis_return.py @@ -12,6 +12,19 @@ config, these are the defaults: redis.host: 'salt' redis.port: 6379 +Cluster Mode Example: + +.. code-block::yaml + + redis.db: '0' + redis.cluster_mode: true + redis.cluster.skip_full_coverage_check: true + redis.cluster.startup_nodes: + - host: redis-member-1 + port: 6379 + - host: redis-member-2 + port: 6379 + Alternative configuration values can be used by prefacing the configuration. Any values not found in the alternative configuration will be pulled from the default location: @@ -44,20 +57,46 @@ To override individual configuration items, append --return_kwargs '{"key:": "va salt '*' test.ping --return redis --return_kwargs '{"db": "another-salt"}' +Redis Cluster Mode Options: + +cluster_mode: ``False`` + Whether cluster_mode is enabled or not + +cluster.startup_nodes: + A list of host, port dictionaries pointing to cluster members. At least one is required + but multiple nodes are better + + .. code-block::yaml + + cache.redis.cluster.startup_nodes + - host: redis-member-1 + port: 6379 + - host: redis-member-2 + port: 6379 + +cluster.skip_full_coverage_check: ``False`` + Some cluster providers restrict certain redis commands such as CONFIG for enhanced security. + Set this option to true to skip checks that required advanced privileges. + + .. note:: + + Most cloud hosted redis clusters will require this to be set to ``True`` + + ''' -# Import python libs +# Import Python libs from __future__ import absolute_import import json import logging # Import Salt libs -import salt.ext.six as six -import salt.utils -import salt.utils.jid import salt.returners +import salt.utils.jid +import salt.utils.platform -# Import third party libs +# Import 3rd-party libs +from salt.ext import six try: import redis HAS_REDIS = True @@ -66,14 +105,28 @@ except ImportError: log = logging.getLogger(__name__) +try: + from rediscluster import StrictRedisCluster + HAS_REDIS_CLUSTER = True +except ImportError: + HAS_REDIS_CLUSTER = False + # Define the module's virtual name __virtualname__ = 'redis' def __virtual__(): + ''' + The redis library must be installed for this module to work. + + The redis redis cluster library must be installed if cluster_mode is True + ''' + if not HAS_REDIS: return False, 'Could not import redis returner; ' \ 'redis python client is not installed.' + if not HAS_REDIS_CLUSTER and _get_options()['cluster_mode']: + return (False, "Please install the redis-py-cluster package.") return __virtualname__ @@ -83,13 +136,20 @@ def _get_options(ret=None): ''' attrs = {'host': 'host', 'port': 'port', - 'db': 'db'} + 'db': 'db', + 'cluster_mode': 'cluster_mode', + 'startup_nodes': 'cluster.startup_nodes', + 'skip_full_coverage_check': 'cluster.skip_full_coverage_check', + } - if salt.utils.is_proxy(): + if salt.utils.platform.is_proxy(): return { 'host': __opts__.get('redis.host', 'salt'), 'port': __opts__.get('redis.port', 6379), - 'db': __opts__.get('redis.db', '0') + 'db': __opts__.get('redis.db', '0'), + 'cluster_mode': __opts__.get('redis.cluster_mode', False), + 'startup_nodes': __opts__.get('redis.cluster.startup_nodes', {}), + 'skip_full_coverage_check': __opts__.get('redis.cluster.skip_full_coverage_check', False) } _options = salt.returners.get_returner_options(__virtualname__, @@ -103,10 +163,12 @@ def _get_options(ret=None): CONN_POOL = None -def _get_conn_pool(): +def _get_conn_pool(_options): global CONN_POOL if CONN_POOL is None: - CONN_POOL = redis.ConnectionPool() + CONN_POOL = redis.ConnectionPool(host=_options.get('host'), + port=_options.get('port'), + db=_options.get('db')) return CONN_POOL @@ -115,16 +177,14 @@ def _get_serv(ret=None): Return a redis server object ''' _options = _get_options(ret) - host = _options.get('host') - port = _options.get('port') - db = _options.get('db') - pool = _get_conn_pool() - return redis.Redis( - host=host, - port=port, - db=db, - connection_pool=pool) + if _options.get('cluster_mode'): + return StrictRedisCluster(startup_nodes=_options.get('startup_nodes'), + skip_full_coverage_check=_options.get('skip_full_coverage_check'), + decode_responses=True) + else: + pool = _get_conn_pool(_options) + return redis.StrictRedis(connection_pool=pool) def _get_ttl(): @@ -150,7 +210,7 @@ def save_load(jid, load, minions=None): Save the load to the specified jid ''' serv = _get_serv(ret=None) - serv.setex('load:{0}'.format(jid), json.dumps(load), _get_ttl()) + serv.setex('load:{0}'.format(jid), _get_ttl(), json.dumps(load)) def save_minions(jid, minions, syndic_id=None): # pylint: disable=unused-argument @@ -252,4 +312,4 @@ def prep_jid(nocache=False, passed_jid=None): # pylint: disable=unused-argument ''' Do any work necessary to prepare a JID, including sending a custom id ''' - return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid() + return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid(__opts__) diff --git a/salt/returners/sentry_return.py b/salt/returners/sentry_return.py index 6fda73db6d6..3b3c5b11eb7 100644 --- a/salt/returners/sentry_return.py +++ b/salt/returners/sentry_return.py @@ -48,11 +48,10 @@ import logging # Import Salt libs import salt.utils.jid -import salt.ext.six as six +from salt.ext import six try: from raven import Client - from raven.transport.http import HTTPTransport has_raven = True except ImportError: @@ -130,7 +129,7 @@ def returner(ret): return if raven_config.get('dsn'): - client = Client(raven_config.get('dsn'), transport=HTTPTransport) + client = Client(raven_config.get('dsn')) else: try: servers = [] @@ -140,8 +139,7 @@ def returner(ret): servers=servers, public_key=raven_config['public_key'], secret_key=raven_config['secret_key'], - project=raven_config['project'], - transport=HTTPTransport + project=raven_config['project'] ) except KeyError as missing_key: logger.error( @@ -172,4 +170,4 @@ def prep_jid(nocache=False, passed_jid=None): # pylint: disable=unused-argument ''' Do any work necessary to prepare a JID, including sending a custom id ''' - return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid() + return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid(__opts__) diff --git a/salt/returners/smtp_return.py b/salt/returners/smtp_return.py index 0dc904ebb0b..f630aa18f6d 100644 --- a/salt/returners/smtp_return.py +++ b/salt/returners/smtp_return.py @@ -113,7 +113,7 @@ import smtplib from email.utils import formatdate # Import Salt libs -import salt.ext.six as six +from salt.ext import six import salt.utils.jid import salt.returners import salt.loader @@ -264,7 +264,7 @@ def prep_jid(nocache=False, passed_jid=None): # pylint: disable=unused-argument ''' Do any work necessary to prepare a JID, including sending a custom id ''' - return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid() + return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid(__opts__) def event_return(events): diff --git a/salt/returners/sqlite3_return.py b/salt/returners/sqlite3_return.py index 55c6c0b36be..1b2159980d1 100644 --- a/salt/returners/sqlite3_return.py +++ b/salt/returners/sqlite3_return.py @@ -303,4 +303,4 @@ def prep_jid(nocache=False, passed_jid=None): # pylint: disable=unused-argument ''' Do any work necessary to prepare a JID, including sending a custom id ''' - return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid() + return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid(__opts__) diff --git a/salt/returners/syslog_return.py b/salt/returners/syslog_return.py index 67adbaa5099..f963c751d8c 100644 --- a/salt/returners/syslog_return.py +++ b/salt/returners/syslog_return.py @@ -100,7 +100,7 @@ except ImportError: # Import Salt libs import salt.utils.jid import salt.returners -import salt.ext.six as six +from salt.ext import six log = logging.getLogger(__name__) # Define the module's virtual name @@ -213,4 +213,4 @@ def prep_jid(nocache=False, ''' Do any work necessary to prepare a JID, including sending a custom id ''' - return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid() + return passed_jid if passed_jid is not None else salt.utils.jid.gen_jid(__opts__) diff --git a/salt/returners/zabbix_return.py b/salt/returners/zabbix_return.py index 356be1bd583..6470fe31ff1 100644 --- a/salt/returners/zabbix_return.py +++ b/salt/returners/zabbix_return.py @@ -25,7 +25,7 @@ import logging import os # Import Salt libs -import salt.ext.six as six +from salt.ext import six # Get logging started log = logging.getLogger(__name__) diff --git a/salt/roster/ansible.py b/salt/roster/ansible.py index 01aea562860..d97b1068841 100644 --- a/salt/roster/ansible.py +++ b/salt/roster/ansible.py @@ -96,11 +96,13 @@ import json import subprocess # Import Salt libs -import salt.utils +import salt.utils.args +import salt.utils.files +import salt.utils.stringutils from salt.roster import get_roster_file # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six CONVERSION = { 'ansible_ssh_host': 'host', @@ -183,7 +185,7 @@ class Inventory(Target): blocks = re.compile(r'^\[.*\]$') hostvar = re.compile(r'^\[([^:]+):vars\]$') parents = re.compile(r'^\[([^:]+):children\]$') - with salt.utils.fopen(inventory_file) as config: + with salt.utils.files.fopen(inventory_file) as config: for line in config.read().split('\n'): if not line or line.startswith('#'): continue @@ -206,7 +208,7 @@ class Inventory(Target): ''' Parse lines in the inventory file that are under the same group block ''' - line_args = salt.utils.shlex_split(line) + line_args = salt.utils.args.shlex_split(line) name = line_args[0] host = {line_args[0]: dict()} for arg in line_args[1:]: @@ -245,7 +247,7 @@ class Script(Target): self.tgt = tgt self.tgt_type = tgt_type inventory, error = subprocess.Popen([inventory_file], shell=True, stdout=subprocess.PIPE).communicate() - self.inventory = json.loads(salt.utils.to_str(inventory)) + self.inventory = json.loads(salt.utils.stringutils.to_str(inventory)) self.meta = self.inventory.get('_meta', {}) self.groups = dict() self.hostvars = dict() diff --git a/salt/roster/cache.py b/salt/roster/cache.py index fb848507ff0..bdb1cc81711 100644 --- a/salt/roster/cache.py +++ b/salt/roster/cache.py @@ -101,9 +101,10 @@ import re # Salt libs import salt.utils.minions +import salt.utils.versions import salt.cache from salt._compat import ipaddress -import salt.ext.six as six +from salt.ext import six log = logging.getLogger(__name__) @@ -116,7 +117,8 @@ def targets(tgt, tgt_type='glob', **kwargs): # pylint: disable=W0613 The resulting roster can be configured using ``roster_order`` and ``roster_default``. ''' minions = salt.utils.minions.CkMinions(__opts__) - minions = minions.check_minions(tgt, tgt_type) + _res = minions.check_minions(tgt, tgt_type) + minions = _res['minions'] ret = {} if not minions: @@ -129,17 +131,21 @@ def targets(tgt, tgt_type='glob', **kwargs): # pylint: disable=W0613 'host': ('ipv6-private', 'ipv6-global', 'ipv4-private', 'ipv4-public') }) if isinstance(roster_order, (tuple, list)): - salt.utils.warn_until('Oxygen', - 'Using legacy syntax for roster_order') + salt.utils.versions.warn_until( + 'Fluorine', + 'Using legacy syntax for roster_order' + ) roster_order = { 'host': roster_order } for config_key, order in roster_order.items(): for idx, key in enumerate(order): if key in ('public', 'private', 'local'): - salt.utils.warn_until('Oxygen', - 'roster_order {0} will include IPv6 soon. ' - 'Set order to ipv4-{0} if needed.'.format(key)) + salt.utils.versions.warn_until( + 'Fluorine', + 'roster_order {0} will include IPv6 soon. ' + 'Set order to ipv4-{0} if needed.'.format(key) + ) order[idx] = 'ipv4-' + key # log.debug(roster_order) diff --git a/salt/roster/flat.py b/salt/roster/flat.py index 9d5af1c333d..e470aea9076 100644 --- a/salt/roster/flat.py +++ b/salt/roster/flat.py @@ -19,6 +19,7 @@ except ImportError: # Import Salt libs import salt.loader +import salt.config from salt.template import compile_template from salt.ext.six import string_types from salt.roster import get_roster_file @@ -43,7 +44,7 @@ def targets(tgt, tgt_type='glob', **kwargs): **kwargs) conditioned_raw = {} for minion in raw: - conditioned_raw[str(minion)] = raw[minion] + conditioned_raw[str(minion)] = salt.config.apply_sdb(raw[minion]) rmatcher = RosterMatcher(conditioned_raw, tgt, tgt_type, 'ipv4') return rmatcher.targets() diff --git a/salt/roster/sshconfig.py b/salt/roster/sshconfig.py new file mode 100644 index 00000000000..29c01ccaf32 --- /dev/null +++ b/salt/roster/sshconfig.py @@ -0,0 +1,146 @@ +# -*- coding: utf-8 -*- +''' +Parses roster entries out of Host directives from SSH config + +.. code-block:: bash + + salt-ssh --roster sshconfig '*' -r "echo hi" +''' +from __future__ import absolute_import + +# Import python libs +import os +import collections +import fnmatch +import re + +# Import Salt libs +import salt.utils +from salt.ext.six import string_types + +import logging +log = logging.getLogger(__name__) + +_SSHConfRegex = collections.namedtuple('_SSHConfRegex', ['target_field', 'pattern']) +_ROSTER_FIELDS = ( + _SSHConfRegex(target_field='user', pattern=r'\s+User (.*)'), + _SSHConfRegex(target_field='port', pattern=r'\s+Port (.*)'), + _SSHConfRegex(target_field='priv', pattern=r'\s+IdentityFile (.*)'), +) + + +def _get_ssh_config_file(opts): + ''' + :return: Path to the .ssh/config file - usually /.ssh/config + ''' + ssh_config_file = opts.get('ssh_config_file') + if not os.path.isfile(ssh_config_file): + raise IOError('Cannot find SSH config file') + if not os.access(ssh_config_file, os.R_OK): + raise IOError('Cannot access SSH config file: {}'.format(ssh_config_file)) + return ssh_config_file + + +def parse_ssh_config(lines): + ''' + Parses lines from the SSH config to create roster targets. + + :param lines: Individual lines from the ssh config file + :return: Dictionary of targets in similar style to the flat roster + ''' + # transform the list of individual lines into a list of sublists where each + # sublist represents a single Host definition + hosts = [] + for line in lines: + if not line or line.startswith('#'): + continue + elif line.startswith('Host '): + hosts.append([]) + hosts[-1].append(line) + + # construct a dictionary of Host names to mapped roster properties + targets = collections.OrderedDict() + for host_data in hosts: + target = collections.OrderedDict() + hostnames = host_data[0].split()[1:] + for line in host_data[1:]: + for field in _ROSTER_FIELDS: + match = re.match(field.pattern, line) + if match: + target[field.target_field] = match.group(1) + for hostname in hostnames: + targets[hostname] = target + + # apply matching for glob hosts + wildcard_targets = [] + non_wildcard_targets = [] + for target in targets.keys(): + if '*' in target or '?' in target: + wildcard_targets.append(target) + else: + non_wildcard_targets.append(target) + for pattern in wildcard_targets: + for candidate in non_wildcard_targets: + if fnmatch.fnmatch(candidate, pattern): + targets[candidate].update(targets[pattern]) + del targets[pattern] + + # finally, update the 'host' to refer to its declaration in the SSH config + # so that its connection parameters can be utilized + for target in targets: + targets[target]['host'] = target + return targets + + +def targets(tgt, tgt_type='glob', **kwargs): + ''' + Return the targets from the flat yaml file, checks opts for location but + defaults to /etc/salt/roster + ''' + ssh_config_file = _get_ssh_config_file(__opts__) + with salt.utils.fopen(ssh_config_file, 'r') as fp: + all_minions = parse_ssh_config([line.rstrip() for line in fp]) + rmatcher = RosterMatcher(all_minions, tgt, tgt_type) + matched = rmatcher.targets() + return matched + + +class RosterMatcher(object): + ''' + Matcher for the roster data structure + ''' + def __init__(self, raw, tgt, tgt_type): + self.tgt = tgt + self.tgt_type = tgt_type + self.raw = raw + + def targets(self): + ''' + Execute the correct tgt_type routine and return + ''' + try: + return getattr(self, 'ret_{0}_minions'.format(self.tgt_type))() + except AttributeError: + return {} + + def ret_glob_minions(self): + ''' + Return minions that match via glob + ''' + minions = {} + for minion in self.raw: + if fnmatch.fnmatch(minion, self.tgt): + data = self.get_data(minion) + if data: + minions[minion] = data + return minions + + def get_data(self, minion): + ''' + Return the configured ip + ''' + if isinstance(self.raw[minion], string_types): + return {'host': self.raw[minion]} + if isinstance(self.raw[minion], dict): + return self.raw[minion] + return False diff --git a/salt/runner.py b/salt/runner.py index 3651e3c9600..fa71e520ee5 100644 --- a/salt/runner.py +++ b/salt/runner.py @@ -12,9 +12,10 @@ import logging import salt.exceptions import salt.loader import salt.minion -import salt.utils +import salt.utils # Can be removed when get_specific_user is moved import salt.utils.args import salt.utils.event +import salt.utils.files from salt.client import mixins from salt.output import display_output from salt.utils.lazy import verify_fun @@ -37,16 +38,16 @@ class RunnerClient(mixins.SyncClientMixin, mixins.AsyncClientMixin, object): eauth user must be authorized to execute runner modules: (``@runner``). Only the :py:meth:`master_call` below supports eauth. ''' - client = 'runner' - tag_prefix = 'run' + client = u'runner' + tag_prefix = u'run' def __init__(self, opts): self.opts = opts @property def functions(self): - if not hasattr(self, '_functions'): - if not hasattr(self, 'utils'): + if not hasattr(self, u'_functions'): + if not hasattr(self, u'utils'): self.utils = salt.loader.utils(self.opts) # Must be self.functions for mixin to work correctly :-/ try: @@ -69,29 +70,38 @@ class RunnerClient(mixins.SyncClientMixin, mixins.AsyncClientMixin, object): New-style: ``{'fun': 'jobs.lookup_jid', 'kwarg': {'jid': '1234'}}`` CLI-style: ``{'fun': 'jobs.lookup_jid', 'arg': ['jid="1234"']}`` ''' - fun = low.pop('fun') + fun = low.pop(u'fun') verify_fun(self.functions, fun) - reserved_kwargs = dict([(i, low.pop(i)) for i in [ - 'username', 'password', 'eauth', 'token', 'client', 'user', 'key', - '__current_eauth_groups', '__current_eauth_user', + eauth_creds = dict([(i, low.pop(i)) for i in [ + u'username', u'password', u'eauth', u'token', u'client', u'user', u'key', ] if i in low]) # Run name=value args through parse_input. We don't need to run kwargs # through because there is no way to send name=value strings in the low # dict other than by including an `arg` array. - arg, kwarg = salt.utils.args.parse_input( - low.pop('arg', []), - condition=False, - no_parse=self.opts.get('no_parse', [])) - kwarg.update(low.pop('kwarg', {})) + _arg, _kwarg = salt.utils.args.parse_input( + low.pop(u'arg', []), condition=False) + _kwarg.update(low.pop(u'kwarg', {})) # If anything hasn't been pop()'ed out of low by this point it must be # an old-style kwarg. - kwarg.update(low) + _kwarg.update(low) - return dict(fun=fun, kwarg={'kwarg': kwarg, 'arg': arg}, - **reserved_kwargs) + # Finally, mung our kwargs to a format suitable for the byzantine + # load_args_and_kwargs so that we can introspect the function being + # called and fish for invalid kwargs. + munged = [] + munged.extend(_arg) + munged.append(dict(__kwarg__=True, **_kwarg)) + arg, kwarg = salt.minion.load_args_and_kwargs( + self.functions[fun], + munged, + self.opts, + ignore_invalid=True) + + return dict(fun=fun, kwarg={u'kwarg': kwarg, u'arg': arg}, + **eauth_creds) def cmd_async(self, low): ''' @@ -157,10 +167,10 @@ class Runner(RunnerClient): ''' Print out the documentation! ''' - arg = self.opts.get('fun', None) + arg = self.opts.get(u'fun', None) docs = super(Runner, self).get_docs(arg) for fun in sorted(docs): - display_output('{0}:'.format(fun), 'text', self.opts) + display_output(u'{0}:'.format(fun), u'text', self.opts) print(docs[fun]) # TODO: move to mixin whenever we want a salt-wheel cli @@ -170,106 +180,117 @@ class Runner(RunnerClient): ''' import salt.minion ret = {} - if self.opts.get('doc', False): + if self.opts.get(u'doc', False): self.print_docs() else: - low = {'fun': self.opts['fun']} + low = {u'fun': self.opts[u'fun']} try: # Allocate a jid async_pub = self._gen_async_pub() - self.jid = async_pub['jid'] + self.jid = async_pub[u'jid'] fun_args = salt.utils.args.parse_input( - self.opts['arg'], - no_parse=self.opts.get('no_parse', [])) + self.opts[u'arg'], + no_parse=self.opts.get(u'no_parse', [])) - verify_fun(self.functions, low['fun']) + verify_fun(self.functions, low[u'fun']) args, kwargs = salt.minion.load_args_and_kwargs( - self.functions[low['fun']], + self.functions[low[u'fun']], fun_args, self.opts, ) - low['arg'] = args - low['kwarg'] = kwargs + low[u'arg'] = args + low[u'kwarg'] = kwargs - if self.opts.get('eauth'): - if 'token' in self.opts: + if self.opts.get(u'eauth'): + if u'token' in self.opts: try: - with salt.utils.fopen(os.path.join(self.opts['cachedir'], '.root_key'), 'r') as fp_: - low['key'] = fp_.readline() + with salt.utils.files.fopen(os.path.join(self.opts[u'cachedir'], u'.root_key'), u'r') as fp_: + low[u'key'] = fp_.readline() except IOError: - low['token'] = self.opts['token'] + low[u'token'] = self.opts[u'token'] # If using eauth and a token hasn't already been loaded into # low, prompt the user to enter auth credentials - if 'token' not in low and 'key' not in low and self.opts['eauth']: + if u'token' not in low and u'key' not in low and self.opts[u'eauth']: # This is expensive. Don't do it unless we need to. import salt.auth resolver = salt.auth.Resolver(self.opts) - res = resolver.cli(self.opts['eauth']) - if self.opts['mktoken'] and res: + res = resolver.cli(self.opts[u'eauth']) + if self.opts[u'mktoken'] and res: tok = resolver.token_cli( - self.opts['eauth'], + self.opts[u'eauth'], res ) if tok: - low['token'] = tok.get('token', '') + low[u'token'] = tok.get(u'token', u'') if not res: - log.error('Authentication failed') + log.error(u'Authentication failed') return ret low.update(res) - low['eauth'] = self.opts['eauth'] + low[u'eauth'] = self.opts[u'eauth'] else: user = salt.utils.get_specific_user() - if low['fun'] == 'state.orchestrate': - low['kwarg']['orchestration_jid'] = async_pub['jid'] + if low[u'fun'] == u'state.orchestrate': + low[u'kwarg'][u'orchestration_jid'] = async_pub[u'jid'] # Run the runner! - if self.opts.get('async', False): - if self.opts.get('eauth'): + if self.opts.get(u'async', False): + if self.opts.get(u'eauth'): async_pub = self.cmd_async(low) else: - async_pub = self.async(self.opts['fun'], + async_pub = self.async(self.opts[u'fun'], low, user=user, pub=async_pub) # by default: info will be not enougth to be printed out ! - log.warning('Running in async mode. Results of this execution may ' - 'be collected by attaching to the master event bus or ' - 'by examing the master job cache, if configured. ' - 'This execution is running under tag {tag}'.format(**async_pub)) - return async_pub['jid'] # return the jid + log.warning( + u'Running in async mode. Results of this execution may ' + u'be collected by attaching to the master event bus or ' + u'by examing the master job cache, if configured. ' + u'This execution is running under tag %s', async_pub[u'tag'] + ) + return async_pub[u'jid'] # return the jid # otherwise run it in the main process - if self.opts.get('eauth'): + if self.opts.get(u'eauth'): ret = self.cmd_sync(low) - if isinstance(ret, dict) and set(ret) == set(('data', 'outputter')): - outputter = ret['outputter'] - ret = ret['data'] + if isinstance(ret, dict) and set(ret) == set((u'data', u'outputter')): + outputter = ret[u'outputter'] + ret = ret[u'data'] else: outputter = None display_output(ret, outputter, self.opts) else: - ret = self._proc_function(self.opts['fun'], + ret = self._proc_function(self.opts[u'fun'], low, user, - async_pub['tag'], - async_pub['jid'], + async_pub[u'tag'], + async_pub[u'jid'], daemonize=False) except salt.exceptions.SaltException as exc: - evt = salt.utils.event.get_event('master', opts=self.opts) - evt.fire_event({'success': False, - 'return': "{0}".format(exc), - 'retcode': 254, - 'fun': self.opts['fun'], - 'fun_args': fun_args, - 'jid': self.jid}, - tag='salt/run/{0}/ret'.format(self.jid)) - ret = '{0}'.format(exc) - if not self.opts.get('quiet', False): - display_output(ret, 'nested', self.opts) + evt = salt.utils.event.get_event(u'master', opts=self.opts) + evt.fire_event({u'success': False, + u'return': u'{0}'.format(exc), + u'retcode': 254, + u'fun': self.opts[u'fun'], + u'fun_args': fun_args, + u'jid': self.jid}, + tag=u'salt/run/{0}/ret'.format(self.jid)) + # Attempt to grab documentation + if u'fun' in low: + ret = self.get_docs(u'{0}*'.format(low[u'fun'])) + else: + ret = None + + # If we didn't get docs returned then + # return the `not availble` message. + if not ret: + ret = u'{0}'.format(exc) + if not self.opts.get(u'quiet', False): + display_output(ret, u'nested', self.opts) else: - log.debug('Runner return: {0}'.format(ret)) + log.debug(u'Runner return: %s', ret) return ret diff --git a/salt/runners/auth.py b/salt/runners/auth.py index 8aec8c33893..7e8e64855a4 100644 --- a/salt/runners/auth.py +++ b/salt/runners/auth.py @@ -17,16 +17,28 @@ import salt.netapi def mk_token(**load): - ''' + r''' Create an eauth token using provided credentials + Non-root users may specify an expiration date -- if allowed via the + :conf_master:`token_expire_user_override` setting -- by passing an + additional ``token_expire`` param. This overrides the + :conf_master:`token_expire` setting of the same name in the Master config + and is how long a token should live in seconds. + CLI Example: .. code-block:: shell salt-run auth.mk_token username=saltdev password=saltdev eauth=auto - salt-run auth.mk_token username=saltdev password=saltdev eauth=auto \\ + + # Create a token valid for three years. + salt-run auth.mk_token username=saltdev password=saltdev eauth=auto \ token_expire=94670856 + + # Calculate the number of seconds using expr. + salt-run auth.mk_token username=saltdev password=saltdev eauth=auto \ + token_expire=$(expr \( 365 \* 24 \* 60 \* 60 \) \* 3) ''' # This will hang if the master daemon is not running. netapi = salt.netapi.NetapiClient(__opts__) diff --git a/salt/runners/cache.py b/salt/runners/cache.py index 8260b074df6..af074e4aba0 100644 --- a/salt/runners/cache.py +++ b/salt/runners/cache.py @@ -10,10 +10,12 @@ import os # Import salt libs import salt.config -import salt.ext.six as six +from salt.ext import six import salt.log import salt.utils +import salt.utils.args import salt.utils.master +import salt.utils.versions import salt.payload import salt.cache from salt.exceptions import SaltInvocationError @@ -45,7 +47,7 @@ def grains(tgt=None, tgt_type='glob', **kwargs): salt-run cache.grains ''' if 'expr_form' in kwargs: - salt.utils.warn_until( + salt.utils.versions.warn_until( 'Fluorine', 'The target type should be passed using the \'tgt_type\' ' 'argument instead of \'expr_form\'. Support for using ' @@ -76,7 +78,7 @@ def pillar(tgt=None, tgt_type='glob', **kwargs): salt-run cache.pillar ''' if 'expr_form' in kwargs: - salt.utils.warn_until( + salt.utils.versions.warn_until( 'Fluorine', 'The target type should be passed using the \'tgt_type\' ' 'argument instead of \'expr_form\'. Support for using ' @@ -109,7 +111,7 @@ def mine(tgt=None, tgt_type='glob', **kwargs): salt-run cache.mine ''' if 'expr_form' in kwargs: - salt.utils.warn_until( + salt.utils.versions.warn_until( 'Fluorine', 'The target type should be passed using the \'tgt_type\' ' 'argument instead of \'expr_form\'. Support for using ' @@ -167,7 +169,7 @@ def clear_pillar(tgt=None, tgt_type='glob', expr_form=None): # remember to remove the expr_form argument from this function when # performing the cleanup on this deprecation. if expr_form is not None: - salt.utils.warn_until( + salt.utils.versions.warn_until( 'Fluorine', 'the target type should be passed using the \'tgt_type\' ' 'argument instead of \'expr_form\'. Support for using ' @@ -195,7 +197,7 @@ def clear_grains(tgt=None, tgt_type='glob', expr_form=None): # remember to remove the expr_form argument from this function when # performing the cleanup on this deprecation. if expr_form is not None: - salt.utils.warn_until( + salt.utils.versions.warn_until( 'Fluorine', 'the target type should be passed using the \'tgt_type\' ' 'argument instead of \'expr_form\'. Support for using ' @@ -223,7 +225,7 @@ def clear_mine(tgt=None, tgt_type='glob', expr_form=None): # remember to remove the expr_form argument from this function when # performing the cleanup on this deprecation. if expr_form is not None: - salt.utils.warn_until( + salt.utils.versions.warn_until( 'Fluorine', 'the target type should be passed using the \'tgt_type\' ' 'argument instead of \'expr_form\'. Support for using ' @@ -271,7 +273,7 @@ def clear_all(tgt=None, tgt_type='glob', expr_form=None): # remember to remove the expr_form argument from this function when # performing the cleanup on this deprecation. if expr_form is not None: - salt.utils.warn_until( + salt.utils.versions.warn_until( 'Fluorine', 'the target type should be passed using the \'tgt_type\' ' 'argument instead of \'expr_form\'. Support for using ' @@ -321,10 +323,10 @@ def clear_git_lock(role, remote=None, **kwargs): salt-run cache.clear_git_lock git_pillar ''' - kwargs = salt.utils.clean_kwargs(**kwargs) + kwargs = salt.utils.args.clean_kwargs(**kwargs) type_ = salt.utils.split_input(kwargs.pop('type', ['update', 'checkout'])) if kwargs: - salt.utils.invalid_kwargs(kwargs) + salt.utils.args.invalid_kwargs(kwargs) if role == 'gitfs': git_objects = [salt.utils.gitfs.GitFS(__opts__)] diff --git a/salt/runners/cloud.py b/salt/runners/cloud.py index b00d843c680..eb46d4afa5d 100644 --- a/salt/runners/cloud.py +++ b/salt/runners/cloud.py @@ -14,16 +14,13 @@ import os # Import Salt libs import salt.cloud +import salt.utils.args from salt.exceptions import SaltCloudConfigError # Get logging started log = logging.getLogger(__name__) -def _filter_kwargs(k): - return dict((x, k[x]) for x in k if not x.startswith('__')) - - def _get_client(): ''' Return cloud client @@ -114,7 +111,7 @@ def profile(prof=None, instances=None, opts=None, **kwargs): client = _get_client() if isinstance(opts, dict): client.opts.update(opts) - info = client.profile(prof, instances, **_filter_kwargs(kwargs)) + info = client.profile(prof, instances, **salt.utils.args.clean_kwargs(**kwargs)) return info @@ -123,7 +120,7 @@ def map_run(path=None, **kwargs): Execute a salt cloud map file ''' client = _get_client() - info = client.map_run(path, **_filter_kwargs(kwargs)) + info = client.map_run(path, **salt.utils.args.clean_kwargs(**kwargs)) return info @@ -154,7 +151,14 @@ def action(func=None, info = {} client = _get_client() try: - info = client.action(func, cloudmap, instances, provider, instance, **_filter_kwargs(kwargs)) + info = client.action( + func, + cloudmap, + instances, + provider, + instance, + **salt.utils.args.clean_kwargs(**kwargs) + ) except SaltCloudConfigError as err: log.error(err) return info @@ -172,12 +176,8 @@ def create(provider, instances, opts=None, **kwargs): image=ami-1624987f size='t1.micro' ssh_username=ec2-user \ securitygroup=default delvol_on_destroy=True ''' - create_kwargs = {} - for kwarg in kwargs: - if not kwarg.startswith('__'): - create_kwargs[kwarg] = kwargs[kwarg] client = _get_client() if isinstance(opts, dict): client.opts.update(opts) - info = client.create(provider, instances, **create_kwargs) + info = client.create(provider, instances, **salt.utils.args.clean_kwargs(**kwargs)) return info diff --git a/salt/runners/ddns.py b/salt/runners/ddns.py index 3bb2930a1f4..6ec805679c6 100644 --- a/salt/runners/ddns.py +++ b/salt/runners/ddns.py @@ -28,7 +28,7 @@ except ImportError: HAS_LIBS = False # Import salt libs -import salt.utils +import salt.utils.files log = logging.getLogger(__name__) @@ -47,7 +47,7 @@ def __virtual__(): def _get_keyring(keyfile): keyring = None if keyfile and os.path.isfile(os.path.expanduser(keyfile)): - with salt.utils.fopen(keyfile) as _f: + with salt.utils.files.fopen(keyfile) as _f: keyring = dns.tsigkeyring.from_text(json.load(_f)) return keyring diff --git a/salt/runners/digicertapi.py b/salt/runners/digicertapi.py index 9646c7ef5ee..70f5cd20277 100644 --- a/salt/runners/digicertapi.py +++ b/salt/runners/digicertapi.py @@ -45,9 +45,9 @@ import json import re import salt.syspaths as syspaths import salt.cache -import salt.utils +import salt.utils.files import salt.utils.http -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import range from salt.exceptions import (CommandExecutionError, SaltRunnerError) from Crypto.PublicKey import RSA @@ -568,7 +568,7 @@ def gen_csr( tmppriv = '{0}/priv'.format(tmpdir) tmpcsr = '{0}/csr'.format(tmpdir) - with salt.utils.fopen(tmppriv, 'w') as if_: + with salt.utils.files.fopen(tmppriv, 'w') as if_: if_.write(data['private_key']) subject = '/C={0}/ST={1}/L={2}/O={3}'.format( @@ -596,7 +596,7 @@ def gen_csr( 'have a valid Organization established inside CertCentral' ) - with salt.utils.fopen(tmpcsr, 'r') as of_: + with salt.utils.files.fopen(tmpcsr, 'r') as of_: csr = of_.read() data['minion_id'] = minion_id diff --git a/salt/runners/doc.py b/salt/runners/doc.py index d61e6b8cd82..92a06b3a443 100644 --- a/salt/runners/doc.py +++ b/salt/runners/doc.py @@ -13,7 +13,7 @@ import salt.runner import salt.wheel # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six from salt.exceptions import SaltClientError diff --git a/salt/runners/git_pillar.py b/salt/runners/git_pillar.py index 8dfc4412eb8..6826268076c 100644 --- a/salt/runners/git_pillar.py +++ b/salt/runners/git_pillar.py @@ -11,7 +11,6 @@ import logging import salt.pillar.git_pillar import salt.utils.gitfs from salt.exceptions import SaltRunnerError -from salt.ext import six log = logging.getLogger(__name__) @@ -21,18 +20,18 @@ def update(branch=None, repo=None): .. versionadded:: 2014.1.0 .. versionchanged:: 2015.8.4 - This runner function now supports the :ref:`new git_pillar - configuration schema ` introduced in + This runner function now supports the :ref:`git_pillar + configuration schema ` introduced in 2015.8.0. Additionally, the branch and repo can now be omitted to - update all git_pillar remotes. The return data has also changed. For - releases 2015.8.3 and earlier, there is no value returned. Starting - with 2015.8.4, the return data is a dictionary. If using the :ref:`old - git_pillar configuration schema `, then the - dictionary values will be ``True`` if the update completed without - error, and ``False`` if an error occurred. If using the :ref:`new - git_pillar configuration schema `, the - values will be ``True`` only if new commits were fetched, and ``False`` - if there were errors or no new commits were fetched. + update all git_pillar remotes. The return data has also changed to + a dictionary. The values will be ``True`` only if new commits were + fetched, and ``False`` if there were errors or no new commits were + fetched. + + .. versionchanged:: Oxygen + The return for a given git_pillar remote will now be ``None`` when no + changes were fetched. ``False`` now is reserved only for instances in + which there were errors. Fetch one or all configured git_pillar remotes. @@ -56,7 +55,7 @@ def update(branch=None, repo=None): # Update specific branch and repo salt-run git_pillar.update branch='branch' repo='https://foo.com/bar.git' - # Update all repos (2015.8.4 and later) + # Update all repos salt-run git_pillar.update # Run with debug logging salt-run git_pillar.update -l debug @@ -67,46 +66,30 @@ def update(branch=None, repo=None): if pillar_type != 'git': continue pillar_conf = ext_pillar[pillar_type] - if isinstance(pillar_conf, six.string_types): - parts = pillar_conf.split() - if len(parts) >= 2: - desired_branch, desired_repo = parts[:2] - # Skip this remote if it doesn't match the search criteria - if branch is not None: - if branch != desired_branch: - continue - if repo is not None: - if repo != desired_repo: - continue - ret[pillar_conf] = salt.pillar.git_pillar._LegacyGitPillar( - parts[0], - parts[1], - __opts__).update() - - else: - pillar = salt.utils.gitfs.GitPillar(__opts__) - pillar.init_remotes(pillar_conf, - salt.pillar.git_pillar.PER_REMOTE_OVERRIDES) - for remote in pillar.remotes: - # Skip this remote if it doesn't match the search criteria - if branch is not None: - if branch != remote.branch: - continue - if repo is not None: - if repo != remote.url: - continue - try: - result = remote.fetch() - except Exception as exc: - log.error( - 'Exception \'{0}\' caught while fetching git_pillar ' - 'remote \'{1}\''.format(exc, remote.id), - exc_info_on_loglevel=logging.DEBUG - ) - result = False - finally: - remote.clear_lock() - ret[remote.id] = result + pillar = salt.utils.gitfs.GitPillar(__opts__) + pillar.init_remotes(pillar_conf, + salt.pillar.git_pillar.PER_REMOTE_OVERRIDES, + salt.pillar.git_pillar.PER_REMOTE_ONLY) + for remote in pillar.remotes: + # Skip this remote if it doesn't match the search criteria + if branch is not None: + if branch != remote.branch: + continue + if repo is not None: + if repo != remote.url: + continue + try: + result = remote.fetch() + except Exception as exc: + log.error( + 'Exception \'{0}\' caught while fetching git_pillar ' + 'remote \'{1}\''.format(exc, remote.id), + exc_info_on_loglevel=logging.DEBUG + ) + result = False + finally: + remote.clear_lock() + ret[remote.id] = result if not ret: if branch is not None or repo is not None: diff --git a/salt/runners/http.py b/salt/runners/http.py index b30575a1dfa..7104d612b8f 100644 --- a/salt/runners/http.py +++ b/salt/runners/http.py @@ -35,8 +35,12 @@ def query(url, output=True, **kwargs): log.warning('Output option has been deprecated. Please use --quiet.') if 'node' not in kwargs: kwargs['node'] = 'master' + opts = __opts__.copy() + if 'opts' in kwargs: + opts.update(kwargs['opts']) + del kwargs['opts'] - ret = salt.utils.http.query(url=url, opts=__opts__, **kwargs) + ret = salt.utils.http.query(url=url, opts=opts, **kwargs) return ret diff --git a/salt/runners/jobs.py b/salt/runners/jobs.py index 82abd56eae2..65f11580490 100644 --- a/salt/runners/jobs.py +++ b/salt/runners/jobs.py @@ -13,12 +13,13 @@ import os import salt.client import salt.payload import salt.utils +import salt.utils.files import salt.utils.jid import salt.minion import salt.returners # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six from salt.exceptions import SaltClientError try: @@ -577,13 +578,13 @@ def _walk_through(job_dir, display_progress=False): for final in os.listdir(t_path): load_path = os.path.join(t_path, final, '.load.p') - with salt.utils.fopen(load_path, 'rb') as rfh: + with salt.utils.files.fopen(load_path, 'rb') as rfh: job = serial.load(rfh) if not os.path.isfile(load_path): continue - with salt.utils.fopen(load_path, 'rb') as rfh: + with salt.utils.files.fopen(load_path, 'rb') as rfh: job = serial.load(rfh) jid = job['jid'] if display_progress: diff --git a/salt/runners/lxc.py b/salt/runners/lxc.py index 95046a3cef6..bd1f86f971c 100644 --- a/salt/runners/lxc.py +++ b/salt/runners/lxc.py @@ -14,14 +14,15 @@ import logging # Import Salt libs import salt.client -import salt.utils -import salt.utils.virt +import salt.utils.args import salt.utils.cloud +import salt.utils.files +import salt.utils.virt import salt.key from salt.utils.odict import OrderedDict as _OrderedDict # Import 3rd-party lib -import salt.ext.six as six +from salt.ext import six log = logging.getLogger(__name__) @@ -275,7 +276,7 @@ def init(names, host=None, saltcloud_mode=False, quiet=False, **kwargs): ret['result'] = False return ret - kw = salt.utils.clean_kwargs(**kwargs) + kw = salt.utils.args.clean_kwargs(**kwargs) pub_key = kw.get('pub_key', None) priv_key = kw.get('priv_key', None) explicit_auth = pub_key and priv_key @@ -308,7 +309,7 @@ def init(names, host=None, saltcloud_mode=False, quiet=False, **kwargs): cmds = [] for name in names: args = [name] - kw = salt.utils.clean_kwargs(**kwargs) + kw = salt.utils.args.clean_kwargs(**kwargs) if saltcloud_mode: kw = copy.deepcopy(kw) kw['name'] = name @@ -372,10 +373,10 @@ def init(names, host=None, saltcloud_mode=False, quiet=False, **kwargs): if explicit_auth: fcontent = '' if os.path.exists(key): - with salt.utils.fopen(key) as fic: + with salt.utils.files.fopen(key) as fic: fcontent = fic.read().strip() if pub_key.strip() != fcontent: - with salt.utils.fopen(key, 'w') as fic: + with salt.utils.files.fopen(key, 'w') as fic: fic.write(pub_key) fic.flush() mid = j_ret.get('mid', None) diff --git a/salt/runners/manage.py b/salt/runners/manage.py index 249c875c572..e446d0e9854 100644 --- a/salt/runners/manage.py +++ b/salt/runners/manage.py @@ -16,13 +16,16 @@ import logging import uuid # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves.urllib.request import urlopen as _urlopen # pylint: disable=no-name-in-module,import-error # Import salt libs import salt.key -import salt.utils +import salt.utils.compat +import salt.utils.files import salt.utils.minions +import salt.utils.raetevent +import salt.utils.versions import salt.client import salt.client.ssh import salt.wheel @@ -85,7 +88,7 @@ def status(output=True, tgt='*', tgt_type='glob', expr_form=None, timeout=None, # remember to remove the expr_form argument from this function when # performing the cleanup on this deprecation. if expr_form is not None: - salt.utils.warn_until( + salt.utils.versions.warn_until( 'Fluorine', 'the target type should be passed using the \'tgt_type\' ' 'argument instead of \'expr_form\'. Support for using ' @@ -652,6 +655,7 @@ def versions(): return ret labels = { + -2: 'Minion offline', -1: 'Minion requires update', 0: 'Up to date', 1: 'Minion newer than master', @@ -663,12 +667,19 @@ def versions(): master_version = salt.version.__saltstack_version__ for minion in minions: - minion_version = salt.version.SaltStackVersion.parse(minions[minion]) - ver_diff = cmp(minion_version, master_version) + if not minions[minion]: + minion_version = False + ver_diff = -2 + else: + minion_version = salt.version.SaltStackVersion.parse(minions[minion]) + ver_diff = salt.utils.compat.cmp(minion_version, master_version) if ver_diff not in version_status: version_status[ver_diff] = {} - version_status[ver_diff][minion] = minion_version.string + if minion_version: + version_status[ver_diff][minion] = minion_version.string + else: + version_status[ver_diff][minion] = minion_version # Add version of Master to output version_status[2] = master_version.string @@ -685,7 +696,6 @@ def versions(): def bootstrap(version='develop', script=None, hosts='', - root_user=False, script_args='', roster='flat', ssh_user=None, @@ -706,14 +716,6 @@ def bootstrap(version='develop', Comma-separated hosts [example: hosts='host1.local,host2.local']. These hosts need to exist in the specified roster. - root_user : False - Prepend ``root@`` to each host. Default changed in Salt 2016.11.0 from ``True`` - to ``False``. - - .. versionchanged:: 2016.11.0 - - .. deprecated:: Oxygen - script_args Any additional arguments that you want to pass to the script. @@ -771,19 +773,8 @@ def bootstrap(version='develop', salt-run manage.bootstrap hosts='host1,host2' version='v0.17' salt-run manage.bootstrap hosts='host1,host2' version='v0.17' \ script='https://bootstrap.saltstack.com/develop' - salt-run manage.bootstrap hosts='ec2-user@host1,ec2-user@host2' \ - root_user=False ''' - dep_warning = ( - 'Starting with Salt 2016.11.0, manage.bootstrap now uses Salt SSH to ' - 'connect, and requires a roster entry. Please ensure that a roster ' - 'entry exists for this host. Non-roster hosts will no longer be ' - 'supported starting with Salt Oxygen.' - ) - if root_user is True: - salt.utils.warn_until('Oxygen', dep_warning) - if script is None: script = 'https://bootstrap.saltstack.com' @@ -824,21 +815,7 @@ def bootstrap(version='develop', client_opts['argv'] = ['file.remove', tmp_dir] salt.client.ssh.SSH(client_opts).run() except SaltSystemExit as exc: - if 'No hosts found with target' in str(exc): - log.warning('The host {0} was not found in the Salt SSH roster ' - 'system. Attempting to log in without Salt SSH.') - salt.utils.warn_until('Oxygen', dep_warning) - ret = subprocess.call([ - 'ssh', - ('root@' if root_user else '') + host, - 'python -c \'import urllib; ' - 'print urllib.urlopen(' - '"' + script + '"' - ').read()\' | sh -s -- git ' + version - ]) - return ret - else: - log.error(str(exc)) + log.error(str(exc)) def bootstrap_psexec(hosts='', master=None, version=None, arch='win32', @@ -966,7 +943,7 @@ objShell.Exec("{1}{2}")''' ' >>' + x + '.vbs\ncscript.exe /NoLogo ' + x + '.vbs' batch_path = tempfile.mkstemp(suffix='.bat')[1] - with salt.utils.fopen(batch_path, 'wb') as batch_file: + with salt.utils.files.fopen(batch_path, 'wb') as batch_file: batch_file.write(batch) for host in hosts.split(","): diff --git a/salt/runners/mattermost.py b/salt/runners/mattermost.py index 2bd3d928c46..4ca4e7125c3 100644 --- a/salt/runners/mattermost.py +++ b/salt/runners/mattermost.py @@ -20,7 +20,7 @@ import json import logging # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six # Import Salt libs # pylint: disable=import-error,no-name-in-module,redefined-builtin diff --git a/salt/runners/nacl.py b/salt/runners/nacl.py index 76bc20b1a2e..2ccd888bef1 100644 --- a/salt/runners/nacl.py +++ b/salt/runners/nacl.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- ''' -This runner helps create encrypted passwords that can be included in pillars. +This module helps include encrypted passwords in pillars, grains and salt state files. :depends: libnacl, https://github.com/saltstack/libnacl @@ -8,35 +8,166 @@ This is often useful if you wish to store your pillars in source control or share your pillar data with others that you trust. I don't advise making your pillars public regardless if they are encrypted or not. -The following configurations can be defined in the master config -so your users can create encrypted passwords using the runner nacl: +When generating keys and encrypting passwords use --local when using salt-call for extra +security. Also consider using just the salt runner nacl when encrypting pillar passwords. + +:configuration: The following configuration defaults can be + define (pillar or config files) Avoid storing private keys in pillars! Ensure master does not have `pillar_opts=True`: + + .. code-block:: python + + # cat /etc/salt/master.d/nacl.conf + nacl.config: + # NOTE: `key` and `key_file` have been renamed to `sk`, `sk_file` + # also `box_type` default changed from secretbox to sealedbox. + box_type: sealedbox (default) + sk_file: /etc/salt/pki/master/nacl (default) + pk_file: /etc/salt/pki/master/nacl.pub (default) + sk: None + pk: None + + Usage can override the config defaults: + + .. code-block:: bash + + salt-call nacl.enc sk_file=/etc/salt/pki/master/nacl pk_file=/etc/salt/pki/master/nacl.pub + + +The nacl lib uses 32byte keys, these keys are base64 encoded to make your life more simple. +To generate your `sk_file` and `pk_file` use: .. code-block:: bash - cat /etc/salt/master.d/nacl.conf + salt-call --local nacl.keygen sk_file=/etc/salt/pki/master/nacl + # or if you want to work without files. + salt-call --local nacl.keygen + local: + ---------- + pk: + /kfGX7PbWeu099702PBbKWLpG/9p06IQRswkdWHCDk0= + sk: + SVWut5SqNpuPeNzb1b9y6b2eXg2PLIog43GBzp48Sow= + +Now with your keypair, you can encrypt data: + +You have two option, `sealedbox` or `secretbox`. + +SecretBox is data encrypted using private key `pk`. Sealedbox is encrypted using public key `pk`. + +Recommend using Sealedbox because the one way encryption permits developers to encrypt data for source control but not decrypt. +Sealedbox only has one key that is for both encryption and decryption. + +.. code-block:: bash + + salt-call --local nacl.enc asecretpass pk=/kfGX7PbWeu099702PBbKWLpG/9p06IQRswkdWHCDk0= + tqXzeIJnTAM9Xf0mdLcpEdklMbfBGPj2oTKmlgrm3S1DTVVHNnh9h8mU1GKllGq/+cYsk6m5WhGdk58= + +To decrypt the data: + +.. code-block:: bash + + salt-call --local nacl.dec data='tqXzeIJnTAM9Xf0mdLcpEdklMbfBGPj2oTKmlgrm3S1DTVVHNnh9h8mU1GKllGq/+cYsk6m5WhGdk58=' \ + sk='SVWut5SqNpuPeNzb1b9y6b2eXg2PLIog43GBzp48Sow=' + +When the keys are defined in the master config you can use them from the nacl runner +without extra parameters: + +.. code-block:: python + + # cat /etc/salt/master.d/nacl.conf nacl.config: - key: 'cKEzd4kXsbeCE7/nLTIqXwnUiD1ulg4NoeeYcCFpd9k=' - keyfile: /root/.nacl - -Now with the config in the master you can use the runner nacl like: + sk_file: /etc/salt/pki/master/nacl + pk: 'cTIqXwnUiD1ulg4kXsbeCE7/NoeKEzd4nLeYcCFpd9k=' .. code-block:: bash - salt-run nacl.enc 'data' + salt-run nacl.enc 'asecretpass' + salt-run nacl.dec 'tqXzeIJnTAM9Xf0mdLcpEdklMbfBGPj2oTKmlgrm3S1DTVVHNnh9h8mU1GKllGq/+cYsk6m5WhGdk58=' + +.. code-block:: yam + # a salt developers minion could have pillar data that includes a nacl public key + nacl.config: + pk: '/kfGX7PbWeu099702PBbKWLpG/9p06IQRswkdWHCDk0=' + +The developer can then use a less-secure system to encrypt data. + +.. code-block:: bash + + salt-call --local nacl.enc apassword + + +Pillar files can include protected data that the salt master decrypts: + +.. code-block:: jinja + + pillarexample: + user: root + password1: {{salt.nacl.dec('DRB7Q6/X5gGSRCTpZyxS6hlbWj0llUA+uaVyvou3vJ4=')|json}} + cert_key: {{salt.nacl.dec_file('/srv/salt/certs/example.com/key.nacl')|json}} + cert_key2: {{salt.nacl.dec_file('salt:///certs/example.com/key.nacl')|json}} + +Larger files like certificates can be encrypted with: + +.. code-block:: bash + + salt-call nacl.enc_file /tmp/cert.crt out=/tmp/cert.nacl + # or more advanced + cert=$(cat /tmp/cert.crt) + salt-call --out=newline_values_only nacl.enc_pub data="$cert" > /tmp/cert.nacl + +In pillars rended with jinja be sure to include `|json` so line breaks are encoded: + +.. code-block:: jinja + + cert: "{{salt.nacl.dec('S2uogToXkgENz9...085KYt')|json}}" + +In states rendered with jinja it is also good pratice to include `|json`: + +.. code-block:: jinja + + {{sls}} private key: + file.managed: + - name: /etc/ssl/private/cert.key + - mode: 700 + - contents: "{{pillar['pillarexample']['cert_key']|json}}" + + +Optional small program to encrypt data without needing salt modules. + +.. code-block:: python + + #!/bin/python3 + import sys, base64, libnacl.sealed + pk = base64.b64decode('YOURPUBKEY') + b = libnacl.sealed.SealedBox(pk) + data = sys.stdin.buffer.read() + print(base64.b64encode(b.encrypt(data)).decode()) + +.. code-block:: bash + + echo 'apassword' | nacl_enc.py + ''' +# Import Python libs from __future__ import absolute_import import base64 import os -import salt.utils + +# Import Salt libs +import salt.utils.files +import salt.utils.platform +import salt.utils.win_functions +import salt.utils.win_dacl import salt.syspaths REQ_ERROR = None try: import libnacl.secret -except ImportError as e: - REQ_ERROR = 'libnacl import error, perhaps missing python libnacl package' + import libnacl.sealed +except (ImportError, OSError) as e: + REQ_ERROR = 'libnacl import error, perhaps missing python libnacl package or should update.' __virtualname__ = 'nacl' @@ -50,91 +181,294 @@ def _get_config(**kwargs): Return configuration ''' config = { - 'key': None, - 'keyfile': None, + 'box_type': 'sealedbox', + 'sk': None, + 'sk_file': '/etc/salt/pki/master/nacl', + 'pk': None, + 'pk_file': '/etc/salt/pki/master/nacl.pub', } config_key = '{0}.config'.format(__virtualname__) - config.update(__opts__.get(config_key, {})) - for k in set(config) & set(kwargs): + try: + config.update(__salt__['config.get'](config_key, {})) + except (NameError, KeyError) as e: + # likly using salt-run so fallback to __opts__ + config.update(__opts__.get(config_key, {})) + # pylint: disable=C0201 + for k in set(config.keys()) & set(kwargs.keys()): config[k] = kwargs[k] return config -def _get_key(rstrip_newline=True, **kwargs): +def _get_sk(**kwargs): ''' - Return key + Return sk ''' config = _get_config(**kwargs) - key = config['key'] - keyfile = config['keyfile'] - if not key and keyfile: - if not os.path.isfile(keyfile): - raise Exception('file not found: {0}'.format(keyfile)) - with salt.utils.fopen(keyfile, 'rb') as keyf: - key = keyf.read() + key = config['sk'] + sk_file = config['sk_file'] + if not key and sk_file: + with salt.utils.files.fopen(sk_file, 'rb') as keyf: + key = str(keyf.read()).rstrip('\n') if key is None: - raise Exception('no key found') - key = str(key) - if rstrip_newline: - key = key.rstrip('\n') - return key + raise Exception('no key or sk_file found') + return base64.b64decode(key) -def keygen(keyfile=None): +def _get_pk(**kwargs): ''' - Use libnacl to generate a private key + Return pk + ''' + config = _get_config(**kwargs) + pubkey = config['pk'] + pk_file = config['pk_file'] + if not pubkey and pk_file: + with salt.utils.files.fopen(pk_file, 'rb') as keyf: + pubkey = str(keyf.read()).rstrip('\n') + if pubkey is None: + raise Exception('no pubkey or pk_file found') + pubkey = str(pubkey) + return base64.b64decode(pubkey) + + +def keygen(sk_file=None, pk_file=None): + ''' + Use libnacl to generate a keypair. + + If no `sk_file` is defined return a keypair. + + If only the `sk_file` is defined `pk_file` will use the same name with a postfix `.pub`. + + When the `sk_file` is already existing, but `pk_file` is not. The `pk_file` will be generated + using the `sk_file`. CLI Examples: .. code-block:: bash - salt-run nacl.keygen - salt-run nacl.keygen keyfile=/root/.nacl - salt-run --out=newline_values_only nacl.keygen > /root/.nacl + salt-call nacl.keygen + salt-call nacl.keygen sk_file=/etc/salt/pki/master/nacl + salt-call nacl.keygen sk_file=/etc/salt/pki/master/nacl pk_file=/etc/salt/pki/master/nacl.pub + salt-call --local nacl.keygen ''' - b = libnacl.secret.SecretBox() - key = b.sk - key = base64.b64encode(key) - if keyfile: - if os.path.isfile(keyfile): - raise Exception('file already found: {0}'.format(keyfile)) - with salt.utils.fopen(keyfile, 'w') as keyf: - keyf.write(key) - return 'saved: {0}'.format(keyfile) - return key + if sk_file is None: + kp = libnacl.public.SecretKey() + return {'sk': base64.b64encode(kp.sk), 'pk': base64.b64encode(kp.pk)} + + if pk_file is None: + pk_file = '{0}.pub'.format(sk_file) + + if sk_file and pk_file is None: + if not os.path.isfile(sk_file): + kp = libnacl.public.SecretKey() + with salt.utils.files.fopen(sk_file, 'w') as keyf: + keyf.write(base64.b64encode(kp.sk)) + if salt.utils.platform.is_windows(): + cur_user = salt.utils.win_functions.get_current_user() + salt.utils.win_dacl.set_owner(sk_file, cur_user) + salt.utils.win_dacl.set_permissions(sk_file, cur_user, 'full_control', 'grant', reset_perms=True, protected=True) + else: + # chmod 0600 file + os.chmod(sk_file, 1536) + return 'saved sk_file: {0}'.format(sk_file) + else: + raise Exception('sk_file:{0} already exist.'.format(sk_file)) + + if sk_file is None and pk_file: + raise Exception('sk_file: Must be set inorder to generate a public key.') + + if os.path.isfile(sk_file) and os.path.isfile(pk_file): + raise Exception('sk_file:{0} and pk_file:{1} already exist.'.format(sk_file, pk_file)) + + if os.path.isfile(sk_file) and not os.path.isfile(pk_file): + # generate pk using the sk + with salt.utils.files.fopen(sk_file, 'rb') as keyf: + sk = str(keyf.read()).rstrip('\n') + sk = base64.b64decode(sk) + kp = libnacl.public.SecretKey(sk) + with salt.utils.files.fopen(pk_file, 'w') as keyf: + keyf.write(base64.b64encode(kp.pk)) + return 'saved pk_file: {0}'.format(pk_file) + + kp = libnacl.public.SecretKey() + with salt.utils.files.fopen(sk_file, 'w') as keyf: + keyf.write(base64.b64encode(kp.sk)) + if salt.utils.platform.is_windows(): + cur_user = salt.utils.win_functions.get_current_user() + salt.utils.win_dacl.set_owner(sk_file, cur_user) + salt.utils.win_dacl.set_permissions(sk_file, cur_user, 'full_control', 'grant', reset_perms=True, protected=True) + else: + # chmod 0600 file + os.chmod(sk_file, 1536) + with salt.utils.files.fopen(pk_file, 'w') as keyf: + keyf.write(base64.b64encode(kp.pk)) + return 'saved sk_file:{0} pk_file: {1}'.format(sk_file, pk_file) def enc(data, **kwargs): ''' - Takes a key generated from `nacl.keygen` and encrypt some data. + Alias to `{box_type}_encrypt` + + box_type: secretbox, sealedbox(default) + ''' + box_type = _get_config(**kwargs)['box_type'] + if box_type == 'sealedbox': + return sealedbox_encrypt(data, **kwargs) + if box_type == 'secretbox': + return secretbox_encrypt(data, **kwargs) + return sealedbox_encrypt(data, **kwargs) + + +def enc_file(name, out=None, **kwargs): + ''' + This is a helper function to encrypt a file and return its contents. + + You can provide an optional output file using `out` + + `name` can be a local file or when not using `salt-run` can be a url like `salt://`, `https://` etc. CLI Examples: .. code-block:: bash - salt-run nacl.enc datatoenc - salt-run nacl.enc datatoenc keyfile=/root/.nacl - salt-run nacl.enc datatoenc key='cKEzd4kXsbeCE7/nLTIqXwnUiD1ulg4NoeeYcCFpd9k=' + salt-run nacl.enc_file name=/tmp/id_rsa + salt-call nacl.enc_file name=salt://crt/mycert out=/tmp/cert + salt-run nacl.enc_file name=/tmp/id_rsa box_type=secretbox \ + sk_file=/etc/salt/pki/master/nacl.pub ''' - key = _get_key(**kwargs) - sk = base64.b64decode(key) - b = libnacl.secret.SecretBox(sk) - return base64.b64encode(b.encrypt(data)) + try: + data = __salt__['cp.get_file_str'](name) + except Exception as e: + # likly using salt-run so fallback to local filesystem + with salt.utils.files.fopen(name, 'rb') as f: + data = f.read() + d = enc(data, **kwargs) + if out: + if os.path.isfile(out): + raise Exception('file:{0} already exist.'.format(out)) + with salt.utils.files.fopen(out, 'wb') as f: + f.write(d) + return 'Wrote: {0}'.format(out) + return d def dec(data, **kwargs): ''' - Takes a key generated from `nacl.keygen` and decrypt some data. + Alias to `{box_type}_decrypt` + + box_type: secretbox, sealedbox(default) + ''' + box_type = _get_config(**kwargs)['box_type'] + if box_type == 'sealedbox': + return sealedbox_decrypt(data, **kwargs) + if box_type == 'secretbox': + return secretbox_decrypt(data, **kwargs) + return sealedbox_decrypt(data, **kwargs) + + +def dec_file(name, out=None, **kwargs): + ''' + This is a helper function to decrypt a file and return its contents. + + You can provide an optional output file using `out` + + `name` can be a local file or when not using `salt-run` can be a url like `salt://`, `https://` etc. CLI Examples: .. code-block:: bash - salt-run nacl.dec pEXHQM6cuaF7A= - salt-run nacl.dec data='pEXHQM6cuaF7A=' keyfile=/root/.nacl - salt-run nacl.dec data='pEXHQM6cuaF7A=' key='cKEzd4kXsbeCE7/nLTIqXwnUiD1ulg4NoeeYcCFpd9k=' + salt-run nacl.dec_file name=/tmp/id_rsa.nacl + salt-call nacl.dec_file name=salt://crt/mycert.nacl out=/tmp/id_rsa + salt-run nacl.dec_file name=/tmp/id_rsa.nacl box_type=secretbox \ + sk_file=/etc/salt/pki/master/nacl.pub ''' - key = _get_key(**kwargs) - sk = base64.b64decode(key) - b = libnacl.secret.SecretBox(key=sk) + try: + data = __salt__['cp.get_file_str'](name) + except Exception as e: + # likly using salt-run so fallback to local filesystem + with salt.utils.files.fopen(name, 'rb') as f: + data = f.read() + d = dec(data, **kwargs) + if out: + if os.path.isfile(out): + raise Exception('file:{0} already exist.'.format(out)) + with salt.utils.files.fopen(out, 'wb') as f: + f.write(d) + return 'Wrote: {0}'.format(out) + return d + + +def sealedbox_encrypt(data, **kwargs): + ''' + Encrypt data using a public key generated from `nacl.keygen`. + The encryptd data can be decrypted using `nacl.sealedbox_decrypt` only with the secret key. + + CLI Examples: + + .. code-block:: bash + + salt-run nacl.sealedbox_encrypt datatoenc + salt-call --local nacl.sealedbox_encrypt datatoenc pk_file=/etc/salt/pki/master/nacl.pub + salt-call --local nacl.sealedbox_encrypt datatoenc pk='vrwQF7cNiNAVQVAiS3bvcbJUnF0cN6fU9YTZD9mBfzQ=' + ''' + pk = _get_pk(**kwargs) + b = libnacl.sealed.SealedBox(pk) + return base64.b64encode(b.encrypt(data)) + + +def sealedbox_decrypt(data, **kwargs): + ''' + Decrypt data using a secret key that was encrypted using a public key with `nacl.sealedbox_encrypt`. + + CLI Examples: + + .. code-block:: bash + + salt-call nacl.sealedbox_decrypt pEXHQM6cuaF7A= + salt-call --local nacl.sealedbox_decrypt data='pEXHQM6cuaF7A=' sk_file=/etc/salt/pki/master/nacl + salt-call --local nacl.sealedbox_decrypt data='pEXHQM6cuaF7A=' sk='YmFkcGFzcwo=' + ''' + if data is None: + return None + sk = _get_sk(**kwargs) + keypair = libnacl.public.SecretKey(sk) + b = libnacl.sealed.SealedBox(keypair) + return b.decrypt(base64.b64decode(data)) + + +def secretbox_encrypt(data, **kwargs): + ''' + Encrypt data using a secret key generated from `nacl.keygen`. + The same secret key can be used to decrypt the data using `nacl.secretbox_decrypt`. + + CLI Examples: + + .. code-block:: bash + + salt-run nacl.secretbox_encrypt datatoenc + salt-call --local nacl.secretbox_encrypt datatoenc sk_file=/etc/salt/pki/master/nacl + salt-call --local nacl.secretbox_encrypt datatoenc sk='YmFkcGFzcwo=' + ''' + sk = _get_sk(**kwargs) + b = libnacl.secret.SecretBox(sk) + return base64.b64encode(b.encrypt(data)) + + +def secretbox_decrypt(data, **kwargs): + ''' + Decrypt data that was encrypted using `nacl.secretbox_encrypt` using the secret key + that was generated from `nacl.keygen`. + + CLI Examples: + + .. code-block:: bash + + salt-call nacl.secretbox_decrypt pEXHQM6cuaF7A= + salt-call --local nacl.secretbox_decrypt data='pEXHQM6cuaF7A=' sk_file=/etc/salt/pki/master/nacl + salt-call --local nacl.secretbox_decrypt data='pEXHQM6cuaF7A=' sk='YmFkcGFzcwo=' + ''' + if data is None: + return None + key = _get_sk(**kwargs) + b = libnacl.secret.SecretBox(key=key) return b.decrypt(base64.b64decode(data)) diff --git a/salt/runners/network.py b/salt/runners/network.py index a6c3b2cf45e..796cae3e3ad 100644 --- a/salt/runners/network.py +++ b/salt/runners/network.py @@ -11,6 +11,7 @@ import socket # Import salt libs import salt.utils +import salt.utils.files log = logging.getLogger(__name__) @@ -30,7 +31,7 @@ def wollist(maclist, bcast='255.255.255.255', destport=9): ''' ret = [] try: - with salt.utils.fopen(maclist, 'r') as ifile: + with salt.utils.files.fopen(maclist, 'r') as ifile: for mac in ifile: wol(mac.strip(), bcast, destport) print('Waking up {0}'.format(mac.strip())) diff --git a/salt/runners/pkg.py b/salt/runners/pkg.py index 3e1f06ca338..cbb918726ee 100644 --- a/salt/runners/pkg.py +++ b/salt/runners/pkg.py @@ -13,7 +13,7 @@ import salt.output import salt.minion # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six def _get_returner(returner_types): diff --git a/salt/runners/queue.py b/salt/runners/queue.py index 10e9c99a77c..01bad4b4e71 100644 --- a/salt/runners/queue.py +++ b/salt/runners/queue.py @@ -68,7 +68,7 @@ from __future__ import absolute_import # Import salt libs import salt.loader -import salt.ext.six as six +from salt.ext import six from salt.utils.event import get_event, tagify from salt.exceptions import SaltInvocationError diff --git a/salt/runners/saltutil.py b/salt/runners/saltutil.py index 555de1b16b5..b691f827e1d 100644 --- a/salt/runners/saltutil.py +++ b/salt/runners/saltutil.py @@ -38,6 +38,7 @@ def sync_all(saltenv='base', extmod_whitelist=None, extmod_blacklist=None): ''' log.debug('Syncing all') ret = {} + ret['clouds'] = sync_clouds(saltenv=saltenv, extmod_whitelist=extmod_whitelist, extmod_blacklist=extmod_blacklist) ret['modules'] = sync_modules(saltenv=saltenv, extmod_whitelist=extmod_whitelist, extmod_blacklist=extmod_blacklist) ret['states'] = sync_states(saltenv=saltenv, extmod_whitelist=extmod_whitelist, extmod_blacklist=extmod_blacklist) ret['grains'] = sync_grains(saltenv=saltenv, extmod_whitelist=extmod_whitelist, extmod_blacklist=extmod_blacklist) @@ -58,6 +59,7 @@ def sync_all(saltenv='base', extmod_whitelist=None, extmod_blacklist=None): ret['cache'] = sync_cache(saltenv=saltenv, extmod_whitelist=extmod_whitelist, extmod_blacklist=extmod_blacklist) ret['fileserver'] = sync_fileserver(saltenv=saltenv, extmod_whitelist=extmod_whitelist, extmod_blacklist=extmod_blacklist) ret['tops'] = sync_tops(saltenv=saltenv, extmod_whitelist=extmod_whitelist, extmod_blacklist=extmod_blacklist) + ret['tokens'] = sync_eauth_tokens(saltenv=saltenv, extmod_whitelist=extmod_whitelist, extmod_blacklist=extmod_blacklist) return ret @@ -523,3 +525,29 @@ def sync_roster(saltenv='base', extmod_whitelist=None, extmod_blacklist=None): ''' return salt.utils.extmods.sync(__opts__, 'roster', saltenv=saltenv, extmod_whitelist=extmod_whitelist, extmod_blacklist=extmod_blacklist)[0] + + +def sync_eauth_tokens(saltenv='base', extmod_whitelist=None, extmod_blacklist=None): + ''' + .. versionadded:: Oxygen + + Sync eauth token modules from ``salt://_tokens`` to the master + + saltenv : base + The fileserver environment from which to sync. To sync from more than + one environment, pass a comma-separated list. + + extmod_whitelist : None + comma-seperated list of modules to sync + + extmod_blacklist : None + comma-seperated list of modules to blacklist based on type + + CLI Example: + + .. code-block:: bash + + salt-run saltutil.sync_eauth_tokens + ''' + return salt.utils.extmods.sync(__opts__, 'tokens', saltenv=saltenv, extmod_whitelist=extmod_whitelist, + extmod_blacklist=extmod_blacklist)[0] diff --git a/salt/runners/smartos_vmadm.py b/salt/runners/smartos_vmadm.py index 9faeca5f585..15f8cec6f87 100644 --- a/salt/runners/smartos_vmadm.py +++ b/salt/runners/smartos_vmadm.py @@ -12,7 +12,7 @@ from salt.exceptions import SaltClientError from salt.utils.odict import OrderedDict # Import 3rd party libs -import salt.ext.six as six +from salt.ext import six # Function aliases __func_alias__ = { diff --git a/salt/runners/spacewalk.py b/salt/runners/spacewalk.py index ff2ef3803f0..cb9495934a4 100644 --- a/salt/runners/spacewalk.py +++ b/salt/runners/spacewalk.py @@ -36,7 +36,7 @@ import atexit import logging # Import third party libs -import salt.ext.six as six +from salt.ext import six log = logging.getLogger(__name__) diff --git a/salt/runners/ssh.py b/salt/runners/ssh.py index e26b4579e36..a35e9245555 100644 --- a/salt/runners/ssh.py +++ b/salt/runners/ssh.py @@ -10,6 +10,7 @@ from __future__ import absolute_import # Import Salt Libs import salt.client.ssh.client +import salt.utils.versions def cmd(tgt, @@ -34,7 +35,7 @@ def cmd(tgt, # remember to remove the expr_form argument from this function when # performing the cleanup on this deprecation. if expr_form is not None: - salt.utils.warn_until( + salt.utils.versions.warn_until( 'Fluorine', 'the target type should be passed using the \'tgt_type\' ' 'argument instead of \'expr_form\'. Support for using ' diff --git a/salt/runners/state.py b/salt/runners/state.py index da101d4be34..25f12a814e5 100644 --- a/salt/runners/state.py +++ b/salt/runners/state.py @@ -73,15 +73,15 @@ def orchestrate(mods, minion = salt.minion.MasterMinion(__opts__) running = minion.functions['state.sls']( mods, - saltenv, test, exclude, pillar=pillar, + saltenv=saltenv, pillarenv=pillarenv, pillar_enc=pillar_enc, orchestration_jid=orchestration_jid) ret = {'data': {minion.opts['id']: running}, 'outputter': 'highstate'} - res = salt.utils.check_state_result(ret['data']) + res = __utils__['state.check_result'](ret['data']) if res: ret['retcode'] = 0 else: diff --git a/salt/runners/survey.py b/salt/runners/survey.py index 077e6382307..3fd2b1014f7 100644 --- a/salt/runners/survey.py +++ b/salt/runners/survey.py @@ -20,7 +20,7 @@ import salt.client from salt.exceptions import SaltClientError # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import range diff --git a/salt/runners/test.py b/salt/runners/test.py index 108c9775eea..0d56edf17fe 100644 --- a/salt/runners/test.py +++ b/salt/runners/test.py @@ -6,7 +6,7 @@ from __future__ import absolute_import from __future__ import print_function # Import python libs import time -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import range @@ -37,6 +37,18 @@ def raw_arg(*args, **kwargs): return ret +def metasyntactic(locality='us'): + ''' + Return common metasyntactic variables for the given locality + ''' + lookup = { + 'us': ['foo', 'bar', 'baz', 'qux', 'quux', 'quuz', 'corge', 'grault', + 'garply', 'waldo', 'fred', 'plugh', 'xyzzy', 'thud'], + 'uk': ['wibble', 'wobble', 'wubble', 'flob'], + } + return lookup.get(locality, None) + + def stdout_print(): ''' Print 'foo' and return 'bar' @@ -63,3 +75,18 @@ def stream(): __jid_event__.fire_event({'message': 'Runner is {0}% done'.format(i)}, 'progress') time.sleep(0.1) return ret + + +def get_opts(): + ''' + .. versionadded:: Oxygen + + Return the configuration options of the master. + + CLI Example: + + .. code-block:: + + salt-run test.get_opts + ''' + return __opts__ diff --git a/salt/runners/venafiapi.py b/salt/runners/venafiapi.py index 7fa04c6ed0b..887b3e5bd05 100644 --- a/salt/runners/venafiapi.py +++ b/salt/runners/venafiapi.py @@ -37,8 +37,8 @@ from Crypto.PublicKey import RSA import json import salt.syspaths as syspaths import salt.cache -import salt.utils -import salt.ext.six as six +import salt.utils.files +from salt.ext import six from salt.exceptions import CommandExecutionError __virtualname__ = 'venafi' @@ -188,7 +188,7 @@ def gen_csr( tmppriv = '{0}/priv'.format(tmpdir) tmpcsr = '{0}/csr'.format(tmpdir) - with salt.utils.fopen(tmppriv, 'w') as if_: + with salt.utils.files.fopen(tmppriv, 'w') as if_: if_.write(data['private_key']) if country is None: @@ -232,7 +232,7 @@ def gen_csr( 'country, state, loc, org, org_unit' ) - with salt.utils.fopen(tmpcsr, 'r') as of_: + with salt.utils.files.fopen(tmpcsr, 'r') as of_: csr = of_.read() data['minion_id'] = minion_id diff --git a/salt/runners/virt.py b/salt/runners/virt.py index d008f6f1880..6a9822d5b7f 100644 --- a/salt/runners/virt.py +++ b/salt/runners/virt.py @@ -10,13 +10,13 @@ import logging # Import Salt libs import salt.client -import salt.utils.virt +import salt.utils.files import salt.utils.cloud import salt.key from salt.exceptions import SaltClientError # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six log = logging.getLogger(__name__) @@ -245,7 +245,7 @@ def init( __jid_event__.fire_event({'message': 'Minion will be preseeded'}, 'progress') priv_key, pub_key = salt.utils.cloud.gen_keys() accepted_key = os.path.join(__opts__['pki_dir'], 'minions', name) - with salt.utils.fopen(accepted_key, 'w') as fp_: + with salt.utils.files.fopen(accepted_key, 'w') as fp_: fp_.write(pub_key) client = salt.client.get_local_client(__opts__['conf_file']) diff --git a/salt/runners/winrepo.py b/salt/runners/winrepo.py index fc899d16706..01c209d3820 100644 --- a/salt/runners/winrepo.py +++ b/salt/runners/winrepo.py @@ -11,7 +11,7 @@ from __future__ import absolute_import, print_function import os # Import third party libs -import salt.ext.six as six +from salt.ext import six try: import msgpack except ImportError: @@ -19,7 +19,7 @@ except ImportError: # Import salt libs from salt.exceptions import CommandExecutionError, SaltRenderError -import salt.utils +import salt.utils.files import salt.utils.gitfs import logging import salt.minion @@ -119,7 +119,7 @@ def genrepo(opts=None, fire_event=True): revmap[repodata['full_name']] = pkgname ret.setdefault('repo', {}).update(config) ret.setdefault('name_map', {}).update(revmap) - with salt.utils.fopen( + with salt.utils.files.fopen( os.path.join(winrepo_dir, winrepo_cachefile), 'w+b') as repo: repo.write(msgpack.dumps(ret)) return ret diff --git a/salt/scripts.py b/salt/scripts.py index 20104bf31c1..acef4987f97 100644 --- a/salt/scripts.py +++ b/salt/scripts.py @@ -24,7 +24,7 @@ import salt.defaults.exitcodes # pylint: disable=unused-import log = logging.getLogger(__name__) -def _handle_interrupt(exc, original_exc, hardfail=False, trace=''): +def _handle_interrupt(exc, original_exc, hardfail=False, trace=u''): ''' if hardfailing: If we got the original stacktrace, log it @@ -50,16 +50,16 @@ def _handle_signals(client, signum, sigframe): hardcrash = False if signum == signal.SIGINT: - exit_msg = '\nExiting gracefully on Ctrl-c' + exit_msg = u'\nExiting gracefully on Ctrl-c' try: - jid = client.local_client.pub_data['jid'] + jid = client.local_client.pub_data[u'jid'] exit_msg += ( - '\n' - 'This job\'s jid is: {0}\n' - 'The minions may not have all finished running and any remaining ' - 'minions will return upon completion. To look up the return data ' - 'for this job later, run the following command:\n\n' - 'salt-run jobs.lookup_jid {0}'.format(jid) + u'\n' + u'This job\'s jid is: {0}\n' + u'The minions may not have all finished running and any remaining ' + u'minions will return upon completion. To look up the return data ' + u'for this job later, run the following command:\n\n' + u'salt-run jobs.lookup_jid {0}'.format(jid) ) except (AttributeError, KeyError): pass @@ -68,7 +68,7 @@ def _handle_signals(client, signum, sigframe): _handle_interrupt( SystemExit(exit_msg), - Exception('\nExiting with hard crash on Ctrl-c'), + Exception(u'\nExiting with hard crash on Ctrl-c'), hardcrash, trace=trace) @@ -96,12 +96,12 @@ def minion_process(): ''' Start a minion process ''' - import salt.utils + import salt.utils.platform import salt.cli.daemons # salt_minion spawns this function in a new process - salt.utils.appendproctitle('KeepAlive') + salt.utils.appendproctitle(u'KeepAlive') def handle_hup(manager, sig, frame): manager.minion.reload() @@ -117,15 +117,15 @@ def minion_process(): time.sleep(5) try: # check pid alive (Unix only trick!) - if os.getuid() == 0 and not salt.utils.is_windows(): + if os.getuid() == 0 and not salt.utils.platform.is_windows(): os.kill(parent_pid, 0) except OSError as exc: # forcibly exit, regular sys.exit raises an exception-- which # isn't sufficient in a thread - log.error('Minion process encountered exception: {0}'.format(exc)) + log.error(u'Minion process encountered exception: %s', exc) os._exit(salt.defaults.exitcodes.EX_GENERIC) - if not salt.utils.is_windows(): + if not salt.utils.platform.is_windows(): thread = threading.Thread(target=suicide_when_without_parent, args=(os.getppid(),)) thread.start() @@ -137,13 +137,13 @@ def minion_process(): try: minion.start() except (SaltClientError, SaltReqTimeoutError, SaltSystemExit) as exc: - log.warning('Fatal functionality error caught by minion handler:\n', exc_info=True) - log.warning('** Restarting minion **') + log.warning(u'Fatal functionality error caught by minion handler:\n', exc_info=True) + log.warning(u'** Restarting minion **') delay = 60 - if minion is not None and hasattr(minion, 'config'): - delay = minion.config.get('random_reauth_delay', 60) + if minion is not None and hasattr(minion, u'config'): + delay = minion.config.get(u'random_reauth_delay', 60) delay = randint(1, delay) - log.info('waiting random_reauth_delay {0}s'.format(delay)) + log.info(u'waiting random_reauth_delay %ss', delay) time.sleep(delay) sys.exit(salt.defaults.exitcodes.SALT_KEEPALIVE) @@ -155,21 +155,22 @@ def salt_minion(): ''' import signal + import salt.utils.platform import salt.utils.process salt.utils.process.notify_systemd() import salt.cli.daemons import multiprocessing - if '' in sys.path: - sys.path.remove('') + if u'' in sys.path: + sys.path.remove(u'') - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): minion = salt.cli.daemons.Minion() minion.start() return - if '--disable-keepalive' in sys.argv: - sys.argv.remove('--disable-keepalive') + if u'--disable-keepalive' in sys.argv: + sys.argv.remove(u'--disable-keepalive') minion = salt.cli.daemons.Minion() minion.start() return @@ -230,6 +231,7 @@ def proxy_minion_process(queue): Start a proxy minion process ''' import salt.cli.daemons + import salt.utils.platform # salt_minion spawns this function in a new process def suicide_when_without_parent(parent_pid): @@ -249,7 +251,7 @@ def proxy_minion_process(queue): # isn't sufficient in a thread os._exit(999) - if not salt.utils.is_windows(): + if not salt.utils.platform.is_windows(): thread = threading.Thread(target=suicide_when_without_parent, args=(os.getppid(),)) thread.start() @@ -260,7 +262,7 @@ def proxy_minion_process(queue): proxyminion = salt.cli.daemons.ProxyMinion() proxyminion.start() except (Exception, SaltClientError, SaltReqTimeoutError, SaltSystemExit) as exc: - log.error('Proxy Minion failed to start: ', exc_info=True) + log.error(u'Proxy Minion failed to start: ', exc_info=True) restart = True # status is superfluous since the process will be restarted status = salt.defaults.exitcodes.SALT_KEEPALIVE @@ -269,13 +271,13 @@ def proxy_minion_process(queue): status = exc.code if restart is True: - log.warning('** Restarting proxy minion **') + log.warning(u'** Restarting proxy minion **') delay = 60 if proxyminion is not None: - if hasattr(proxyminion, 'config'): - delay = proxyminion.config.get('random_reauth_delay', 60) + if hasattr(proxyminion, u'config'): + delay = proxyminion.config.get(u'random_reauth_delay', 60) random_delay = randint(1, delay) - log.info('Sleeping random_reauth_delay of {0} seconds'.format(random_delay)) + log.info(u'Sleeping random_reauth_delay of %s seconds', random_delay) # preform delay after minion resources have been cleaned queue.put(random_delay) else: @@ -288,17 +290,18 @@ def salt_proxy(): Start a proxy minion. ''' import salt.cli.daemons + import salt.utils.platform import multiprocessing - if '' in sys.path: - sys.path.remove('') + if u'' in sys.path: + sys.path.remove(u'') - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): proxyminion = salt.cli.daemons.ProxyMinion() proxyminion.start() return - if '--disable-keepalive' in sys.argv: - sys.argv.remove('--disable-keepalive') + if u'--disable-keepalive' in sys.argv: + sys.argv.remove(u'--disable-keepalive') proxyminion = salt.cli.daemons.ProxyMinion() proxyminion.start() return @@ -364,7 +367,7 @@ def salt_key(): _install_signal_handlers(client) client.run() except Exception as err: - sys.stderr.write("Error: {0}\n".format(err)) + sys.stderr.write(u"Error: {0}\n".format(err)) def salt_cp(): @@ -384,8 +387,8 @@ def salt_call(): salt minion to run. ''' import salt.cli.call - if '' in sys.path: - sys.path.remove('') + if u'' in sys.path: + sys.path.remove(u'') client = salt.cli.call.SaltCall() _install_signal_handlers(client) client.run() @@ -396,8 +399,8 @@ def salt_run(): Execute a salt convenience routine. ''' import salt.cli.run - if '' in sys.path: - sys.path.remove('') + if u'' in sys.path: + sys.path.remove(u'') client = salt.cli.run.SaltRun() _install_signal_handlers(client) client.run() @@ -408,8 +411,8 @@ def salt_ssh(): Execute the salt-ssh system ''' import salt.cli.ssh - if '' in sys.path: - sys.path.remove('') + if u'' in sys.path: + sys.path.remove(u'') try: client = salt.cli.ssh.SaltSSH() _install_signal_handlers(client) @@ -440,11 +443,11 @@ def salt_cloud(): import salt.cloud.cli except ImportError as e: # No salt cloud on Windows - log.error("Error importing salt cloud {0}".format(e)) - print('salt-cloud is not available in this system') + log.error(u'Error importing salt cloud: %s', e) + print(u'salt-cloud is not available in this system') sys.exit(salt.defaults.exitcodes.EX_UNAVAILABLE) - if '' in sys.path: - sys.path.remove('') + if u'' in sys.path: + sys.path.remove(u'') client = salt.cloud.cli.SaltCloud() _install_signal_handlers(client) @@ -469,8 +472,8 @@ def salt_main(): master. ''' import salt.cli.salt - if '' in sys.path: - sys.path.remove('') + if u'' in sys.path: + sys.path.remove(u'') client = salt.cli.salt.SaltCMD() _install_signal_handlers(client) client.run() diff --git a/salt/sdb/sqlite3.py b/salt/sdb/sqlite3.py index a5a094746f6..e0b85c65e82 100644 --- a/salt/sdb/sqlite3.py +++ b/salt/sdb/sqlite3.py @@ -54,7 +54,7 @@ except ImportError: HAS_SQLITE3 = False # Import salt libs -import salt.ext.six as six +from salt.ext import six # Import third party libs import msgpack diff --git a/salt/sdb/yaml.py b/salt/sdb/yaml.py index aafc63246cb..028feb5ff29 100644 --- a/salt/sdb/yaml.py +++ b/salt/sdb/yaml.py @@ -42,6 +42,7 @@ import logging import salt.exceptions import salt.loader import salt.utils +import salt.utils.files import salt.utils.dictupdate log = logging.getLogger(__name__) @@ -76,7 +77,7 @@ def _get_values(profile=None): ret = {} for fname in profile.get('files', []): try: - with salt.utils.flopen(fname) as f: + with salt.utils.files.flopen(fname) as f: contents = serializers.yaml.deserialize(f) ret = salt.utils.dictupdate.merge(ret, contents, **profile.get('merge', {})) diff --git a/salt/serializers/configparser.py b/salt/serializers/configparser.py index 3df65e93f13..787ce88f3a7 100644 --- a/salt/serializers/configparser.py +++ b/salt/serializers/configparser.py @@ -12,7 +12,7 @@ from __future__ import absolute_import # Import Salt Libs -import salt.ext.six as six +from salt.ext import six import salt.ext.six.moves.configparser as configparser # pylint: disable=E0611 from salt.serializers import DeserializationError, SerializationError diff --git a/salt/serializers/json.py b/salt/serializers/json.py index d0f42efab9f..024842a43ce 100644 --- a/salt/serializers/json.py +++ b/salt/serializers/json.py @@ -15,9 +15,12 @@ try: except ImportError: import json -from salt.ext.six import string_types +# Import Salt libs from salt.serializers import DeserializationError, SerializationError +# Import 3rd-party libs +from salt.ext import six + __all__ = ['deserialize', 'serialize', 'available'] available = True @@ -32,7 +35,7 @@ def deserialize(stream_or_string, **options): ''' try: - if not isinstance(stream_or_string, (bytes, string_types)): + if not isinstance(stream_or_string, (bytes, six.string_types)): return json.load(stream_or_string, **options) if isinstance(stream_or_string, bytes): diff --git a/salt/serializers/msgpack.py b/salt/serializers/msgpack.py index 553ab622fad..6085f3525ad 100644 --- a/salt/serializers/msgpack.py +++ b/salt/serializers/msgpack.py @@ -16,7 +16,7 @@ from salt.log import setup_console_logger from salt.serializers import DeserializationError, SerializationError # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six log = logging.getLogger(__name__) diff --git a/salt/serializers/yaml.py b/salt/serializers/yaml.py index 5ebc94bfc06..2fad384d1bb 100644 --- a/salt/serializers/yaml.py +++ b/salt/serializers/yaml.py @@ -17,7 +17,7 @@ from yaml.constructor import ConstructorError from yaml.scanner import ScannerError from salt.serializers import DeserializationError, SerializationError -import salt.ext.six as six +from salt.ext import six from salt.utils.odict import OrderedDict __all__ = ['deserialize', 'serialize', 'available'] @@ -42,7 +42,7 @@ def deserialize(stream_or_string, **options): :param options: options given to lower yaml module. ''' - options.setdefault('Loader', BaseLoader) + options.setdefault('Loader', Loader) try: return yaml.load(stream_or_string, **options) except ScannerError as error: diff --git a/salt/serializers/yamlex.py b/salt/serializers/yamlex.py index 65fbf58e485..6e1c22b39c5 100644 --- a/salt/serializers/yamlex.py +++ b/salt/serializers/yamlex.py @@ -119,7 +119,7 @@ import yaml from yaml.nodes import MappingNode from yaml.constructor import ConstructorError from yaml.scanner import ScannerError -import salt.ext.six as six +from salt.ext import six __all__ = ['deserialize', 'serialize', 'available'] diff --git a/salt/spm/__init__.py b/salt/spm/__init__.py index b2f367e5ac9..10da68a9b4b 100644 --- a/salt/spm/__init__.py +++ b/salt/spm/__init__.py @@ -11,26 +11,30 @@ import os import yaml import tarfile import shutil -import msgpack import hashlib import logging -import pwd -import grp import sys +try: + import pwd + import grp +except ImportError: + pass # Import Salt libs import salt.client import salt.config import salt.loader import salt.cache -import salt.utils -import salt.utils.http as http import salt.syspaths as syspaths -import salt.ext.six as six +from salt.ext import six from salt.ext.six import string_types from salt.ext.six.moves import input from salt.ext.six.moves import filter from salt.template import compile_template +import salt.utils.files +import salt.utils.http as http +import salt.utils.platform +import salt.utils.win_functions from salt.utils.yamldumper import SafeOrderedDumper # Get logging started @@ -354,11 +358,12 @@ class SPMClient(object): dl_url = dl_url.replace('file://', '') shutil.copyfile(dl_url, out_file) else: - with salt.utils.fopen(out_file, 'w') as outf: + with salt.utils.files.fopen(out_file, 'w') as outf: outf.write(self._query_http(dl_url, repo_info['info'])) # First we download everything, then we install for package in dl_list: + out_file = dl_list[package]['dest_file'] # Kick off the install self._install_indv_pkg(package, out_file) return @@ -490,10 +495,16 @@ class SPMClient(object): # No defaults for this in config.py; default to the current running # user and group - uid = self.opts.get('spm_uid', os.getuid()) - gid = self.opts.get('spm_gid', os.getgid()) - uname = pwd.getpwuid(uid)[0] - gname = grp.getgrgid(gid)[0] + if salt.utils.platform.is_windows(): + uname = gname = salt.utils.win_functions.get_current_user() + uname_sid = salt.utils.win_functions.get_sid_from_name(uname) + uid = self.opts.get('spm_uid', uname_sid) + gid = self.opts.get('spm_gid', uname_sid) + else: + uid = self.opts.get('spm_uid', os.getuid()) + gid = self.opts.get('spm_gid', os.getgid()) + uname = pwd.getpwuid(uid)[0] + gname = grp.getgrgid(gid)[0] # Second pass: install the files for member in pkg_files: @@ -612,7 +623,7 @@ class SPMClient(object): for repo_file in repo_files: repo_path = '{0}.d/{1}'.format(self.opts['spm_repos_config'], repo_file) - with salt.utils.fopen(repo_path) as rph: + with salt.utils.files.fopen(repo_path) as rph: repo_data = yaml.safe_load(rph) for repo in repo_data: if repo_data[repo].get('enabled', True) is False: @@ -670,7 +681,7 @@ class SPMClient(object): dl_path = '{0}/SPM-METADATA'.format(repo_info['url']) if dl_path.startswith('file://'): dl_path = dl_path.replace('file://', '') - with salt.utils.fopen(dl_path, 'r') as rpm: + with salt.utils.files.fopen(dl_path, 'r') as rpm: metadata = yaml.safe_load(rpm) else: metadata = self._query_http(dl_path, repo_info) @@ -709,7 +720,7 @@ class SPMClient(object): raise SPMInvocationError('A path to a directory must be specified') if args[1] == '.': - repo_path = os.environ['PWD'] + repo_path = os.getcwdu() else: repo_path = args[1] @@ -781,7 +792,7 @@ class SPMClient(object): repo_metadata[spm_name]['filename'] = spm_file metadata_filename = '{0}/SPM-METADATA'.format(repo_path) - with salt.utils.fopen(metadata_filename, 'w') as mfh: + with salt.utils.files.fopen(metadata_filename, 'w') as mfh: yaml.dump( repo_metadata, mfh, @@ -1016,7 +1027,7 @@ class SPMClient(object): formula_path = '{0}/FORMULA'.format(self.abspath) if not os.path.exists(formula_path): raise SPMPackageError('Formula file {0} not found'.format(formula_path)) - with salt.utils.fopen(formula_path) as fp_: + with salt.utils.files.fopen(formula_path) as fp_: formula_conf = yaml.safe_load(fp_) for field in ('name', 'version', 'release', 'summary', 'description'): diff --git a/salt/spm/pkgfiles/local.py b/salt/spm/pkgfiles/local.py index 5e97fc2125d..2d7f8ae3d42 100644 --- a/salt/spm/pkgfiles/local.py +++ b/salt/spm/pkgfiles/local.py @@ -13,7 +13,8 @@ import logging # Import Salt libs import salt.syspaths -import salt.utils +import salt.utils.files +import salt.utils.stringutils # Get logging started log = logging.getLogger(__name__) @@ -183,8 +184,8 @@ def hash_file(path, hashobj, conn=None): if os.path.isdir(path): return '' - with salt.utils.fopen(path, 'r') as f: - hashobj.update(salt.utils.to_bytes(f.read())) + with salt.utils.files.fopen(path, 'r') as f: + hashobj.update(salt.utils.stringutils.to_bytes(f.read())) return hashobj.hexdigest() diff --git a/salt/state.py b/salt/state.py index 0bd85e6db4c..5527a0f87c7 100644 --- a/salt/state.py +++ b/salt/state.py @@ -32,13 +32,17 @@ import salt.loader import salt.minion import salt.pillar import salt.fileclient +import salt.utils.args import salt.utils.crypt import salt.utils.dictupdate import salt.utils.event +import salt.utils.files +import salt.utils.immutabletypes as immutabletypes import salt.utils.url +import salt.utils.platform import salt.utils.process +import salt.utils.files import salt.syspaths as syspaths -from salt.utils import immutabletypes from salt.template import compile_template, compile_template_str from salt.exceptions import ( SaltException, @@ -52,7 +56,7 @@ import salt.utils.yamlloader as yamlloader # Import third party libs # pylint: disable=import-error,no-name-in-module,redefined-builtin -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import map, range, reload_module # pylint: enable=import-error,no-name-in-module,redefined-builtin import msgpack @@ -63,57 +67,58 @@ log = logging.getLogger(__name__) # These are keywords passed to state module functions which are to be used # by salt in this state module and not on the actual state module function STATE_REQUISITE_KEYWORDS = frozenset([ - 'onchanges', - 'onfail', - 'onfail_stop', - 'prereq', - 'prerequired', - 'watch', - 'require', - 'listen', + u'onchanges', + u'onfail', + u'onfail_stop', + u'prereq', + u'prerequired', + u'watch', + u'require', + u'listen', ]) STATE_REQUISITE_IN_KEYWORDS = frozenset([ - 'onchanges_in', - 'onfail_in', - 'prereq_in', - 'watch_in', - 'require_in', - 'listen_in', + u'onchanges_in', + u'onfail_in', + u'prereq_in', + u'watch_in', + u'require_in', + u'listen_in', ]) STATE_RUNTIME_KEYWORDS = frozenset([ - 'fun', - 'state', - 'check_cmd', - 'failhard', - 'onlyif', - 'unless', - 'retry', - 'order', - 'parallel', - 'prereq', - 'prereq_in', - 'prerequired', - 'reload_modules', - 'reload_grains', - 'reload_pillar', - 'runas', - 'fire_event', - 'saltenv', - 'use', - 'use_in', - '__env__', - '__sls__', - '__id__', - '__orchestration_jid__', - '__pub_user', - '__pub_arg', - '__pub_jid', - '__pub_fun', - '__pub_tgt', - '__pub_ret', - '__pub_pid', - '__pub_tgt_type', - '__prereq__', + u'fun', + u'state', + u'check_cmd', + u'failhard', + u'onlyif', + u'unless', + u'retry', + u'order', + u'parallel', + u'prereq', + u'prereq_in', + u'prerequired', + u'reload_modules', + u'reload_grains', + u'reload_pillar', + u'runas', + u'runas_password', + u'fire_event', + u'saltenv', + u'use', + u'use_in', + u'__env__', + u'__sls__', + u'__id__', + u'__orchestration_jid__', + u'__pub_user', + u'__pub_arg', + u'__pub_jid', + u'__pub_fun', + u'__pub_tgt', + u'__pub_ret', + u'__pub_pid', + u'__pub_tgt_type', + u'__prereq__', ]) STATE_INTERNAL_KEYWORDS = STATE_REQUISITE_KEYWORDS.union(STATE_REQUISITE_IN_KEYWORDS).union(STATE_RUNTIME_KEYWORDS) @@ -130,26 +135,33 @@ def split_low_tag(tag): ''' Take a low tag and split it back into the low dict that it came from ''' - state, id_, name, fun = tag.split('_|-') + state, id_, name, fun = tag.split(u'_|-') - return {'state': state, - '__id__': id_, - 'name': name, - 'fun': fun} + return {u'state': state, + u'__id__': id_, + u'name': name, + u'fun': fun} def _gen_tag(low): ''' Generate the running dict tag string from the low data structure ''' - return '{0[state]}_|-{0[__id__]}_|-{0[name]}_|-{0[fun]}'.format(low) + return u'{0[state]}_|-{0[__id__]}_|-{0[name]}_|-{0[fun]}'.format(low) + + +def _clean_tag(tag): + ''' + Make tag name safe for filenames + ''' + return salt.utils.files.safe_filename_leaf(tag) def _l_tag(name, id_): - low = {'name': 'listen_{0}'.format(name), - '__id__': 'listen_{0}'.format(id_), - 'state': 'Listen_Error', - 'fun': 'Listen_Error'} + low = {u'name': u'listen_{0}'.format(name), + u'__id__': u'listen_{0}'.format(id_), + u'state': u'Listen_Error', + u'fun': u'Listen_Error'} return _gen_tag(low) @@ -158,8 +170,8 @@ def trim_req(req): Trim any function off of a requisite ''' reqfirst = next(iter(req)) - if '.' in reqfirst: - return {reqfirst.split('.')[0]: req[reqfirst]} + if u'.' in reqfirst: + return {reqfirst.split(u'.')[0]: req[reqfirst]} return req @@ -191,9 +203,9 @@ def find_name(name, state, high): if name in high: ext_id.append((name, state)) # if we are requiring an entire SLS, then we need to add ourselves to everything in that SLS - elif state == 'sls': + elif state == u'sls': for nid, item in six.iteritems(high): - if item['__sls__'] == name: + if item[u'__sls__'] == name: ext_id.append((nid, next(iter(item)))) # otherwise we are requiring a single state, lets find it else: @@ -217,9 +229,9 @@ def find_sls_ids(sls, high): ''' ret = [] for nid, item in six.iteritems(high): - if item['__sls__'] == sls: + if item[u'__sls__'] == sls: for st_ in item: - if st_.startswith('__'): + if st_.startswith(u'__'): continue ret.append((nid, st_)) return ret @@ -229,38 +241,38 @@ def format_log(ret): ''' Format the state into a log message ''' - msg = '' + msg = u'' if isinstance(ret, dict): # Looks like the ret may be a valid state return - if 'changes' in ret: + if u'changes' in ret: # Yep, looks like a valid state return - chg = ret['changes'] + chg = ret[u'changes'] if not chg: - if ret['comment']: - msg = ret['comment'] + if ret[u'comment']: + msg = ret[u'comment'] else: - msg = 'No changes made for {0[name]}'.format(ret) + msg = u'No changes made for {0[name]}'.format(ret) elif isinstance(chg, dict): - if 'diff' in chg: - if isinstance(chg['diff'], six.string_types): - msg = 'File changed:\n{0}'.format(chg['diff']) + if u'diff' in chg: + if isinstance(chg[u'diff'], six.string_types): + msg = u'File changed:\n{0}'.format(chg[u'diff']) if all([isinstance(x, dict) for x in six.itervalues(chg)]): - if all([('old' in x and 'new' in x) + if all([(u'old' in x and u'new' in x) for x in six.itervalues(chg)]): - msg = 'Made the following changes:\n' + msg = u'Made the following changes:\n' for pkg in chg: - old = chg[pkg]['old'] + old = chg[pkg][u'old'] if not old and old not in (False, None): - old = 'absent' - new = chg[pkg]['new'] + old = u'absent' + new = chg[pkg][u'new'] if not new and new not in (False, None): - new = 'absent' + new = u'absent' # This must be able to handle unicode as some package names contain # non-ascii characters like "Français" or "Español". See Issue #33605. msg += u'\'{0}\' changed from \'{1}\' to \'{2}\'\n'.format(pkg, old, new) if not msg: - msg = str(ret['changes']) - if ret['result'] is True or ret['result'] is None: + msg = str(ret[u'changes']) + if ret[u'result'] is True or ret[u'result'] is None: log.info(msg) else: log.error(msg) @@ -292,14 +304,14 @@ def mock_ret(cdata): ''' # As this is expanded it should be sent into the execution module # layer or it should be turned into a standalone loader system - if cdata['args']: - name = cdata['args'][0] + if cdata[u'args']: + name = cdata[u'args'][0] else: - name = cdata['kwargs']['name'] - return {'name': name, - 'comment': 'Not called, mocked', - 'changes': {}, - 'result': True} + name = cdata[u'kwargs'][u'name'] + return {u'name': name, + u'comment': u'Not called, mocked', + u'changes': {}, + u'result': True} class StateError(Exception): @@ -323,9 +335,9 @@ class Compiler(object): ''' high = compile_template(template, self.rend, - self.opts['renderer'], - self.opts['renderer_blacklist'], - self.opts['renderer_whitelist'], + self.opts[u'renderer'], + self.opts[u'renderer_blacklist'], + self.opts[u'renderer_whitelist'], **kwargs) if not high: return high @@ -339,11 +351,11 @@ class Compiler(object): if not isinstance(high[name], dict): if isinstance(high[name], six.string_types): # Is this is a short state? It needs to be padded! - if '.' in high[name]: - comps = high[name].split('.') + if u'.' in high[name]: + comps = high[name].split(u'.') if len(comps) >= 2: # Merge the comps - comps[1] = '.'.join(comps[1:len(comps)]) + comps[1] = u'.'.join(comps[1:len(comps)]) high[name] = { # '__sls__': template, # '__env__': None, @@ -353,15 +365,15 @@ class Compiler(object): continue skeys = set() for key in sorted(high[name]): - if key.startswith('_'): + if key.startswith(u'_'): continue if not isinstance(high[name][key], list): continue - if '.' in key: - comps = key.split('.') + if u'.' in key: + comps = key.split(u'.') if len(comps) >= 2: # Merge the comps - comps[1] = '.'.join(comps[1:len(comps)]) + comps[1] = u'.'.join(comps[1:len(comps)]) # Salt doesn't support state files such as: # # /etc/redis/redis.conf: @@ -386,49 +398,49 @@ class Compiler(object): ''' errors = [] if not isinstance(high, dict): - errors.append('High data is not a dictionary and is invalid') + errors.append(u'High data is not a dictionary and is invalid') reqs = OrderedDict() for name, body in six.iteritems(high): - if name.startswith('__'): + if name.startswith(u'__'): continue if not isinstance(name, six.string_types): errors.append( - 'ID \'{0}\' in SLS \'{1}\' is not formed as a string, but ' - 'is a {2}'.format( + u'ID \'{0}\' in SLS \'{1}\' is not formed as a string, but ' + u'is a {2}'.format( name, - body['__sls__'], + body[u'__sls__'], type(name).__name__ ) ) if not isinstance(body, dict): - err = ('The type {0} in {1} is not formatted as a dictionary' + err = (u'The type {0} in {1} is not formatted as a dictionary' .format(name, body)) errors.append(err) continue for state in body: - if state.startswith('__'): + if state.startswith(u'__'): continue if not isinstance(body[state], list): errors.append( - 'State \'{0}\' in SLS \'{1}\' is not formed as a list' - .format(name, body['__sls__']) + u'State \'{0}\' in SLS \'{1}\' is not formed as a list' + .format(name, body[u'__sls__']) ) else: fun = 0 - if '.' in state: + if u'.' in state: fun += 1 for arg in body[state]: if isinstance(arg, six.string_types): fun += 1 - if ' ' in arg.strip(): - errors.append(('The function "{0}" in state ' - '"{1}" in SLS "{2}" has ' - 'whitespace, a function with whitespace is ' - 'not supported, perhaps this is an argument ' - 'that is missing a ":"').format( + if u' ' in arg.strip(): + errors.append((u'The function "{0}" in state ' + u'"{1}" in SLS "{2}" has ' + u'whitespace, a function with whitespace is ' + u'not supported, perhaps this is an argument ' + u'that is missing a ":"').format( arg, name, - body['__sls__'])) + body[u'__sls__'])) elif isinstance(arg, dict): # The arg is a dict, if the arg is require or # watch, it must be a list. @@ -436,52 +448,52 @@ class Compiler(object): # Add the requires to the reqs dict and check them # all for recursive requisites. argfirst = next(iter(arg)) - if argfirst in ('require', 'watch', 'prereq', 'onchanges'): + if argfirst in (u'require', u'watch', u'prereq', u'onchanges'): if not isinstance(arg[argfirst], list): - errors.append(('The {0}' - ' statement in state \'{1}\' in SLS \'{2}\' ' - 'needs to be formed as a list').format( + errors.append((u'The {0}' + u' statement in state \'{1}\' in SLS \'{2}\' ' + u'needs to be formed as a list').format( argfirst, name, - body['__sls__'] + body[u'__sls__'] )) # It is a list, verify that the members of the # list are all single key dicts. else: - reqs[name] = {'state': state} + reqs[name] = {u'state': state} for req in arg[argfirst]: if isinstance(req, six.string_types): - req = {'id': req} + req = {u'id': req} if not isinstance(req, dict): - err = ('Requisite declaration {0}' - ' in SLS {1} is not formed as a' - ' single key dictionary').format( + err = (u'Requisite declaration {0}' + u' in SLS {1} is not formed as a' + u' single key dictionary').format( req, - body['__sls__']) + body[u'__sls__']) errors.append(err) continue req_key = next(iter(req)) req_val = req[req_key] - if '.' in req_key: + if u'.' in req_key: errors.append(( - 'Invalid requisite type \'{0}\' ' - 'in state \'{1}\', in SLS ' - '\'{2}\'. Requisite types must ' - 'not contain dots, did you ' - 'mean \'{3}\'?'.format( + u'Invalid requisite type \'{0}\' ' + u'in state \'{1}\', in SLS ' + u'\'{2}\'. Requisite types must ' + u'not contain dots, did you ' + u'mean \'{3}\'?'.format( req_key, name, - body['__sls__'], - req_key[:req_key.find('.')] + body[u'__sls__'], + req_key[:req_key.find(u'.')] ) )) if not ishashable(req_val): errors.append(( - 'Illegal requisite "{0}", ' - 'is SLS {1}\n' + u'Illegal requisite "{0}", ' + u'is SLS {1}\n' ).format( str(req_val), - body['__sls__'])) + body[u'__sls__'])) continue # Check for global recursive requisites @@ -492,12 +504,12 @@ class Compiler(object): if req_val in reqs: if name in reqs[req_val]: if reqs[req_val][name] == state: - if reqs[req_val]['state'] == reqs[name][req_val]: - err = ('A recursive ' - 'requisite was found, SLS ' - '"{0}" ID "{1}" ID "{2}"' + if reqs[req_val][u'state'] == reqs[name][req_val]: + err = (u'A recursive ' + u'requisite was found, SLS ' + u'"{0}" ID "{1}" ID "{2}"' ).format( - body['__sls__'], + body[u'__sls__'], name, req_val ) @@ -505,20 +517,20 @@ class Compiler(object): # Make sure that there is only one key in the # dict if len(list(arg)) != 1: - errors.append(('Multiple dictionaries ' - 'defined in argument of state \'{0}\' in SLS' - ' \'{1}\'').format( + errors.append((u'Multiple dictionaries ' + u'defined in argument of state \'{0}\' in SLS' + u' \'{1}\'').format( name, - body['__sls__'])) + body[u'__sls__'])) if not fun: - if state == 'require' or state == 'watch': + if state == u'require' or state == u'watch': continue - errors.append(('No function declared in state \'{0}\' in' - ' SLS \'{1}\'').format(state, body['__sls__'])) + errors.append((u'No function declared in state \'{0}\' in' + u' SLS \'{1}\'').format(state, body[u'__sls__'])) elif fun > 1: errors.append( - 'Too many functions declared in state \'{0}\' in ' - 'SLS \'{1}\''.format(state, body['__sls__']) + u'Too many functions declared in state \'{0}\' in ' + u'SLS \'{1}\''.format(state, body[u'__sls__']) ) return errors @@ -529,31 +541,31 @@ class Compiler(object): ''' cap = 1 for chunk in chunks: - if 'order' in chunk: - if not isinstance(chunk['order'], int): + if u'order' in chunk: + if not isinstance(chunk[u'order'], int): continue - chunk_order = chunk['order'] + chunk_order = chunk[u'order'] if chunk_order > cap - 1 and chunk_order > 0: cap = chunk_order + 100 for chunk in chunks: - if 'order' not in chunk: - chunk['order'] = cap + if u'order' not in chunk: + chunk[u'order'] = cap continue - if not isinstance(chunk['order'], (int, float)): - if chunk['order'] == 'last': - chunk['order'] = cap + 1000000 - elif chunk['order'] == 'first': - chunk['order'] = 0 + if not isinstance(chunk[u'order'], (int, float)): + if chunk[u'order'] == u'last': + chunk[u'order'] = cap + 1000000 + elif chunk[u'order'] == u'first': + chunk[u'order'] = 0 else: - chunk['order'] = cap - if 'name_order' in chunk: - chunk['order'] = chunk['order'] + chunk.pop('name_order') / 10000.0 - if chunk['order'] < 0: - chunk['order'] = cap + 1000000 + chunk['order'] - chunk['name'] = sdecode(chunk['name']) - chunks.sort(key=lambda chunk: (chunk['order'], '{0[state]}{0[name]}{0[fun]}'.format(chunk))) + chunk[u'order'] = cap + if u'name_order' in chunk: + chunk[u'order'] = chunk[u'order'] + chunk.pop(u'name_order') / 10000.0 + if chunk[u'order'] < 0: + chunk[u'order'] = cap + 1000000 + chunk[u'order'] + chunk[u'name'] = sdecode(chunk[u'name']) + chunks.sort(key=lambda chunk: (chunk[u'order'], u'{0[state]}{0[name]}{0[fun]}'.format(chunk))) return chunks def compile_high_data(self, high): @@ -563,27 +575,27 @@ class Compiler(object): ''' chunks = [] for name, body in six.iteritems(high): - if name.startswith('__'): + if name.startswith(u'__'): continue for state, run in six.iteritems(body): funcs = set() names = [] - if state.startswith('__'): + if state.startswith(u'__'): continue - chunk = {'state': state, - 'name': name} - if '__sls__' in body: - chunk['__sls__'] = body['__sls__'] - if '__env__' in body: - chunk['__env__'] = body['__env__'] - chunk['__id__'] = name + chunk = {u'state': state, + u'name': name} + if u'__sls__' in body: + chunk[u'__sls__'] = body[u'__sls__'] + if u'__env__' in body: + chunk[u'__env__'] = body[u'__env__'] + chunk[u'__id__'] = name for arg in run: if isinstance(arg, six.string_types): funcs.add(arg) continue if isinstance(arg, dict): for key, val in six.iteritems(arg): - if key == 'names': + if key == u'names': for _name in val: if _name not in names: names.append(_name) @@ -596,19 +608,19 @@ class Compiler(object): live = copy.deepcopy(chunk) if isinstance(entry, dict): low_name = next(six.iterkeys(entry)) - live['name'] = low_name + live[u'name'] = low_name list(map(live.update, entry[low_name])) else: - live['name'] = entry - live['name_order'] = name_order + live[u'name'] = entry + live[u'name_order'] = name_order name_order = name_order + 1 for fun in funcs: - live['fun'] = fun + live[u'fun'] = fun chunks.append(live) else: live = copy.deepcopy(chunk) for fun in funcs: - live['fun'] = fun + live[u'fun'] = fun chunks.append(live) chunks = self.order_chunks(chunks) return chunks @@ -618,13 +630,13 @@ class Compiler(object): Read in the __exclude__ list and remove all excluded objects from the high data ''' - if '__exclude__' not in high: + if u'__exclude__' not in high: return high ex_sls = set() ex_id = set() - exclude = high.pop('__exclude__') + exclude = high.pop(u'__exclude__') for exc in exclude: - if isinstance(exc, str): + if isinstance(exc, six.string_types): # The exclude statement is a string, assume it is an sls ex_sls.add(exc) if isinstance(exc, dict): @@ -632,17 +644,17 @@ class Compiler(object): if len(exc) != 1: continue key = next(six.iterkeys(exc)) - if key == 'sls': - ex_sls.add(exc['sls']) - elif key == 'id': - ex_id.add(exc['id']) + if key == u'sls': + ex_sls.add(exc[u'sls']) + elif key == u'id': + ex_id.add(exc[u'id']) # Now the excludes have been simplified, use them if ex_sls: # There are sls excludes, find the associtaed ids for name, body in six.iteritems(high): - if name.startswith('__'): + if name.startswith(u'__'): continue - if body.get('__sls__', '') in ex_sls: + if body.get(u'__sls__', u'') in ex_sls: ex_id.add(name) for id_ in ex_id: if id_ in high: @@ -657,26 +669,37 @@ class State(object): def __init__( self, opts, - pillar=None, + pillar_override=None, jid=None, pillar_enc=None, proxy=None, context=None, mocked=False, - loader='states'): + loader=u'states', + initial_pillar=None): self.states_loader = loader - if 'grains' not in opts: - opts['grains'] = salt.loader.grains(opts) + if u'grains' not in opts: + opts[u'grains'] = salt.loader.grains(opts) self.opts = opts self.proxy = proxy - self._pillar_override = pillar + self._pillar_override = pillar_override if pillar_enc is not None: try: pillar_enc = pillar_enc.lower() except AttributeError: pillar_enc = str(pillar_enc).lower() self._pillar_enc = pillar_enc - self.opts['pillar'] = self._gather_pillar() + if initial_pillar is not None: + self.opts[u'pillar'] = initial_pillar + if self._pillar_override: + self.opts[u'pillar'] = salt.utils.dictupdate.merge( + self.opts[u'pillar'], + self._pillar_override, + self.opts.get(u'pillar_source_merging_strategy', u'smart'), + self.opts.get(u'renderer', u'yaml'), + self.opts.get(u'pillar_merge_lists', False)) + else: + self.opts[u'pillar'] = self._gather_pillar() self.state_con = context or {} self.load_modules() self.active = set() @@ -699,11 +722,11 @@ class State(object): self._pillar_override, self._pillar_enc, translate_newlines=True, - renderers=getattr(self, 'rend', None), + renderers=getattr(self, u'rend', None), opts=self.opts, - valid_rend=self.opts['decrypt_pillar_renderers']) + valid_rend=self.opts[u'decrypt_pillar_renderers']) except Exception as exc: - log.error('Failed to decrypt pillar override: %s', exc) + log.error(u'Failed to decrypt pillar override: %s', exc) if isinstance(self._pillar_override, six.string_types): # This can happen if an entire pillar dictionary was passed as @@ -715,20 +738,20 @@ class State(object): self._pillar_override, Loader=yamlloader.SaltYamlSafeLoader) except Exception as exc: - log.error('Failed to load CLI pillar override') + log.error(u'Failed to load CLI pillar override') log.exception(exc) if not isinstance(self._pillar_override, dict): - log.error('Pillar override was not passed as a dictionary') + log.error(u'Pillar override was not passed as a dictionary') self._pillar_override = None pillar = salt.pillar.get_pillar( self.opts, - self.opts['grains'], - self.opts['id'], - self.opts['environment'], - pillar=self._pillar_override, - pillarenv=self.opts.get('pillarenv')) + self.opts[u'grains'], + self.opts[u'id'], + self.opts[u'environment'], + pillar_override=self._pillar_override, + pillarenv=self.opts.get(u'pillarenv')) return pillar.compile_pillar() def _mod_init(self, low): @@ -739,86 +762,86 @@ class State(object): ''' # ensure that the module is loaded try: - self.states['{0}.{1}'.format(low['state'], low['fun'])] # pylint: disable=W0106 + self.states[u'{0}.{1}'.format(low[u'state'], low[u'fun'])] # pylint: disable=W0106 except KeyError: return - minit = '{0}.mod_init'.format(low['state']) - if low['state'] not in self.mod_init: + minit = u'{0}.mod_init'.format(low[u'state']) + if low[u'state'] not in self.mod_init: if minit in self.states._dict: mret = self.states[minit](low) if not mret: return - self.mod_init.add(low['state']) + self.mod_init.add(low[u'state']) def _mod_aggregate(self, low, running, chunks): ''' Execute the aggregation systems to runtime modify the low chunk ''' - agg_opt = self.functions['config.option']('state_aggregate') - if 'aggregate' in low: - agg_opt = low['aggregate'] + agg_opt = self.functions[u'config.option'](u'state_aggregate') + if u'aggregate' in low: + agg_opt = low[u'aggregate'] if agg_opt is True: - agg_opt = [low['state']] - else: + agg_opt = [low[u'state']] + elif not isinstance(agg_opt, list): return low - if low['state'] in agg_opt and not low.get('__agg__'): - agg_fun = '{0}.mod_aggregate'.format(low['state']) + if low[u'state'] in agg_opt and not low.get(u'__agg__'): + agg_fun = u'{0}.mod_aggregate'.format(low[u'state']) if agg_fun in self.states: try: low = self.states[agg_fun](low, chunks, running) - low['__agg__'] = True + low[u'__agg__'] = True except TypeError: - log.error('Failed to execute aggregate for state {0}'.format(low['state'])) + log.error(u'Failed to execute aggregate for state %s', low[u'state']) return low def _run_check(self, low_data): ''' Check that unless doesn't return 0, and that onlyif returns a 0. ''' - ret = {'result': False} + ret = {u'result': False} cmd_opts = {} - if 'shell' in self.opts['grains']: - cmd_opts['shell'] = self.opts['grains'].get('shell') - if 'onlyif' in low_data: - if not isinstance(low_data['onlyif'], list): - low_data_onlyif = [low_data['onlyif']] + if u'shell' in self.opts[u'grains']: + cmd_opts[u'shell'] = self.opts[u'grains'].get(u'shell') + if u'onlyif' in low_data: + if not isinstance(low_data[u'onlyif'], list): + low_data_onlyif = [low_data[u'onlyif']] else: - low_data_onlyif = low_data['onlyif'] + low_data_onlyif = low_data[u'onlyif'] for entry in low_data_onlyif: if not isinstance(entry, six.string_types): - ret.update({'comment': 'onlyif execution failed, bad type passed', 'result': False}) + ret.update({u'comment': u'onlyif execution failed, bad type passed', u'result': False}) return ret - cmd = self.functions['cmd.retcode']( + cmd = self.functions[u'cmd.retcode']( entry, ignore_retcode=True, python_shell=True, **cmd_opts) - log.debug('Last command return code: {0}'.format(cmd)) - if cmd != 0 and ret['result'] is False: - ret.update({'comment': 'onlyif execution failed', - 'skip_watch': True, - 'result': True}) + log.debug(u'Last command return code: %s', cmd) + if cmd != 0 and ret[u'result'] is False: + ret.update({u'comment': u'onlyif execution failed', + u'skip_watch': True, + u'result': True}) return ret elif cmd == 0: - ret.update({'comment': 'onlyif execution succeeded', 'result': False}) + ret.update({u'comment': u'onlyif execution succeeded', u'result': False}) return ret - if 'unless' in low_data: - if not isinstance(low_data['unless'], list): - low_data_unless = [low_data['unless']] + if u'unless' in low_data: + if not isinstance(low_data[u'unless'], list): + low_data_unless = [low_data[u'unless']] else: - low_data_unless = low_data['unless'] + low_data_unless = low_data[u'unless'] for entry in low_data_unless: if not isinstance(entry, six.string_types): - ret.update({'comment': 'unless execution failed, bad type passed', 'result': False}) + ret.update({u'comment': u'unless execution failed, bad type passed', u'result': False}) return ret - cmd = self.functions['cmd.retcode']( + cmd = self.functions[u'cmd.retcode']( entry, ignore_retcode=True, python_shell=True, **cmd_opts) - log.debug('Last command return code: {0}'.format(cmd)) - if cmd == 0 and ret['result'] is False: - ret.update({'comment': 'unless execution succeeded', - 'skip_watch': True, - 'result': True}) + log.debug(u'Last command return code: %s', cmd) + if cmd == 0 and ret[u'result'] is False: + ret.update({u'comment': u'unless execution succeeded', + u'skip_watch': True, + u'result': True}) elif cmd != 0: - ret.update({'comment': 'unless execution failed', 'result': False}) + ret.update({u'comment': u'unless execution failed', u'result': False}) return ret # No reason to stop, return ret @@ -828,18 +851,18 @@ class State(object): ''' Alter the way a successful state run is determined ''' - ret = {'result': False} + ret = {u'result': False} cmd_opts = {} - if 'shell' in self.opts['grains']: - cmd_opts['shell'] = self.opts['grains'].get('shell') - for entry in low_data['check_cmd']: - cmd = self.functions['cmd.retcode']( + if u'shell' in self.opts[u'grains']: + cmd_opts[u'shell'] = self.opts[u'grains'].get(u'shell') + for entry in low_data[u'check_cmd']: + cmd = self.functions[u'cmd.retcode']( entry, ignore_retcode=True, python_shell=True, **cmd_opts) - log.debug('Last command return code: {0}'.format(cmd)) - if cmd == 0 and ret['result'] is False: - ret.update({'comment': 'check_cmd determined the state succeeded', 'result': True}) + log.debug(u'Last command return code: %s', cmd) + if cmd == 0 and ret[u'result'] is False: + ret.update({u'comment': u'check_cmd determined the state succeeded', u'result': True}) elif cmd != 0: - ret.update({'comment': 'check_cmd determined the state failed', 'result': False}) + ret.update({u'comment': u'check_cmd determined the state failed', u'result': False}) return ret return ret @@ -853,7 +876,7 @@ class State(object): ''' Read the state loader value and loadup the correct states subsystem ''' - if self.states_loader == 'thorium': + if self.states_loader == u'thorium': self.states = salt.loader.thorium(self.opts, self.functions, {}) # TODO: Add runners, proxy? else: self.states = salt.loader.states(self.opts, self.functions, self.utils, @@ -863,17 +886,17 @@ class State(object): ''' Load the modules into the state ''' - log.info('Loading fresh modules for state activity') + log.info(u'Loading fresh modules for state activity') self.utils = salt.loader.utils(self.opts) self.functions = salt.loader.minion_mods(self.opts, self.state_con, utils=self.utils, proxy=self.proxy) if isinstance(data, dict): - if data.get('provider', False): - if isinstance(data['provider'], str): - providers = [{data['state']: data['provider']}] - elif isinstance(data['provider'], list): - providers = data['provider'] + if data.get(u'provider', False): + if isinstance(data[u'provider'], six.string_types): + providers = [{data[u'state']: data[u'provider']}] + elif isinstance(data[u'provider'], list): + providers = data[u'provider'] else: providers = {} for provider in providers: @@ -883,33 +906,34 @@ class State(object): self.functions) if funcs: for func in funcs: - f_key = '{0}{1}'.format( + f_key = u'{0}{1}'.format( mod, - func[func.rindex('.'):] + func[func.rindex(u'.'):] ) self.functions[f_key] = funcs[func] self.serializers = salt.loader.serializers(self.opts) self._load_states() - self.rend = salt.loader.render(self.opts, self.functions, states=self.states) + self.rend = salt.loader.render(self.opts, self.functions, + states=self.states, proxy=self.proxy) def module_refresh(self): ''' Refresh all the modules ''' - log.debug('Refreshing modules...') - if self.opts['grains'].get('os') != 'MacOS': + log.debug(u'Refreshing modules...') + if self.opts[u'grains'].get(u'os') != u'MacOS': # In case a package has been installed into the current python # process 'site-packages', the 'site' module needs to be reloaded in # order for the newly installed package to be importable. try: reload_module(site) except RuntimeError: - log.error('Error encountered during module reload. Modules were not reloaded.') + log.error(u'Error encountered during module reload. Modules were not reloaded.') except TypeError: - log.error('Error encountered during module reload. Modules were not reloaded.') + log.error(u'Error encountered during module reload. Modules were not reloaded.') self.load_modules() - if not self.opts.get('local', False) and self.opts.get('multiprocessing', True): - self.functions['saltutil.refresh_modules']() + if not self.opts.get(u'local', False) and self.opts.get(u'multiprocessing', True): + self.functions[u'saltutil.refresh_modules']() def check_refresh(self, data, ret): ''' @@ -920,93 +944,136 @@ class State(object): function is recurse, since that can lay down anything. ''' _reload_modules = False - if data.get('reload_grains', False): - log.debug('Refreshing grains...') - self.opts['grains'] = salt.loader.grains(self.opts) + if data.get(u'reload_grains', False): + log.debug(u'Refreshing grains...') + self.opts[u'grains'] = salt.loader.grains(self.opts) _reload_modules = True - if data.get('reload_pillar', False): - log.debug('Refreshing pillar...') - self.opts['pillar'] = self._gather_pillar() + if data.get(u'reload_pillar', False): + log.debug(u'Refreshing pillar...') + self.opts[u'pillar'] = self._gather_pillar() _reload_modules = True - if not ret['changes']: - if data.get('force_reload_modules', False): + if not ret[u'changes']: + if data.get(u'force_reload_modules', False): self.module_refresh() return - if data.get('reload_modules', False) or _reload_modules: + if data.get(u'reload_modules', False) or _reload_modules: # User explicitly requests a reload self.module_refresh() return - if data['state'] == 'file': - if data['fun'] == 'managed': - if data['name'].endswith( - ('.py', '.pyx', '.pyo', '.pyc', '.so')): + if data[u'state'] == u'file': + if data[u'fun'] == u'managed': + if data[u'name'].endswith( + (u'.py', u'.pyx', u'.pyo', u'.pyc', u'.so')): self.module_refresh() - elif data['fun'] == 'recurse': + elif data[u'fun'] == u'recurse': self.module_refresh() - elif data['fun'] == 'symlink': - if 'bin' in data['name']: + elif data[u'fun'] == u'symlink': + if u'bin' in data[u'name']: self.module_refresh() - elif data['state'] in ('pkg', 'ports'): + elif data[u'state'] in (u'pkg', u'ports'): self.module_refresh() - def verify_ret(self, ret): + @staticmethod + def verify_ret(ret): ''' - Verify the state return data + Perform basic verification of the raw state return data ''' if not isinstance(ret, dict): raise SaltException( - 'Malformed state return, return must be a dict' - ) + u'Malformed state return, return must be a dict' + ) bad = [] - for val in ['name', 'result', 'changes', 'comment']: + for val in [u'name', u'result', u'changes', u'comment']: if val not in ret: bad.append(val) if bad: - raise SaltException( - 'The following keys were not present in the state ' - 'return: {0}'.format(','.join(bad))) + m = u'The following keys were not present in the state return: {0}' + raise SaltException(m.format(u','.join(bad))) + + @staticmethod + def munge_ret_for_export(ret): + ''' + Process raw state return data to make it suitable for export, + to ensure consistency of the data format seen by external systems + ''' + # We support lists of strings for ret['comment'] internal + # to the state system for improved ergonomics. + # However, to maintain backwards compatability with external tools, + # the list representation is not allowed to leave the state system, + # and should be converted like this at external boundaries. + if isinstance(ret[u'comment'], list): + ret[u'comment'] = u'\n'.join(ret[u'comment']) + + @staticmethod + def verify_ret_for_export(ret): + ''' + Verify the state return data for export outside the state system + ''' + State.verify_ret(ret) + + for key in [u'name', u'comment']: + if not isinstance(ret[key], six.string_types): + msg = ( + u'The value for the {0} key in the state return ' + u'must be a string, found {1}' + ) + raise SaltException(msg.format(key, repr(ret[key]))) + + if ret[u'result'] not in [True, False, None]: + msg = ( + u'The value for the result key in the state return ' + u'must be True, False, or None, found {0}' + ) + raise SaltException(msg.format(repr(ret[u'result']))) + + if not isinstance(ret[u'changes'], dict): + msg = ( + u'The value for the changes key in the state return ' + u'must be a dict, found {0}' + ) + raise SaltException(msg.format(repr(ret[u'changes']))) def verify_data(self, data): ''' Verify the data, return an error statement if something is wrong ''' errors = [] - if 'state' not in data: - errors.append('Missing "state" data') - if 'fun' not in data: - errors.append('Missing "fun" data') - if 'name' not in data: - errors.append('Missing "name" data') - if data['name'] and not isinstance(data['name'], six.string_types): + if u'state' not in data: + errors.append(u'Missing "state" data') + if u'fun' not in data: + errors.append(u'Missing "fun" data') + if u'name' not in data: + errors.append(u'Missing "name" data') + if data[u'name'] and not isinstance(data[u'name'], six.string_types): errors.append( - 'ID \'{0}\' {1}is not formed as a string, but is a {2}'.format( - data['name'], - 'in SLS \'{0}\' '.format(data['__sls__']) - if '__sls__' in data else '', - type(data['name']).__name__ + u'ID \'{0}\' {1}is not formed as a string, but is a {2}'.format( + data[u'name'], + u'in SLS \'{0}\' '.format(data[u'__sls__']) + if u'__sls__' in data else u'', + type(data[u'name']).__name__ ) ) if errors: return errors - full = data['state'] + '.' + data['fun'] + full = data[u'state'] + u'.' + data[u'fun'] if full not in self.states: - if '__sls__' in data: + if u'__sls__' in data: errors.append( - 'State \'{0}\' was not found in SLS \'{1}\''.format( + u'State \'{0}\' was not found in SLS \'{1}\''.format( full, - data['__sls__'] + data[u'__sls__'] ) ) reason = self.states.missing_fun_string(full) if reason: - errors.append('Reason: {0}'.format(reason)) + errors.append(u'Reason: {0}'.format(reason)) else: errors.append( - 'Specified state \'{0}\' was not found'.format( + u'Specified state \'{0}\' was not found'.format( full ) ) @@ -1022,41 +1089,41 @@ class State(object): for ind in range(arglen - deflen): if aspec.args[ind] not in data: errors.append( - 'Missing parameter {0} for state {1}'.format( + u'Missing parameter {0} for state {1}'.format( aspec.args[ind], full ) ) # If this chunk has a recursive require, then it will cause a # recursive loop when executing, check for it - reqdec = '' - if 'require' in data: - reqdec = 'require' - if 'watch' in data: + reqdec = u'' + if u'require' in data: + reqdec = u'require' + if u'watch' in data: # Check to see if the service has a mod_watch function, if it does # not, then just require # to just require extend the require statement with the contents # of watch so that the mod_watch function is not called and the # requisite capability is still used - if '{0}.mod_watch'.format(data['state']) not in self.states: - if 'require' in data: - data['require'].extend(data.pop('watch')) + if u'{0}.mod_watch'.format(data[u'state']) not in self.states: + if u'require' in data: + data[u'require'].extend(data.pop(u'watch')) else: - data['require'] = data.pop('watch') - reqdec = 'require' + data[u'require'] = data.pop(u'watch') + reqdec = u'require' else: - reqdec = 'watch' + reqdec = u'watch' if reqdec: for req in data[reqdec]: reqfirst = next(iter(req)) - if data['state'] == reqfirst: - if (fnmatch.fnmatch(data['name'], req[reqfirst]) - or fnmatch.fnmatch(data['__id__'], req[reqfirst])): - err = ('Recursive require detected in SLS {0} for' - ' require {1} in ID {2}').format( - data['__sls__'], + if data[u'state'] == reqfirst: + if (fnmatch.fnmatch(data[u'name'], req[reqfirst]) + or fnmatch.fnmatch(data[u'__id__'], req[reqfirst])): + err = (u'Recursive require detected in SLS {0} for' + u' require {1} in ID {2}').format( + data[u'__sls__'], req, - data['__id__']) + data[u'__id__']) errors.append(err) return errors @@ -1066,57 +1133,57 @@ class State(object): ''' errors = [] if not isinstance(high, dict): - errors.append('High data is not a dictionary and is invalid') + errors.append(u'High data is not a dictionary and is invalid') reqs = OrderedDict() for name, body in six.iteritems(high): try: - if name.startswith('__'): + if name.startswith(u'__'): continue except AttributeError: pass if not isinstance(name, six.string_types): errors.append( - 'ID \'{0}\' in SLS \'{1}\' is not formed as a string, but ' - 'is a {2}. It may need to be quoted.'.format( - name, body['__sls__'], type(name).__name__) + u'ID \'{0}\' in SLS \'{1}\' is not formed as a string, but ' + u'is a {2}. It may need to be quoted.'.format( + name, body[u'__sls__'], type(name).__name__) ) if not isinstance(body, dict): - err = ('The type {0} in {1} is not formatted as a dictionary' + err = (u'The type {0} in {1} is not formatted as a dictionary' .format(name, body)) errors.append(err) continue for state in body: - if state.startswith('__'): + if state.startswith(u'__'): continue if body[state] is None: errors.append( - 'ID \'{0}\' in SLS \'{1}\' contains a short declaration ' - '({2}) with a trailing colon. When not passing any ' - 'arguments to a state, the colon must be omitted.' - .format(name, body['__sls__'], state) + u'ID \'{0}\' in SLS \'{1}\' contains a short declaration ' + u'({2}) with a trailing colon. When not passing any ' + u'arguments to a state, the colon must be omitted.' + .format(name, body[u'__sls__'], state) ) continue if not isinstance(body[state], list): errors.append( - 'State \'{0}\' in SLS \'{1}\' is not formed as a list' - .format(name, body['__sls__']) + u'State \'{0}\' in SLS \'{1}\' is not formed as a list' + .format(name, body[u'__sls__']) ) else: fun = 0 - if '.' in state: + if u'.' in state: fun += 1 for arg in body[state]: if isinstance(arg, six.string_types): fun += 1 - if ' ' in arg.strip(): - errors.append(('The function "{0}" in state ' - '"{1}" in SLS "{2}" has ' - 'whitespace, a function with whitespace is ' - 'not supported, perhaps this is an argument ' - 'that is missing a ":"').format( + if u' ' in arg.strip(): + errors.append((u'The function "{0}" in state ' + u'"{1}" in SLS "{2}" has ' + u'whitespace, a function with whitespace is ' + u'not supported, perhaps this is an argument ' + u'that is missing a ":"').format( arg, name, - body['__sls__'])) + body[u'__sls__'])) elif isinstance(arg, dict): # The arg is a dict, if the arg is require or # watch, it must be a list. @@ -1124,22 +1191,22 @@ class State(object): # Add the requires to the reqs dict and check them # all for recursive requisites. argfirst = next(iter(arg)) - if argfirst == 'names': + if argfirst == u'names': if not isinstance(arg[argfirst], list): errors.append( - 'The \'names\' argument in state ' - '\'{0}\' in SLS \'{1}\' needs to be ' - 'formed as a list' - .format(name, body['__sls__']) + u'The \'names\' argument in state ' + u'\'{0}\' in SLS \'{1}\' needs to be ' + u'formed as a list' + .format(name, body[u'__sls__']) ) - if argfirst in ('require', 'watch', 'prereq', 'onchanges'): + if argfirst in (u'require', u'watch', u'prereq', u'onchanges'): if not isinstance(arg[argfirst], list): errors.append( - 'The {0} statement in state \'{1}\' in ' - 'SLS \'{2}\' needs to be formed as a ' - 'list'.format(argfirst, + u'The {0} statement in state \'{1}\' in ' + u'SLS \'{2}\' needs to be formed as a ' + u'list'.format(argfirst, name, - body['__sls__']) + body[u'__sls__']) ) # It is a list, verify that the members of the # list are all single key dicts. @@ -1147,34 +1214,34 @@ class State(object): reqs[name] = OrderedDict(state=state) for req in arg[argfirst]: if isinstance(req, six.string_types): - req = {'id': req} + req = {u'id': req} if not isinstance(req, dict): - err = ('Requisite declaration {0}' - ' in SLS {1} is not formed as a' - ' single key dictionary').format( + err = (u'Requisite declaration {0}' + u' in SLS {1} is not formed as a' + u' single key dictionary').format( req, - body['__sls__']) + body[u'__sls__']) errors.append(err) continue req_key = next(iter(req)) req_val = req[req_key] - if '.' in req_key: + if u'.' in req_key: errors.append( - 'Invalid requisite type \'{0}\' ' - 'in state \'{1}\', in SLS ' - '\'{2}\'. Requisite types must ' - 'not contain dots, did you ' - 'mean \'{3}\'?'.format( + u'Invalid requisite type \'{0}\' ' + u'in state \'{1}\', in SLS ' + u'\'{2}\'. Requisite types must ' + u'not contain dots, did you ' + u'mean \'{3}\'?'.format( req_key, name, - body['__sls__'], - req_key[:req_key.find('.')] + body[u'__sls__'], + req_key[:req_key.find(u'.')] ) ) if not ishashable(req_val): errors.append(( - 'Illegal requisite "{0}", ' - 'please check your syntax.\n' + u'Illegal requisite "{0}", ' + u'please check your syntax.\n' ).format(req_val)) continue @@ -1186,12 +1253,12 @@ class State(object): if req_val in reqs: if name in reqs[req_val]: if reqs[req_val][name] == state: - if reqs[req_val]['state'] == reqs[name][req_val]: - err = ('A recursive ' - 'requisite was found, SLS ' - '"{0}" ID "{1}" ID "{2}"' + if reqs[req_val][u'state'] == reqs[name][req_val]: + err = (u'A recursive ' + u'requisite was found, SLS ' + u'"{0}" ID "{1}" ID "{2}"' ).format( - body['__sls__'], + body[u'__sls__'], name, req_val ) @@ -1200,21 +1267,21 @@ class State(object): # dict if len(list(arg)) != 1: errors.append( - 'Multiple dictionaries defined in ' - 'argument of state \'{0}\' in SLS \'{1}\'' - .format(name, body['__sls__']) + u'Multiple dictionaries defined in ' + u'argument of state \'{0}\' in SLS \'{1}\'' + .format(name, body[u'__sls__']) ) if not fun: - if state == 'require' or state == 'watch': + if state == u'require' or state == u'watch': continue errors.append( - 'No function declared in state \'{0}\' in SLS \'{1}\'' - .format(state, body['__sls__']) + u'No function declared in state \'{0}\' in SLS \'{1}\'' + .format(state, body[u'__sls__']) ) elif fun > 1: errors.append( - 'Too many functions declared in state \'{0}\' in ' - 'SLS \'{1}\''.format(state, body['__sls__']) + u'Too many functions declared in state \'{0}\' in ' + u'SLS \'{1}\''.format(state, body[u'__sls__']) ) return errors @@ -1234,30 +1301,30 @@ class State(object): ''' cap = 1 for chunk in chunks: - if 'order' in chunk: - if not isinstance(chunk['order'], int): + if u'order' in chunk: + if not isinstance(chunk[u'order'], int): continue - chunk_order = chunk['order'] + chunk_order = chunk[u'order'] if chunk_order > cap - 1 and chunk_order > 0: cap = chunk_order + 100 for chunk in chunks: - if 'order' not in chunk: - chunk['order'] = cap + if u'order' not in chunk: + chunk[u'order'] = cap continue - if not isinstance(chunk['order'], (int, float)): - if chunk['order'] == 'last': - chunk['order'] = cap + 1000000 - elif chunk['order'] == 'first': - chunk['order'] = 0 + if not isinstance(chunk[u'order'], (int, float)): + if chunk[u'order'] == u'last': + chunk[u'order'] = cap + 1000000 + elif chunk[u'order'] == u'first': + chunk[u'order'] = 0 else: - chunk['order'] = cap - if 'name_order' in chunk: - chunk['order'] = chunk['order'] + chunk.pop('name_order') / 10000.0 - if chunk['order'] < 0: - chunk['order'] = cap + 1000000 + chunk['order'] - chunks.sort(key=lambda chunk: (chunk['order'], '{0[state]}{0[name]}{0[fun]}'.format(chunk))) + chunk[u'order'] = cap + if u'name_order' in chunk: + chunk[u'order'] = chunk[u'order'] + chunk.pop(u'name_order') / 10000.0 + if chunk[u'order'] < 0: + chunk[u'order'] = cap + 1000000 + chunk[u'order'] + chunks.sort(key=lambda chunk: (chunk[u'order'], u'{0[state]}{0[name]}{0[fun]}'.format(chunk))) return chunks def compile_high_data(self, high, orchestration_jid=None): @@ -1267,36 +1334,36 @@ class State(object): ''' chunks = [] for name, body in six.iteritems(high): - if name.startswith('__'): + if name.startswith(u'__'): continue for state, run in six.iteritems(body): funcs = set() names = [] - if state.startswith('__'): + if state.startswith(u'__'): continue - chunk = {'state': state, - 'name': name} + chunk = {u'state': state, + u'name': name} if orchestration_jid is not None: - chunk['__orchestration_jid__'] = orchestration_jid - if '__sls__' in body: - chunk['__sls__'] = body['__sls__'] - if '__env__' in body: - chunk['__env__'] = body['__env__'] - chunk['__id__'] = name + chunk[u'__orchestration_jid__'] = orchestration_jid + if u'__sls__' in body: + chunk[u'__sls__'] = body[u'__sls__'] + if u'__env__' in body: + chunk[u'__env__'] = body[u'__env__'] + chunk[u'__id__'] = name for arg in run: if isinstance(arg, six.string_types): funcs.add(arg) continue if isinstance(arg, dict): for key, val in six.iteritems(arg): - if key == 'names': + if key == u'names': for _name in val: if _name not in names: names.append(_name) - elif key == 'state': + elif key == u'state': # Don't pass down a state override continue - elif (key == 'name' and + elif (key == u'name' and not isinstance(val, six.string_types)): # Invalid name, fall back to ID chunk[key] = name @@ -1308,19 +1375,19 @@ class State(object): live = copy.deepcopy(chunk) if isinstance(entry, dict): low_name = next(six.iterkeys(entry)) - live['name'] = low_name + live[u'name'] = low_name list(map(live.update, entry[low_name])) else: - live['name'] = entry - live['name_order'] = name_order + live[u'name'] = entry + live[u'name_order'] = name_order name_order += 1 for fun in funcs: - live['fun'] = fun + live[u'fun'] = fun chunks.append(live) else: live = copy.deepcopy(chunk) for fun in funcs: - live['fun'] = fun + live[u'fun'] = fun chunks.append(live) chunks = self.order_chunks(chunks) return chunks @@ -1330,35 +1397,35 @@ class State(object): Pull the extend data and add it to the respective high data ''' errors = [] - if '__extend__' not in high: + if u'__extend__' not in high: return high, errors - ext = high.pop('__extend__') + ext = high.pop(u'__extend__') for ext_chunk in ext: for name, body in six.iteritems(ext_chunk): if name not in high: state_type = next( - x for x in body if not x.startswith('__') + x for x in body if not x.startswith(u'__') ) # Check for a matching 'name' override in high data ids = find_name(name, state_type, high) if len(ids) != 1: errors.append( - 'Cannot extend ID \'{0}\' in \'{1}:{2}\'. It is not ' - 'part of the high state.\n' - 'This is likely due to a missing include statement ' - 'or an incorrectly typed ID.\nEnsure that a ' - 'state with an ID of \'{0}\' is available\nin ' - 'environment \'{1}\' and to SLS \'{2}\''.format( + u'Cannot extend ID \'{0}\' in \'{1}:{2}\'. It is not ' + u'part of the high state.\n' + u'This is likely due to a missing include statement ' + u'or an incorrectly typed ID.\nEnsure that a ' + u'state with an ID of \'{0}\' is available\nin ' + u'environment \'{1}\' and to SLS \'{2}\''.format( name, - body.get('__env__', 'base'), - body.get('__sls__', 'base')) + body.get(u'__env__', u'base'), + body.get(u'__sls__', u'base')) ) continue else: name = ids[0][0] for state, run in six.iteritems(body): - if state.startswith('__'): + if state.startswith(u'__'): continue if state not in high[name]: high[name][state] = run @@ -1385,8 +1452,8 @@ class State(object): else: high[name][state][hind] = arg update = True - if (argfirst == 'name' and - next(iter(high[name][state][hind])) == 'names'): + if (argfirst == u'name' and + next(iter(high[name][state][hind])) == u'names'): # If names are overwritten by name use the name high[name][state][hind] = arg if not update: @@ -1398,13 +1465,13 @@ class State(object): Read in the __exclude__ list and remove all excluded objects from the high data ''' - if '__exclude__' not in high: + if u'__exclude__' not in high: return high ex_sls = set() ex_id = set() - exclude = high.pop('__exclude__') + exclude = high.pop(u'__exclude__') for exc in exclude: - if isinstance(exc, str): + if isinstance(exc, six.string_types): # The exclude statement is a string, assume it is an sls ex_sls.add(exc) if isinstance(exc, dict): @@ -1412,17 +1479,17 @@ class State(object): if len(exc) != 1: continue key = next(six.iterkeys(exc)) - if key == 'sls': - ex_sls.add(exc['sls']) - elif key == 'id': - ex_id.add(exc['id']) + if key == u'sls': + ex_sls.add(exc[u'sls']) + elif key == u'id': + ex_id.add(exc[u'id']) # Now the excludes have been simplified, use them if ex_sls: # There are sls excludes, find the associated ids for name, body in six.iteritems(high): - if name.startswith('__'): + if name.startswith(u'__'): continue - sls = body.get('__sls__', '') + sls = body.get(u'__sls__', u'') if not sls: continue for ex_ in ex_sls: @@ -1438,22 +1505,22 @@ class State(object): Extend the data reference with requisite_in arguments ''' req_in = set([ - 'require_in', - 'watch_in', - 'onfail_in', - 'onchanges_in', - 'use', - 'use_in', - 'prereq', - 'prereq_in', + u'require_in', + u'watch_in', + u'onfail_in', + u'onchanges_in', + u'use', + u'use_in', + u'prereq', + u'prereq_in', ]) req_in_all = req_in.union( set([ - 'require', - 'watch', - 'onfail', - 'onfail_stop', - 'onchanges', + u'require', + u'watch', + u'onfail', + u'onfail_stop', + u'onchanges', ])) extend = {} errors = [] @@ -1461,7 +1528,7 @@ class State(object): if not isinstance(body, dict): continue for state, run in six.iteritems(body): - if state.startswith('__'): + if state.startswith(u'__'): continue for arg in run: if isinstance(arg, dict): @@ -1475,7 +1542,7 @@ class State(object): key = next(iter(arg)) if key not in req_in: continue - rkey = key.split('_')[0] + rkey = key.split(u'_')[0] items = arg[key] if isinstance(items, dict): # Formatted as a single req_in @@ -1485,24 +1552,24 @@ class State(object): found = False if name not in extend: extend[name] = OrderedDict() - if '.' in _state: + if u'.' in _state: errors.append( - 'Invalid requisite in {0}: {1} for ' - '{2}, in SLS \'{3}\'. Requisites must ' - 'not contain dots, did you mean \'{4}\'?' + u'Invalid requisite in {0}: {1} for ' + u'{2}, in SLS \'{3}\'. Requisites must ' + u'not contain dots, did you mean \'{4}\'?' .format( rkey, _state, name, - body['__sls__'], - _state[:_state.find('.')] + body[u'__sls__'], + _state[:_state.find(u'.')] ) ) - _state = _state.split(".")[0] + _state = _state.split(u'.')[0] if _state not in extend[name]: extend[name][_state] = [] - extend[name]['__env__'] = body['__env__'] - extend[name]['__sls__'] = body['__sls__'] + extend[name][u'__env__'] = body[u'__env__'] + extend[name][u'__sls__'] = body[u'__sls__'] for ind in range(len(extend[name][_state])): if next(iter( extend[name][_state][ind])) == rkey: @@ -1529,37 +1596,37 @@ class State(object): continue pstate = next(iter(ind)) pname = ind[pstate] - if pstate == 'sls': + if pstate == u'sls': # Expand hinges here hinges = find_sls_ids(pname, high) else: hinges.append((pname, pstate)) - if '.' in pstate: + if u'.' in pstate: errors.append( - 'Invalid requisite in {0}: {1} for ' - '{2}, in SLS \'{3}\'. Requisites must ' - 'not contain dots, did you mean \'{4}\'?' + u'Invalid requisite in {0}: {1} for ' + u'{2}, in SLS \'{3}\'. Requisites must ' + u'not contain dots, did you mean \'{4}\'?' .format( rkey, pstate, pname, - body['__sls__'], - pstate[:pstate.find('.')] + body[u'__sls__'], + pstate[:pstate.find(u'.')] ) ) - pstate = pstate.split(".")[0] + pstate = pstate.split(u".")[0] for tup in hinges: name, _state = tup - if key == 'prereq_in': + if key == u'prereq_in': # Add prerequired to origin if id_ not in extend: extend[id_] = OrderedDict() if state not in extend[id_]: extend[id_][state] = [] extend[id_][state].append( - {'prerequired': [{_state: name}]} + {u'prerequired': [{_state: name}]} ) - if key == 'prereq': + if key == u'prereq': # Add prerequired to prereqs ext_ids = find_name(name, _state, high) for ext_id, _req_state in ext_ids: @@ -1568,10 +1635,10 @@ class State(object): if _req_state not in extend[ext_id]: extend[ext_id][_req_state] = [] extend[ext_id][_req_state].append( - {'prerequired': [{state: id_}]} + {u'prerequired': [{state: id_}]} ) continue - if key == 'use_in': + if key == u'use_in': # Add the running states args to the # use_in states ext_ids = find_name(name, _state, high) @@ -1592,13 +1659,13 @@ class State(object): if next(iter(arg)) in ignore_args: continue # Don't use name or names - if next(six.iterkeys(arg)) == 'name': + if next(six.iterkeys(arg)) == u'name': continue - if next(six.iterkeys(arg)) == 'names': + if next(six.iterkeys(arg)) == u'names': continue extend[ext_id][_req_state].append(arg) continue - if key == 'use': + if key == u'use': # Add the use state's args to the # running state ext_ids = find_name(name, _state, high) @@ -1619,9 +1686,9 @@ class State(object): if next(iter(arg)) in ignore_args: continue # Don't use name or names - if next(six.iterkeys(arg)) == 'name': + if next(six.iterkeys(arg)) == u'name': continue - if next(six.iterkeys(arg)) == 'names': + if next(six.iterkeys(arg)) == u'names': continue extend[id_][state].append(arg) continue @@ -1630,8 +1697,8 @@ class State(object): extend[name] = OrderedDict() if _state not in extend[name]: extend[name][_state] = [] - extend[name]['__env__'] = body['__env__'] - extend[name]['__sls__'] = body['__sls__'] + extend[name][u'__env__'] = body[u'__env__'] + extend[name][u'__sls__'] = body[u'__sls__'] for ind in range(len(extend[name][_state])): if next(iter( extend[name][_state][ind])) == rkey: @@ -1646,9 +1713,9 @@ class State(object): extend[name][_state].append( {rkey: [{state: id_}]} ) - high['__extend__'] = [] + high[u'__extend__'] = [] for key, val in six.iteritems(extend): - high['__extend__'].append({key: val}) + high[u'__extend__'].append({key: val}) req_in_high, req_in_errors = self.reconcile_extend(high) errors.extend(req_in_errors) return req_in_high, errors @@ -1659,8 +1726,8 @@ class State(object): ''' tag = _gen_tag(low) try: - ret = self.states[cdata['full']](*cdata['args'], - **cdata['kwargs']) + ret = self.states[cdata[u'full']](*cdata[u'args'], + **cdata[u'kwargs']) except Exception: trb = traceback.format_exc() # There are a number of possibilities to not have the cdata @@ -1668,21 +1735,21 @@ class State(object): # enough to not raise another KeyError as the name is easily # guessable and fallback in all cases to present the real # exception to the user - if len(cdata['args']) > 0: - name = cdata['args'][0] - elif 'name' in cdata['kwargs']: - name = cdata['kwargs']['name'] + if len(cdata[u'args']) > 0: + name = cdata[u'args'][0] + elif u'name' in cdata[u'kwargs']: + name = cdata[u'kwargs'][u'name'] else: - name = low.get('name', low.get('__id__')) + name = low.get(u'name', low.get(u'__id__')) ret = { - 'result': False, - 'name': name, - 'changes': {}, - 'comment': 'An exception occurred in this state: {0}'.format( + u'result': False, + u'name': name, + u'changes': {}, + u'comment': u'An exception occurred in this state: {0}'.format( trb) } - troot = os.path.join(self.opts['cachedir'], self.jid) - tfile = os.path.join(troot, tag) + troot = os.path.join(self.opts[u'cachedir'], self.jid) + tfile = os.path.join(troot, _clean_tag(tag)) if not os.path.isdir(troot): try: os.makedirs(troot) @@ -1690,7 +1757,7 @@ class State(object): # Looks like the directory was created between the check # and the attempt, we are safe to pass pass - with salt.utils.fopen(tfile, 'wb+') as fp_: + with salt.utils.files.fopen(tfile, u'wb+') as fp_: fp_.write(msgpack.dumps(ret)) def call_parallel(self, cdata, low): @@ -1701,11 +1768,11 @@ class State(object): target=self._call_parallel_target, args=(cdata, low)) proc.start() - ret = {'name': cdata['args'][0], - 'result': None, - 'changes': {}, - 'comment': 'Started in a seperate process', - 'proc': proc} + ret = {u'name': cdata[u'args'][0], + u'result': None, + u'changes': {}, + u'comment': u'Started in a seperate process', + u'proc': proc} return ret def call(self, low, chunks=None, running=None, retries=1): @@ -1715,49 +1782,53 @@ class State(object): ''' utc_start_time = datetime.datetime.utcnow() local_start_time = utc_start_time - (datetime.datetime.utcnow() - datetime.datetime.now()) - log.info('Running state [{0}] at time {1}'.format( - low['name'].strip() if isinstance(low['name'], str) - else low['name'], - local_start_time.time().isoformat()) + log.info(u'Running state [%s] at time %s', + low[u'name'].strip() if isinstance(low[u'name'], six.string_types) + else low[u'name'], + local_start_time.time().isoformat() ) errors = self.verify_data(low) if errors: ret = { - 'result': False, - 'name': low['name'], - 'changes': {}, - 'comment': '', + u'result': False, + u'name': low[u'name'], + u'changes': {}, + u'comment': u'', } for err in errors: - ret['comment'] += '{0}\n'.format(err) - ret['__run_num__'] = self.__run_num + ret[u'comment'] += u'{0}\n'.format(err) + ret[u'__run_num__'] = self.__run_num self.__run_num += 1 format_log(ret) self.check_refresh(low, ret) return ret else: - ret = {'result': False, 'name': low['name'], 'changes': {}} + ret = {u'result': False, u'name': low[u'name'], u'changes': {}} - self.state_con['runas'] = low.get('runas', None) + self.state_con[u'runas'] = low.get(u'runas', None) - if not low.get('__prereq__'): + if low[u'state'] == u'cmd' and u'password' in low: + self.state_con[u'runas_password'] = low[u'password'] + else: + self.state_con[u'runas_password'] = low.get(u'runas_password', None) + + if not low.get(u'__prereq__'): log.info( - 'Executing state {0}.{1} for [{2}]'.format( - low['state'], - low['fun'], - low['name'].strip() if isinstance(low['name'], str) - else low['name'] - ) + u'Executing state %s.%s for [%s]', + low[u'state'], + low[u'fun'], + low[u'name'].strip() if isinstance(low[u'name'], six.string_types) + else low[u'name'] ) - if 'provider' in low: + if u'provider' in low: self.load_modules(low) - state_func_name = '{0[state]}.{0[fun]}'.format(low) + state_func_name = u'{0[state]}.{0[fun]}'.format(low) cdata = salt.utils.format_call( self.states[state_func_name], low, - initial_ret={'full': state_func_name}, + initial_ret={u'full': state_func_name}, expected_extra_kws=STATE_INTERNAL_KEYWORDS ) inject_globals = { @@ -1765,18 +1836,18 @@ class State(object): # the current state dictionaries. # We pass deep copies here because we don't want any misbehaving # state module to change these at runtime. - '__low__': immutabletypes.freeze(low), - '__running__': immutabletypes.freeze(running) if running else {}, - '__instance_id__': self.instance_id, - '__lowstate__': immutabletypes.freeze(chunks) if chunks else {} + u'__low__': immutabletypes.freeze(low), + u'__running__': immutabletypes.freeze(running) if running else {}, + u'__instance_id__': self.instance_id, + u'__lowstate__': immutabletypes.freeze(chunks) if chunks else {} } if self.inject_globals: inject_globals.update(self.inject_globals) - if low.get('__prereq__'): - test = sys.modules[self.states[cdata['full']].__module__].__opts__['test'] - sys.modules[self.states[cdata['full']].__module__].__opts__['test'] = True + if low.get(u'__prereq__'): + test = sys.modules[self.states[cdata[u'full']].__module__].__opts__[u'test'] + sys.modules[self.states[cdata[u'full']].__module__].__opts__[u'test'] = True try: # Let's get a reference to the salt environment to use within this # state call. @@ -1786,47 +1857,48 @@ class State(object): # that's not found in cdata, we look for what we're being passed in # the original data, namely, the special dunder __env__. If that's # not found we default to 'base' - if ('unless' in low and '{0[state]}.mod_run_check'.format(low) not in self.states) or \ - ('onlyif' in low and '{0[state]}.mod_run_check'.format(low) not in self.states): + if (u'unless' in low and u'{0[state]}.mod_run_check'.format(low) not in self.states) or \ + (u'onlyif' in low and u'{0[state]}.mod_run_check'.format(low) not in self.states): ret.update(self._run_check(low)) - if 'saltenv' in low: - inject_globals['__env__'] = str(low['saltenv']) - elif isinstance(cdata['kwargs'].get('env', None), six.string_types): + if u'saltenv' in low: + inject_globals[u'__env__'] = six.text_type(low[u'saltenv']) + elif isinstance(cdata[u'kwargs'].get(u'env', None), six.string_types): # User is using a deprecated env setting which was parsed by # format_call. # We check for a string type since module functions which # allow setting the OS environ also make use of the "env" # keyword argument, which is not a string - inject_globals['__env__'] = str(cdata['kwargs']['env']) - elif '__env__' in low: + inject_globals[u'__env__'] = six.text_type(cdata[u'kwargs'][u'env']) + elif u'__env__' in low: # The user is passing an alternative environment using __env__ # which is also not the appropriate choice, still, handle it - inject_globals['__env__'] = str(low['__env__']) + inject_globals[u'__env__'] = six.text_type(low[u'__env__']) else: # Let's use the default environment - inject_globals['__env__'] = 'base' + inject_globals[u'__env__'] = u'base' - if '__orchestration_jid__' in low: - inject_globals['__orchestration_jid__'] = \ - low['__orchestration_jid__'] + if u'__orchestration_jid__' in low: + inject_globals[u'__orchestration_jid__'] = \ + low[u'__orchestration_jid__'] - if 'result' not in ret or ret['result'] is False: + if u'result' not in ret or ret[u'result'] is False: self.states.inject_globals = inject_globals if self.mocked: ret = mock_ret(cdata) else: # Execute the state function - if not low.get('__prereq__') and low.get('parallel'): + if not low.get(u'__prereq__') and low.get(u'parallel'): # run the state call in parallel, but only if not in a prereq ret = self.call_parallel(cdata, low) else: - ret = self.states[cdata['full']](*cdata['args'], - **cdata['kwargs']) + ret = self.states[cdata[u'full']](*cdata[u'args'], + **cdata[u'kwargs']) self.states.inject_globals = {} - if 'check_cmd' in low and '{0[state]}.mod_run_check_cmd'.format(low) not in self.states: + if u'check_cmd' in low and u'{0[state]}.mod_run_check_cmd'.format(low) not in self.states: ret.update(self._run_check_cmd(low)) self.verify_ret(ret) + self.munge_ret_for_export(ret) except Exception: trb = traceback.format_exc() # There are a number of possibilities to not have the cdata @@ -1834,37 +1906,41 @@ class State(object): # enough to not raise another KeyError as the name is easily # guessable and fallback in all cases to present the real # exception to the user - if len(cdata['args']) > 0: - name = cdata['args'][0] - elif 'name' in cdata['kwargs']: - name = cdata['kwargs']['name'] + if len(cdata[u'args']) > 0: + name = cdata[u'args'][0] + elif u'name' in cdata[u'kwargs']: + name = cdata[u'kwargs'][u'name'] else: - name = low.get('name', low.get('__id__')) + name = low.get(u'name', low.get(u'__id__')) ret = { - 'result': False, - 'name': name, - 'changes': {}, - 'comment': 'An exception occurred in this state: {0}'.format( + u'result': False, + u'name': name, + u'changes': {}, + u'comment': u'An exception occurred in this state: {0}'.format( trb) } finally: - if low.get('__prereq__'): - sys.modules[self.states[cdata['full']].__module__].__opts__[ - 'test'] = test + if low.get(u'__prereq__'): + sys.modules[self.states[cdata[u'full']].__module__].__opts__[ + u'test'] = test + + self.state_con.pop('runas') + self.state_con.pop('runas_password') + self.verify_ret_for_export(ret) # If format_call got any warnings, let's show them to the user - if 'warnings' in cdata: - ret.setdefault('warnings', []).extend(cdata['warnings']) + if u'warnings' in cdata: + ret.setdefault(u'warnings', []).extend(cdata[u'warnings']) - if 'provider' in low: + if u'provider' in low: self.load_modules() - if low.get('__prereq__'): - low['__prereq__'] = False + if low.get(u'__prereq__'): + low[u'__prereq__'] = False return ret - ret['__sls__'] = low.get('__sls__') - ret['__run_num__'] = self.__run_num + ret[u'__sls__'] = low.get(u'__sls__') + ret[u'__run_num__'] = self.__run_num self.__run_num += 1 format_log(ret) self.check_refresh(low, ret) @@ -1872,56 +1948,57 @@ class State(object): timezone_delta = datetime.datetime.utcnow() - datetime.datetime.now() local_finish_time = utc_finish_time - timezone_delta local_start_time = utc_start_time - timezone_delta - ret['start_time'] = local_start_time.time().isoformat() + ret[u'start_time'] = local_start_time.time().isoformat() delta = (utc_finish_time - utc_start_time) # duration in milliseconds.microseconds duration = (delta.seconds * 1000000 + delta.microseconds)/1000.0 - ret['duration'] = duration - ret['__id__'] = low['__id__'] + ret[u'duration'] = duration + ret[u'__id__'] = low[u'__id__'] log.info( - 'Completed state [{0}] at time {1} duration_in_ms={2}'.format( - low['name'].strip() if isinstance(low['name'], str) - else low['name'], - local_finish_time.time().isoformat(), - duration - ) + u'Completed state [%s] at time %s (duration_in_ms=%s)', + low[u'name'].strip() if isinstance(low[u'name'], six.string_types) + else low[u'name'], + local_finish_time.time().isoformat(), + duration ) - if 'retry' in low: - low['retry'] = self.verify_retry_data(low['retry']) - if not sys.modules[self.states[cdata['full']].__module__].__opts__['test']: - if low['retry']['until'] != ret['result']: - if low['retry']['attempts'] > retries: - interval = low['retry']['interval'] - if low['retry']['splay'] != 0: - interval = interval + random.randint(0, low['retry']['splay']) - log.info(('State result does not match retry until value' - ', state will be re-run in {0} seconds'.format(interval))) - self.functions['test.sleep'](interval) + if u'retry' in low: + low[u'retry'] = self.verify_retry_data(low[u'retry']) + if not sys.modules[self.states[cdata[u'full']].__module__].__opts__[u'test']: + if low[u'retry'][u'until'] != ret[u'result']: + if low[u'retry'][u'attempts'] > retries: + interval = low[u'retry'][u'interval'] + if low[u'retry'][u'splay'] != 0: + interval = interval + random.randint(0, low[u'retry'][u'splay']) + log.info( + u'State result does not match retry until value, ' + u'state will be re-run in %s seconds', interval + ) + self.functions[u'test.sleep'](interval) retry_ret = self.call(low, chunks, running, retries=retries+1) orig_ret = ret ret = retry_ret - ret['comment'] = '\n'.join( + ret[u'comment'] = u'\n'.join( [( - 'Attempt {0}: Returned a result of "{1}", ' - 'with the following comment: "{2}"'.format( + u'Attempt {0}: Returned a result of "{1}", ' + u'with the following comment: "{2}"'.format( retries, - orig_ret['result'], - orig_ret['comment']) + orig_ret[u'result'], + orig_ret[u'comment']) ), - '' if not ret['comment'] else ret['comment']]) - ret['duration'] = ret['duration'] + orig_ret['duration'] + (interval * 1000) + u'' if not ret[u'comment'] else ret[u'comment']]) + ret[u'duration'] = ret[u'duration'] + orig_ret[u'duration'] + (interval * 1000) if retries == 1: - ret['start_time'] = orig_ret['start_time'] + ret[u'start_time'] = orig_ret[u'start_time'] else: - ret['comment'] = ' '.join( - ['' if not ret['comment'] else ret['comment'], - ('The state would be retried every {1} seconds ' - '(with a splay of up to {3} seconds) ' - 'a maximum of {0} times or until a result of {2} ' - 'is returned').format(low['retry']['attempts'], - low['retry']['interval'], - low['retry']['until'], - low['retry']['splay'])]) + ret[u'comment'] = u' '.join( + [u'' if not ret[u'comment'] else ret[u'comment'], + (u'The state would be retried every {1} seconds ' + u'(with a splay of up to {3} seconds) ' + u'a maximum of {0} times or until a result of {2} ' + u'is returned').format(low[u'retry'][u'attempts'], + low[u'retry'][u'interval'], + low[u'retry'][u'until'], + low[u'retry'][u'splay'])]) return ret def verify_retry_data(self, retry_data): @@ -1929,16 +2006,16 @@ class State(object): verifies the specified retry data ''' retry_defaults = { - 'until': True, - 'attempts': 2, - 'splay': 0, - 'interval': 30, + u'until': True, + u'attempts': 2, + u'splay': 0, + u'interval': 30, } expected_data = { - 'until': bool, - 'attempts': int, - 'interval': int, - 'splay': int, + u'until': bool, + u'attempts': int, + u'interval': int, + u'splay': int, } validated_retry_data = {} if isinstance(retry_data, dict): @@ -1947,15 +2024,17 @@ class State(object): if isinstance(retry_data[expected_key], value_type): validated_retry_data[expected_key] = retry_data[expected_key] else: - log.warning(('An invalid value was passed for the retry {0}, ' - 'using default value {1}'.format(expected_key, - retry_defaults[expected_key]))) + log.warning( + u'An invalid value was passed for the retry %s, ' + u'using default value \'%s\'', + expected_key, retry_defaults[expected_key] + ) validated_retry_data[expected_key] = retry_defaults[expected_key] else: validated_retry_data[expected_key] = retry_defaults[expected_key] else: - log.warning(('State is set to retry, but a valid dict for retry ' - 'configuration was not found. Using retry defaults')) + log.warning((u'State is set to retry, but a valid dict for retry ' + u'configuration was not found. Using retry defaults')) validated_retry_data = retry_defaults return validated_retry_data @@ -1965,31 +2044,31 @@ class State(object): ''' # Check for any disabled states disabled = {} - if 'state_runs_disabled' in self.opts['grains']: + if u'state_runs_disabled' in self.opts[u'grains']: for low in chunks[:]: - state_ = '{0}.{1}'.format(low['state'], low['fun']) - for pat in self.opts['grains']['state_runs_disabled']: + state_ = u'{0}.{1}'.format(low[u'state'], low[u'fun']) + for pat in self.opts[u'grains'][u'state_runs_disabled']: if fnmatch.fnmatch(state_, pat): comment = ( - 'The state function "{0}" is currently disabled by "{1}", ' - 'to re-enable, run state.enable {1}.' + u'The state function "{0}" is currently disabled by "{1}", ' + u'to re-enable, run state.enable {1}.' ).format( state_, pat, ) _tag = _gen_tag(low) - disabled[_tag] = {'changes': {}, - 'result': False, - 'comment': comment, - '__run_num__': self.__run_num, - '__sls__': low['__sls__']} + disabled[_tag] = {u'changes': {}, + u'result': False, + u'comment': comment, + u'__run_num__': self.__run_num, + u'__sls__': low[u'__sls__']} self.__run_num += 1 chunks.remove(low) break running = {} for low in chunks: - if '__FAILHARD__' in running: - running.pop('__FAILHARD__') + if u'__FAILHARD__' in running: + running.pop(u'__FAILHARD__') return running tag = _gen_tag(low) if tag not in running: @@ -2009,12 +2088,12 @@ class State(object): Check if the low data chunk should send a failhard signal ''' tag = _gen_tag(low) - if self.opts.get('test', False): + if self.opts.get(u'test', False): return False - if (low.get('failhard', False) or self.opts['failhard']) and tag in running: - if running[tag]['result'] is None: + if (low.get(u'failhard', False) or self.opts[u'failhard']) and tag in running: + if running[tag][u'result'] is None: return False - return not running[tag]['result'] + return not running[tag][u'result'] return False def reconcile_procs(self, running): @@ -2023,25 +2102,25 @@ class State(object): ''' retset = set() for tag in running: - proc = running[tag].get('proc') + proc = running[tag].get(u'proc') if proc: if not proc.is_alive(): - ret_cache = os.path.join(self.opts['cachedir'], self.jid, tag) + ret_cache = os.path.join(self.opts[u'cachedir'], self.jid, _clean_tag(tag)) if not os.path.isfile(ret_cache): - ret = {'result': False, - 'comment': 'Parallel process failed to return', - 'name': running[tag]['name'], - 'changes': {}} + ret = {u'result': False, + u'comment': u'Parallel process failed to return', + u'name': running[tag][u'name'], + u'changes': {}} try: - with salt.utils.fopen(ret_cache, 'rb') as fp_: + with salt.utils.files.fopen(ret_cache, u'rb') as fp_: ret = msgpack.loads(fp_.read()) except (OSError, IOError): - ret = {'result': False, - 'comment': 'Parallel cache failure', - 'name': running[tag]['name'], - 'changes': {}} + ret = {u'result': False, + u'comment': u'Parallel cache failure', + u'name': running[tag][u'name'], + u'changes': {}} running[tag].update(ret) - running[tag].pop('proc') + running[tag].pop(u'proc') else: retset.add(False) return False not in retset @@ -2053,40 +2132,40 @@ class State(object): ''' present = False # If mod_watch is not available make it a require - if 'watch' in low: - if '{0}.mod_watch'.format(low['state']) not in self.states: - if 'require' in low: - low['require'].extend(low.pop('watch')) + if u'watch' in low: + if u'{0}.mod_watch'.format(low[u'state']) not in self.states: + if u'require' in low: + low[u'require'].extend(low.pop(u'watch')) else: - low['require'] = low.pop('watch') + low[u'require'] = low.pop(u'watch') else: present = True - if 'require' in low: + if u'require' in low: present = True - if 'prerequired' in low: + if u'prerequired' in low: present = True - if 'prereq' in low: + if u'prereq' in low: present = True - if 'onfail' in low: + if u'onfail' in low: present = True - if 'onchanges' in low: + if u'onchanges' in low: present = True if not present: - return 'met', () + return u'met', () self.reconcile_procs(running) reqs = { - 'require': [], - 'watch': [], - 'prereq': [], - 'onfail': [], - 'onchanges': []} + u'require': [], + u'watch': [], + u'prereq': [], + u'onfail': [], + u'onchanges': []} if pre: - reqs['prerequired'] = [] + reqs[u'prerequired'] = [] for r_state in reqs: if r_state in low and low[r_state] is not None: for req in low[r_state]: if isinstance(req, six.string_types): - req = {'id': req} + req = {u'id': req} req = trim_req(req) found = False for chunk in chunks: @@ -2094,87 +2173,90 @@ class State(object): req_val = req[req_key] if req_val is None: continue - if req_key == 'sls': + if req_key == u'sls': # Allow requisite tracking of entire sls files - if fnmatch.fnmatch(chunk['__sls__'], req_val): + if fnmatch.fnmatch(chunk[u'__sls__'], req_val): found = True reqs[r_state].append(chunk) continue try: - if (fnmatch.fnmatch(chunk['name'], req_val) or - fnmatch.fnmatch(chunk['__id__'], req_val)): - if req_key == 'id' or chunk['state'] == req_key: - found = True - reqs[r_state].append(chunk) + if isinstance(req_val, six.string_types): + if (fnmatch.fnmatch(chunk[u'name'], req_val) or + fnmatch.fnmatch(chunk[u'__id__'], req_val)): + if req_key == u'id' or chunk[u'state'] == req_key: + found = True + reqs[r_state].append(chunk) + else: + raise KeyError except KeyError as exc: raise SaltRenderError( - 'Could not locate requisite of [{0}] present in state with name [{1}]'.format( - req_key, chunk['name'])) + u'Could not locate requisite of [{0}] present in state with name [{1}]'.format( + req_key, chunk[u'name'])) except TypeError: # On Python 2, the above req_val, being an OrderedDict, will raise a KeyError, # however on Python 3 it will raise a TypeError # This was found when running tests.unit.test_state.StateCompilerTestCase.test_render_error_on_invalid_requisite raise SaltRenderError( - 'Could not locate requisite of [{0}] present in state with name [{1}]'.format( - req_key, chunk['name'])) + u'Could not locate requisite of [{0}] present in state with name [{1}]'.format( + req_key, chunk[u'name'])) if not found: - return 'unmet', () + return u'unmet', () fun_stats = set() for r_state, chunks in six.iteritems(reqs): - if r_state == 'prereq': + if r_state == u'prereq': run_dict = self.pre else: run_dict = running for chunk in chunks: tag = _gen_tag(chunk) if tag not in run_dict: - fun_stats.add('unmet') + fun_stats.add(u'unmet') continue - if run_dict[tag].get('proc'): + if run_dict[tag].get(u'proc'): # Run in parallel, first wait for a touch and then recheck time.sleep(0.01) return self.check_requisite(low, running, chunks, pre) - if r_state == 'onfail': - if run_dict[tag]['result'] is True: - fun_stats.add('onfail') # At least one state is OK + if r_state == u'onfail': + if run_dict[tag][u'result'] is True: + fun_stats.add(u'onfail') # At least one state is OK continue else: - if run_dict[tag]['result'] is False: - fun_stats.add('fail') + if run_dict[tag][u'result'] is False: + fun_stats.add(u'fail') continue - if r_state == 'onchanges': - if not run_dict[tag]['changes']: - fun_stats.add('onchanges') + if r_state == u'onchanges': + if not run_dict[tag][u'changes']: + fun_stats.add(u'onchanges') else: - fun_stats.add('onchangesmet') + fun_stats.add(u'onchangesmet') continue - if r_state == 'watch' and run_dict[tag]['changes']: - fun_stats.add('change') + if r_state == u'watch' and run_dict[tag][u'changes']: + fun_stats.add(u'change') continue - if r_state == 'prereq' and run_dict[tag]['result'] is None: - fun_stats.add('premet') - if r_state == 'prereq' and not run_dict[tag]['result'] is None: - fun_stats.add('pre') + if r_state == u'prereq' and run_dict[tag][u'result'] is None: + fun_stats.add(u'premet') + if r_state == u'prereq' and not run_dict[tag][u'result'] is None: + fun_stats.add(u'pre') else: - fun_stats.add('met') + fun_stats.add(u'met') - if 'unmet' in fun_stats: - status = 'unmet' - elif 'fail' in fun_stats: - status = 'fail' - elif 'pre' in fun_stats: - if 'premet' in fun_stats: - status = 'met' + if u'unmet' in fun_stats: + status = u'unmet' + elif u'fail' in fun_stats: + status = u'fail' + elif u'pre' in fun_stats: + if u'premet' in fun_stats: + status = u'met' else: - status = 'pre' - elif 'onfail' in fun_stats and 'met' not in fun_stats: - status = 'onfail' # all onfail states are OK - elif 'onchanges' in fun_stats and 'onchangesmet' not in fun_stats: - status = 'onchanges' - elif 'change' in fun_stats: - status = 'change' + status = u'pre' + elif u'onfail' in fun_stats and u'met' not in fun_stats: + status = u'onfail' # all onfail states are OK + elif u'onchanges' in fun_stats and u'onchangesmet' not in fun_stats: + status = u'onchanges' + elif u'change' in fun_stats: + status = u'change' else: - status = 'met' + status = u'met' return status, reqs @@ -2193,28 +2275,28 @@ class State(object): chunk is evaluated an event will be set up to the master with the results. ''' - if not self.opts.get('local') and (self.opts.get('state_events', True) or fire_event): - if not self.opts.get('master_uri'): + if not self.opts.get(u'local') and (self.opts.get(u'state_events', True) or fire_event): + if not self.opts.get(u'master_uri'): ev_func = lambda ret, tag, preload=None: salt.utils.event.get_master_event( - self.opts, self.opts['sock_dir'], listen=False).fire_event(ret, tag) + self.opts, self.opts[u'sock_dir'], listen=False).fire_event(ret, tag) else: - ev_func = self.functions['event.fire_master'] + ev_func = self.functions[u'event.fire_master'] - ret = {'ret': chunk_ret} + ret = {u'ret': chunk_ret} if fire_event is True: tag = salt.utils.event.tagify( - [self.jid, self.opts['id'], str(chunk_ret['name'])], 'state_result' + [self.jid, self.opts[u'id'], str(chunk_ret[u'name'])], u'state_result' ) elif isinstance(fire_event, six.string_types): tag = salt.utils.event.tagify( - [self.jid, self.opts['id'], str(fire_event)], 'state_result' + [self.jid, self.opts[u'id'], str(fire_event)], u'state_result' ) else: tag = salt.utils.event.tagify( - [self.jid, 'prog', self.opts['id'], str(chunk_ret['__run_num__'])], 'job' + [self.jid, u'prog', self.opts[u'id'], str(chunk_ret[u'__run_num__'])], u'job' ) - ret['len'] = length - preload = {'jid': self.jid} + ret[u'len'] = length + preload = {u'jid': self.jid} ev_func(ret, tag, preload=preload) def call_chunk(self, low, running, chunks): @@ -2225,15 +2307,15 @@ class State(object): low = self._mod_aggregate(low, running, chunks) self._mod_init(low) tag = _gen_tag(low) - if not low.get('prerequired'): + if not low.get(u'prerequired'): self.active.add(tag) - requisites = ['require', 'watch', 'prereq', 'onfail', 'onchanges'] - if not low.get('__prereq__'): - requisites.append('prerequired') + requisites = [u'require', u'watch', u'prereq', u'onfail', u'onchanges'] + if not low.get(u'__prereq__'): + requisites.append(u'prerequired') status, reqs = self.check_requisite(low, running, chunks, pre=True) else: status, reqs = self.check_requisite(low, running, chunks) - if status == 'unmet': + if status == u'unmet': lost = {} reqs = [] for requisite in requisites: @@ -2242,7 +2324,7 @@ class State(object): continue for req in low[requisite]: if isinstance(req, six.string_types): - req = {'id': req} + req = {u'id': req} req = trim_req(req) found = False req_key = next(iter(req)) @@ -2250,44 +2332,50 @@ class State(object): for chunk in chunks: if req_val is None: continue - if req_key == 'sls': + if req_key == u'sls': # Allow requisite tracking of entire sls files - if fnmatch.fnmatch(chunk['__sls__'], req_val): - if requisite == 'prereq': - chunk['__prereq__'] = True + if fnmatch.fnmatch(chunk[u'__sls__'], req_val): + if requisite == u'prereq': + chunk[u'__prereq__'] = True reqs.append(chunk) found = True continue - if (fnmatch.fnmatch(chunk['name'], req_val) or - fnmatch.fnmatch(chunk['__id__'], req_val)): - if req_key == 'id' or chunk['state'] == req_key: - if requisite == 'prereq': - chunk['__prereq__'] = True - elif requisite == 'prerequired': - chunk['__prerequired__'] = True + if (fnmatch.fnmatch(chunk[u'name'], req_val) or + fnmatch.fnmatch(chunk[u'__id__'], req_val)): + if req_key == u'id' or chunk[u'state'] == req_key: + if requisite == u'prereq': + chunk[u'__prereq__'] = True + elif requisite == u'prerequired': + chunk[u'__prerequired__'] = True reqs.append(chunk) found = True if not found: lost[requisite].append(req) - if lost['require'] or lost['watch'] or lost['prereq'] or lost['onfail'] or lost['onchanges'] or lost.get('prerequired'): - comment = 'The following requisites were not found:\n' + if lost[u'require'] or lost[u'watch'] or lost[u'prereq'] \ + or lost[u'onfail'] or lost[u'onchanges'] \ + or lost.get(u'prerequired'): + comment = u'The following requisites were not found:\n' for requisite, lreqs in six.iteritems(lost): if not lreqs: continue comment += \ - '{0}{1}:\n'.format(' ' * 19, requisite) + u'{0}{1}:\n'.format(u' ' * 19, requisite) for lreq in lreqs: req_key = next(iter(lreq)) req_val = lreq[req_key] comment += \ - '{0}{1}: {2}\n'.format(' ' * 23, req_key, req_val) - running[tag] = {'changes': {}, - 'result': False, - 'comment': comment, - '__run_num__': self.__run_num, - '__sls__': low['__sls__']} + u'{0}{1}: {2}\n'.format(u' ' * 23, req_key, req_val) + if low.get('__prereq__'): + run_dict = self.pre + else: + run_dict = running + run_dict[tag] = {u'changes': {}, + u'result': False, + u'comment': comment, + u'__run_num__': self.__run_num, + u'__sls__': low[u'__sls__']} self.__run_num += 1 - self.event(running[tag], len(chunks), fire_event=low.get('fire_event')) + self.event(run_dict[tag], len(chunks), fire_event=low.get(u'fire_event')) return running for chunk in reqs: # Check to see if the chunk has been run, only run it if @@ -2295,52 +2383,52 @@ class State(object): ctag = _gen_tag(chunk) if ctag not in running: if ctag in self.active: - if chunk.get('__prerequired__'): + if chunk.get(u'__prerequired__'): # Prereq recusive, run this chunk with prereq on if tag not in self.pre: - low['__prereq__'] = True + low[u'__prereq__'] = True self.pre[ctag] = self.call(low, chunks, running) return running else: return running elif ctag not in running: - log.error('Recursive requisite found') + log.error(u'Recursive requisite found') running[tag] = { - 'changes': {}, - 'result': False, - 'comment': 'Recursive requisite found', - '__run_num__': self.__run_num, - '__sls__': low['__sls__']} + u'changes': {}, + u'result': False, + u'comment': u'Recursive requisite found', + u'__run_num__': self.__run_num, + u'__sls__': low[u'__sls__']} self.__run_num += 1 - self.event(running[tag], len(chunks), fire_event=low.get('fire_event')) + self.event(running[tag], len(chunks), fire_event=low.get(u'fire_event')) return running running = self.call_chunk(chunk, running, chunks) if self.check_failhard(chunk, running): - running['__FAILHARD__'] = True + running[u'__FAILHARD__'] = True return running - if low.get('__prereq__'): + if low.get(u'__prereq__'): status, reqs = self.check_requisite(low, running, chunks) self.pre[tag] = self.call(low, chunks, running) - if not self.pre[tag]['changes'] and status == 'change': - self.pre[tag]['changes'] = {'watch': 'watch'} - self.pre[tag]['result'] = None + if not self.pre[tag][u'changes'] and status == u'change': + self.pre[tag][u'changes'] = {u'watch': u'watch'} + self.pre[tag][u'result'] = None else: running = self.call_chunk(low, running, chunks) if self.check_failhard(chunk, running): - running['__FAILHARD__'] = True + running[u'__FAILHARD__'] = True return running - elif status == 'met': - if low.get('__prereq__'): + elif status == u'met': + if low.get(u'__prereq__'): self.pre[tag] = self.call(low, chunks, running) else: running[tag] = self.call(low, chunks, running) - elif status == 'fail': + elif status == u'fail': # if the requisite that failed was due to a prereq on this low state # show the normal error if tag in self.pre: running[tag] = self.pre[tag] - running[tag]['__run_num__'] = self.__run_num - running[tag]['__sls__'] = low['__sls__'] + running[tag][u'__run_num__'] = self.__run_num + running[tag][u'__sls__'] = low[u'__sls__'] # otherwise the failure was due to a requisite down the chain else: # determine what the requisite failures where, and return @@ -2356,62 +2444,62 @@ class State(object): if req_ret is None: continue # If the result was False (not None) it was a failure - if req_ret['result'] is False: + if req_ret[u'result'] is False: # use SLS.ID for the key-- so its easier to find - key = '{sls}.{_id}'.format(sls=req_low['__sls__'], - _id=req_low['__id__']) + key = u'{sls}.{_id}'.format(sls=req_low[u'__sls__'], + _id=req_low[u'__id__']) failed_requisites.add(key) - _cmt = 'One or more requisite failed: {0}'.format( - ', '.join(str(i) for i in failed_requisites) + _cmt = u'One or more requisite failed: {0}'.format( + u', '.join(str(i) for i in failed_requisites) ) running[tag] = { - 'changes': {}, - 'result': False, - 'comment': _cmt, - '__run_num__': self.__run_num, - '__sls__': low['__sls__'] + u'changes': {}, + u'result': False, + u'comment': _cmt, + u'__run_num__': self.__run_num, + u'__sls__': low[u'__sls__'] } self.__run_num += 1 - elif status == 'change' and not low.get('__prereq__'): + elif status == u'change' and not low.get(u'__prereq__'): ret = self.call(low, chunks, running) - if not ret['changes'] and not ret.get('skip_watch', False): + if not ret[u'changes'] and not ret.get(u'skip_watch', False): low = low.copy() - low['sfun'] = low['fun'] - low['fun'] = 'mod_watch' - low['__reqs__'] = reqs + low[u'sfun'] = low[u'fun'] + low[u'fun'] = u'mod_watch' + low[u'__reqs__'] = reqs ret = self.call(low, chunks, running) running[tag] = ret - elif status == 'pre': - pre_ret = {'changes': {}, - 'result': True, - 'comment': 'No changes detected', - '__run_num__': self.__run_num, - '__sls__': low['__sls__']} + elif status == u'pre': + pre_ret = {u'changes': {}, + u'result': True, + u'comment': u'No changes detected', + u'__run_num__': self.__run_num, + u'__sls__': low[u'__sls__']} running[tag] = pre_ret self.pre[tag] = pre_ret self.__run_num += 1 - elif status == 'onfail': - running[tag] = {'changes': {}, - 'result': True, - 'comment': 'State was not run because onfail req did not change', - '__run_num__': self.__run_num, - '__sls__': low['__sls__']} + elif status == u'onfail': + running[tag] = {u'changes': {}, + u'result': True, + u'comment': u'State was not run because onfail req did not change', + u'__run_num__': self.__run_num, + u'__sls__': low[u'__sls__']} self.__run_num += 1 - elif status == 'onchanges': - running[tag] = {'changes': {}, - 'result': True, - 'comment': 'State was not run because none of the onchanges reqs changed', - '__run_num__': self.__run_num, - '__sls__': low['__sls__']} + elif status == u'onchanges': + running[tag] = {u'changes': {}, + u'result': True, + u'comment': u'State was not run because none of the onchanges reqs changed', + u'__run_num__': self.__run_num, + u'__sls__': low[u'__sls__']} self.__run_num += 1 else: - if low.get('__prereq__'): + if low.get(u'__prereq__'): self.pre[tag] = self.call(low, chunks, running) else: running[tag] = self.call(low, chunks, running) if tag in running: - self.event(running[tag], len(chunks), fire_event=low.get('fire_event')) + self.event(running[tag], len(chunks), fire_event=low.get(u'fire_event')) return running def call_listen(self, chunks, running): @@ -2421,14 +2509,14 @@ class State(object): listeners = [] crefs = {} for chunk in chunks: - crefs[(chunk['state'], chunk['name'])] = chunk - crefs[(chunk['state'], chunk['__id__'])] = chunk - if 'listen' in chunk: - listeners.append({(chunk['state'], chunk['__id__']): chunk['listen']}) - if 'listen_in' in chunk: - for l_in in chunk['listen_in']: + crefs[(chunk[u'state'], chunk[u'name'])] = chunk + crefs[(chunk[u'state'], chunk[u'__id__'])] = chunk + if u'listen' in chunk: + listeners.append({(chunk[u'state'], chunk[u'__id__']): chunk[u'listen']}) + if u'listen_in' in chunk: + for l_in in chunk[u'listen_in']: for key, val in six.iteritems(l_in): - listeners.append({(key, val): [{chunk['state']: chunk['__id__']}]}) + listeners.append({(key, val): [{chunk[u'state']: chunk[u'__id__']}]}) mod_watchers = [] errors = {} for l_dict in listeners: @@ -2440,30 +2528,30 @@ class State(object): if (lkey, lval) not in crefs: rerror = {_l_tag(lkey, lval): { - 'comment': 'Referenced state {0}: {1} does not exist'.format(lkey, lval), - 'name': 'listen_{0}:{1}'.format(lkey, lval), - 'result': False, - 'changes': {} + u'comment': u'Referenced state {0}: {1} does not exist'.format(lkey, lval), + u'name': u'listen_{0}:{1}'.format(lkey, lval), + u'result': False, + u'changes': {} }} errors.update(rerror) continue to_tag = _gen_tag(crefs[(lkey, lval)]) if to_tag not in running: continue - if running[to_tag]['changes']: + if running[to_tag][u'changes']: if key not in crefs: rerror = {_l_tag(key[0], key[1]): - {'comment': 'Referenced state {0}: {1} does not exist'.format(key[0], key[1]), - 'name': 'listen_{0}:{1}'.format(key[0], key[1]), - 'result': False, - 'changes': {}}} + {u'comment': u'Referenced state {0}: {1} does not exist'.format(key[0], key[1]), + u'name': u'listen_{0}:{1}'.format(key[0], key[1]), + u'result': False, + u'changes': {}}} errors.update(rerror) continue chunk = crefs[key] low = chunk.copy() - low['sfun'] = chunk['fun'] - low['fun'] = 'mod_watch' - low['__id__'] = 'listener_{0}'.format(low['__id__']) + low[u'sfun'] = chunk[u'fun'] + low[u'fun'] = u'mod_watch' + low[u'__id__'] = u'listener_{0}'.format(low[u'__id__']) for req in STATE_REQUISITE_KEYWORDS: if req in low: low.pop(req) @@ -2471,7 +2559,7 @@ class State(object): ret = self.call_chunks(mod_watchers) running.update(ret) for err in errors: - errors[err]['__run_num__'] = self.__run_num + errors[err][u'__run_num__'] = self.__run_num self.__run_num += 1 running.update(errors) return running @@ -2505,18 +2593,14 @@ class State(object): def _cleanup_accumulator_data(): accum_data_path = os.path.join( - salt.utils.get_accumulator_dir(self.opts['cachedir']), + salt.utils.get_accumulator_dir(self.opts[u'cachedir']), self.instance_id ) try: os.remove(accum_data_path) - log.debug('Deleted accumulator data file {0}'.format( - accum_data_path) - ) + log.debug(u'Deleted accumulator data file %s', accum_data_path) except OSError: - log.debug('File {0} does not exist, no need to cleanup.'.format( - accum_data_path) - ) + log.debug(u'File %s does not exist, no need to cleanup', accum_data_path) _cleanup_accumulator_data() return ret @@ -2528,16 +2612,16 @@ class State(object): if not isinstance(high, dict): errors.append( - 'Template {0} does not render to a dictionary'.format(template) + u'Template {0} does not render to a dictionary'.format(template) ) return high, errors - invalid_items = ('include', 'exclude', 'extends') + invalid_items = (u'include', u'exclude', u'extends') for item in invalid_items: if item in high: errors.append( - 'The \'{0}\' declaration found on \'{1}\' is invalid when ' - 'rendering single templates'.format(item, template) + u'The \'{0}\' declaration found on \'{1}\' is invalid when ' + u'rendering single templates'.format(item, template) ) return high, errors @@ -2545,8 +2629,8 @@ class State(object): if not isinstance(high[name], dict): if isinstance(high[name], six.string_types): # Is this is a short state, it needs to be padded - if '.' in high[name]: - comps = high[name].split('.') + if u'.' in high[name]: + comps = high[name].split(u'.') high[name] = { # '__sls__': template, # '__env__': None, @@ -2555,27 +2639,27 @@ class State(object): continue errors.append( - 'ID {0} in template {1} is not a dictionary'.format( + u'ID {0} in template {1} is not a dictionary'.format( name, template ) ) continue skeys = set() for key in sorted(high[name]): - if key.startswith('_'): + if key.startswith(u'_'): continue if high[name][key] is None: errors.append( - 'ID \'{0}\' in template {1} contains a short ' - 'declaration ({2}) with a trailing colon. When not ' - 'passing any arguments to a state, the colon must be ' - 'omitted.'.format(name, template, key) + u'ID \'{0}\' in template {1} contains a short ' + u'declaration ({2}) with a trailing colon. When not ' + u'passing any arguments to a state, the colon must be ' + u'omitted.'.format(name, template, key) ) continue if not isinstance(high[name][key], list): continue - if '.' in key: - comps = key.split('.') + if u'.' in key: + comps = key.split(u'.') # Salt doesn't support state files such as: # # /etc/redis/redis.conf: @@ -2587,8 +2671,8 @@ class State(object): # - regex: ^requirepass if comps[0] in skeys: errors.append( - 'ID \'{0}\' in template \'{1}\' contains multiple ' - 'state declarations of the same type' + u'ID \'{0}\' in template \'{1}\' contains multiple ' + u'state declarations of the same type' .format(name, template) ) continue @@ -2606,9 +2690,9 @@ class State(object): ''' high = compile_template(template, self.rend, - self.opts['renderer'], - self.opts['renderer_blacklist'], - self.opts['renderer_whitelist']) + self.opts[u'renderer'], + self.opts[u'renderer_blacklist'], + self.opts[u'renderer_whitelist']) if not high: return high high, errors = self.render_template(high, template) @@ -2622,12 +2706,12 @@ class State(object): ''' high = compile_template_str(template, self.rend, - self.opts['renderer'], - self.opts['renderer_blacklist'], - self.opts['renderer_whitelist']) + self.opts[u'renderer'], + self.opts[u'renderer_blacklist'], + self.opts[u'renderer_whitelist']) if not high: return high - high, errors = self.render_template(high, '') + high, errors = self.render_template(high, u'') if errors: return errors return self.call_high(high) @@ -2666,51 +2750,51 @@ class BaseHighState(object): # If the state is intended to be applied locally, then the local opts # should have all of the needed data, otherwise overwrite the local # data items with data from the master - if 'local_state' in opts: - if opts['local_state']: + if u'local_state' in opts: + if opts[u'local_state']: return opts mopts = self.client.master_opts() if not isinstance(mopts, dict): # An error happened on the master - opts['renderer'] = 'yaml_jinja' - opts['failhard'] = False - opts['state_top'] = salt.utils.url.create('top.sls') - opts['nodegroups'] = {} - opts['file_roots'] = {'base': [syspaths.BASE_FILE_ROOTS_DIR]} + opts[u'renderer'] = u'yaml_jinja' + opts[u'failhard'] = False + opts[u'state_top'] = salt.utils.url.create(u'top.sls') + opts[u'nodegroups'] = {} + opts[u'file_roots'] = {u'base': [syspaths.BASE_FILE_ROOTS_DIR]} else: - opts['renderer'] = mopts['renderer'] - opts['failhard'] = mopts.get('failhard', False) - if mopts['state_top'].startswith('salt://'): - opts['state_top'] = mopts['state_top'] - elif mopts['state_top'].startswith('/'): - opts['state_top'] = salt.utils.url.create(mopts['state_top'][1:]) + opts[u'renderer'] = mopts[u'renderer'] + opts[u'failhard'] = mopts.get(u'failhard', False) + if mopts[u'state_top'].startswith(u'salt://'): + opts[u'state_top'] = mopts[u'state_top'] + elif mopts[u'state_top'].startswith(u'/'): + opts[u'state_top'] = salt.utils.url.create(mopts[u'state_top'][1:]) else: - opts['state_top'] = salt.utils.url.create(mopts['state_top']) - opts['state_top_saltenv'] = mopts.get('state_top_saltenv', None) - opts['nodegroups'] = mopts.get('nodegroups', {}) - opts['state_auto_order'] = mopts.get( - 'state_auto_order', - opts['state_auto_order']) - opts['file_roots'] = mopts['file_roots'] - opts['top_file_merging_strategy'] = mopts.get('top_file_merging_strategy', - opts.get('top_file_merging_strategy')) - opts['env_order'] = mopts.get('env_order', opts.get('env_order', [])) - opts['default_top'] = mopts.get('default_top', opts.get('default_top')) - opts['state_events'] = mopts.get('state_events') - opts['state_aggregate'] = mopts.get('state_aggregate', opts.get('state_aggregate', False)) - opts['jinja_lstrip_blocks'] = mopts.get('jinja_lstrip_blocks', False) - opts['jinja_trim_blocks'] = mopts.get('jinja_trim_blocks', False) + opts[u'state_top'] = salt.utils.url.create(mopts[u'state_top']) + opts[u'state_top_saltenv'] = mopts.get(u'state_top_saltenv', None) + opts[u'nodegroups'] = mopts.get(u'nodegroups', {}) + opts[u'state_auto_order'] = mopts.get( + u'state_auto_order', + opts[u'state_auto_order']) + opts[u'file_roots'] = mopts[u'file_roots'] + opts[u'top_file_merging_strategy'] = mopts.get(u'top_file_merging_strategy', + opts.get(u'top_file_merging_strategy')) + opts[u'env_order'] = mopts.get(u'env_order', opts.get(u'env_order', [])) + opts[u'default_top'] = mopts.get(u'default_top', opts.get(u'default_top')) + opts[u'state_events'] = mopts.get(u'state_events') + opts[u'state_aggregate'] = mopts.get(u'state_aggregate', opts.get(u'state_aggregate', False)) + opts[u'jinja_lstrip_blocks'] = mopts.get(u'jinja_lstrip_blocks', False) + opts[u'jinja_trim_blocks'] = mopts.get(u'jinja_trim_blocks', False) return opts def _get_envs(self): ''' Pull the file server environments out of the master options ''' - envs = ['base'] - if 'file_roots' in self.opts: - envs.extend([x for x in list(self.opts['file_roots']) + envs = [u'base'] + if u'file_roots' in self.opts: + envs.extend([x for x in list(self.opts[u'file_roots']) if x not in envs]) - env_order = self.opts.get('env_order', []) + env_order = self.opts.get(u'env_order', []) # Remove duplicates while preserving the order members = set() env_order = [env for env in env_order if not (env in members or members.add(env))] @@ -2733,37 +2817,37 @@ class BaseHighState(object): done = DefaultOrderedDict(list) found = 0 # did we find any contents in the top files? # Gather initial top files - merging_strategy = self.opts['top_file_merging_strategy'] - if merging_strategy == 'same' and not self.opts['environment']: - if not self.opts['default_top']: + merging_strategy = self.opts[u'top_file_merging_strategy'] + if merging_strategy == u'same' and not self.opts[u'environment']: + if not self.opts[u'default_top']: raise SaltRenderError( - 'top_file_merging_strategy set to \'same\', but no ' - 'default_top configuration option was set' + u'top_file_merging_strategy set to \'same\', but no ' + u'default_top configuration option was set' ) - if self.opts['environment']: + if self.opts[u'environment']: contents = self.client.cache_file( - self.opts['state_top'], - self.opts['environment'] + self.opts[u'state_top'], + self.opts[u'environment'] ) if contents: found = 1 - tops[self.opts['environment']] = [ + tops[self.opts[u'environment']] = [ compile_template( contents, self.state.rend, - self.state.opts['renderer'], - self.state.opts['renderer_blacklist'], - self.state.opts['renderer_whitelist'], - saltenv=self.opts['environment'] + self.state.opts[u'renderer'], + self.state.opts[u'renderer_blacklist'], + self.state.opts[u'renderer_whitelist'], + saltenv=self.opts[u'environment'] ) ] else: - tops[self.opts['environment']] = [{}] + tops[self.opts[u'environment']] = [{}] else: found = 0 - state_top_saltenv = self.opts.get('state_top_saltenv', False) + state_top_saltenv = self.opts.get(u'state_top_saltenv', False) if state_top_saltenv \ and not isinstance(state_top_saltenv, six.string_types): state_top_saltenv = str(state_top_saltenv) @@ -2771,7 +2855,7 @@ class BaseHighState(object): for saltenv in [state_top_saltenv] if state_top_saltenv \ else self._get_envs(): contents = self.client.cache_file( - self.opts['state_top'], + self.opts[u'state_top'], saltenv ) if contents: @@ -2780,40 +2864,42 @@ class BaseHighState(object): compile_template( contents, self.state.rend, - self.state.opts['renderer'], - self.state.opts['renderer_blacklist'], - self.state.opts['renderer_whitelist'], + self.state.opts[u'renderer'], + self.state.opts[u'renderer_blacklist'], + self.state.opts[u'renderer_whitelist'], saltenv=saltenv ) ) else: tops[saltenv].append({}) - log.debug('No contents loaded for env: {0}'.format(saltenv)) + log.debug(u'No contents loaded for saltenv \'%s\'', saltenv) - if found > 1 and merging_strategy == 'merge' and not self.opts.get('env_order', None): + if found > 1 and merging_strategy == u'merge' and not self.opts.get(u'env_order', None): log.warning( - 'top_file_merging_strategy is set to \'%s\' and ' - 'multiple top files were found. Merging order is not ' - 'deterministic, it may be desirable to either set ' - 'top_file_merging_strategy to \'same\' or use the ' - '\'env_order\' configuration parameter to specify the ' - 'merging order.', merging_strategy + u'top_file_merging_strategy is set to \'%s\' and ' + u'multiple top files were found. Merging order is not ' + u'deterministic, it may be desirable to either set ' + u'top_file_merging_strategy to \'same\' or use the ' + u'\'env_order\' configuration parameter to specify the ' + u'merging order.', merging_strategy ) if found == 0: - log.error('No contents found in top file. Please verify ' - 'that the \'file_roots\' specified in \'etc/master\' are ' - 'accessible: {0}'.format(repr(self.state.opts['file_roots'])) + log.debug( + u'No contents found in top file. If this is not expected, ' + u'verify that the \'file_roots\' specified in \'etc/master\' ' + u'are accessible. The \'file_roots\' configuration is: %s', + repr(self.state.opts[u'file_roots']) ) # Search initial top files for includes for saltenv, ctops in six.iteritems(tops): for ctop in ctops: - if 'include' not in ctop: + if u'include' not in ctop: continue - for sls in ctop['include']: + for sls in ctop[u'include']: include[saltenv].append(sls) - ctop.pop('include') + ctop.pop(u'include') # Go through the includes and pull out the extra tops and add them while include: pops = [] @@ -2830,11 +2916,11 @@ class BaseHighState(object): self.client.get_state( sls, saltenv - ).get('dest', False), + ).get(u'dest', False), self.state.rend, - self.state.opts['renderer'], - self.state.opts['renderer_blacklist'], - self.state.opts['renderer_whitelist'], + self.state.opts[u'renderer'], + self.state.opts[u'renderer_blacklist'], + self.state.opts[u'renderer_whitelist'], saltenv ) ) @@ -2848,18 +2934,18 @@ class BaseHighState(object): ''' Cleanly merge the top files ''' - merging_strategy = self.opts['top_file_merging_strategy'] + merging_strategy = self.opts[u'top_file_merging_strategy'] try: - merge_attr = '_merge_tops_{0}'.format(merging_strategy) + merge_attr = u'_merge_tops_{0}'.format(merging_strategy) merge_func = getattr(self, merge_attr) - if not hasattr(merge_func, '__call__'): - msg = '\'{0}\' is not callable'.format(merge_attr) + if not hasattr(merge_func, u'__call__'): + msg = u'\'{0}\' is not callable'.format(merge_attr) log.error(msg) raise TypeError(msg) except (AttributeError, TypeError): log.warning( - 'Invalid top_file_merging_strategy \'%s\', falling back to ' - '\'merge\'', merging_strategy + u'Invalid top_file_merging_strategy \'%s\', falling back to ' + u'\'merge\'', merging_strategy ) merge_func = self._merge_tops_merge return merge_func(tops) @@ -2875,44 +2961,44 @@ class BaseHighState(object): top = DefaultOrderedDict(OrderedDict) # Check base env first as it is authoritative - base_tops = tops.pop('base', DefaultOrderedDict(OrderedDict)) + base_tops = tops.pop(u'base', DefaultOrderedDict(OrderedDict)) for ctop in base_tops: for saltenv, targets in six.iteritems(ctop): - if saltenv == 'include': + if saltenv == u'include': continue try: for tgt in targets: top[saltenv][tgt] = ctop[saltenv][tgt] except TypeError: - raise SaltRenderError('Unable to render top file. No targets found.') + raise SaltRenderError(u'Unable to render top file. No targets found.') for cenv, ctops in six.iteritems(tops): for ctop in ctops: for saltenv, targets in six.iteritems(ctop): - if saltenv == 'include': + if saltenv == u'include': continue elif saltenv != cenv: log.debug( - 'Section for saltenv \'%s\' in the \'%s\' ' - 'saltenv\'s top file will be ignored, as the ' - 'top_file_merging_strategy is set to \'merge\' ' - 'and the saltenvs do not match', + u'Section for saltenv \'%s\' in the \'%s\' ' + u'saltenv\'s top file will be ignored, as the ' + u'top_file_merging_strategy is set to \'merge\' ' + u'and the saltenvs do not match', saltenv, cenv ) continue elif saltenv in top: log.debug( - 'Section for saltenv \'%s\' in the \'%s\' ' - 'saltenv\'s top file will be ignored, as this ' - 'saltenv was already defined in the \'base\' top ' - 'file', saltenv, cenv + u'Section for saltenv \'%s\' in the \'%s\' ' + u'saltenv\'s top file will be ignored, as this ' + u'saltenv was already defined in the \'base\' top ' + u'file', saltenv, cenv ) continue try: for tgt in targets: top[saltenv][tgt] = ctop[saltenv][tgt] except TypeError: - raise SaltRenderError('Unable to render top file. No targets found.') + raise SaltRenderError(u'Unable to render top file. No targets found.') return top def _merge_tops_same(self, tops): @@ -2925,14 +3011,14 @@ class BaseHighState(object): for cenv, ctops in six.iteritems(tops): if all([x == {} for x in ctops]): # No top file found in this env, check the default_top - default_top = self.opts['default_top'] + default_top = self.opts[u'default_top'] fallback_tops = tops.get(default_top, []) if all([x == {} for x in fallback_tops]): # Nothing in the fallback top file log.error( - 'The \'%s\' saltenv has no top file, and the fallback ' - 'saltenv specified by default_top (%s) also has no ' - 'top file', cenv, default_top + u'The \'%s\' saltenv has no top file, and the fallback ' + u'saltenv specified by default_top (%s) also has no ' + u'top file', cenv, default_top ) continue @@ -2941,17 +3027,17 @@ class BaseHighState(object): if saltenv != cenv: continue log.debug( - 'The \'%s\' saltenv has no top file, using the ' - 'default_top saltenv (%s)', cenv, default_top + u'The \'%s\' saltenv has no top file, using the ' + u'default_top saltenv (%s)', cenv, default_top ) for tgt in targets: top[saltenv][tgt] = ctop[saltenv][tgt] break else: log.error( - 'The \'%s\' saltenv has no top file, and no ' - 'matches were found in the top file for the ' - 'default_top saltenv (%s)', cenv, default_top + u'The \'%s\' saltenv has no top file, and no ' + u'matches were found in the top file for the ' + u'default_top saltenv (%s)', cenv, default_top ) continue @@ -2959,14 +3045,14 @@ class BaseHighState(object): else: for ctop in ctops: for saltenv, targets in six.iteritems(ctop): - if saltenv == 'include': + if saltenv == u'include': continue elif saltenv != cenv: log.debug( - 'Section for saltenv \'%s\' in the \'%s\' ' - 'saltenv\'s top file will be ignored, as the ' - 'top_file_merging_strategy is set to \'same\' ' - 'and the saltenvs do not match', + u'Section for saltenv \'%s\' in the \'%s\' ' + u'saltenv\'s top file will be ignored, as the ' + u'top_file_merging_strategy is set to \'same\' ' + u'and the saltenvs do not match', saltenv, cenv ) continue @@ -2975,7 +3061,7 @@ class BaseHighState(object): for tgt in targets: top[saltenv][tgt] = ctop[saltenv][tgt] except TypeError: - raise SaltRenderError('Unable to render top file. No targets found.') + raise SaltRenderError(u'Unable to render top file. No targets found.') return top def _merge_tops_merge_all(self, tops): @@ -2996,7 +3082,7 @@ class BaseHighState(object): for ctops in six.itervalues(tops): for ctop in ctops: for saltenv, targets in six.iteritems(ctop): - if saltenv == 'include': + if saltenv == u'include': continue try: for tgt in targets: @@ -3013,7 +3099,7 @@ class BaseHighState(object): merged.extend([x for x in m_states2 if x not in merged]) top[saltenv][tgt] = merged except TypeError: - raise SaltRenderError('Unable to render top file. No targets found.') + raise SaltRenderError(u'Unable to render top file. No targets found.') return top def verify_tops(self, tops): @@ -3022,28 +3108,28 @@ class BaseHighState(object): ''' errors = [] if not isinstance(tops, dict): - errors.append('Top data was not formed as a dict') + errors.append(u'Top data was not formed as a dict') # No further checks will work, bail out return errors for saltenv, matches in six.iteritems(tops): - if saltenv == 'include': + if saltenv == u'include': continue if not isinstance(saltenv, six.string_types): errors.append( - 'Environment {0} in top file is not formed as a ' - 'string'.format(saltenv) + u'Environment {0} in top file is not formed as a ' + u'string'.format(saltenv) ) - if saltenv == '': - errors.append('Empty saltenv statement in top file') + if saltenv == u'': + errors.append(u'Empty saltenv statement in top file') if not isinstance(matches, dict): errors.append( - 'The top file matches for saltenv {0} are not ' - 'formatted as a dict'.format(saltenv) + u'The top file matches for saltenv {0} are not ' + u'formatted as a dict'.format(saltenv) ) for slsmods in six.itervalues(matches): if not isinstance(slsmods, list): - errors.append('Malformed topfile (state declarations not ' - 'formed as a list)') + errors.append(u'Malformed topfile (state declarations not ' + u'formed as a list)') continue for slsmod in slsmods: if isinstance(slsmod, dict): @@ -3051,8 +3137,8 @@ class BaseHighState(object): for val in six.itervalues(slsmod): if not val: errors.append( - 'Improperly formatted top file matcher ' - 'in saltenv {0}: {1} file'.format( + u'Improperly formatted top file matcher ' + u'in saltenv {0}: {1} file'.format( slsmod, val ) @@ -3061,8 +3147,8 @@ class BaseHighState(object): # This is a sls module if not slsmod: errors.append( - 'Environment {0} contains an empty sls ' - 'index'.format(saltenv) + u'Environment {0} contains an empty sls ' + u'index'.format(saltenv) ) return errors @@ -3074,7 +3160,7 @@ class BaseHighState(object): try: tops = self.get_tops() except SaltRenderError as err: - log.error('Unable to render top file: ' + str(err.error)) + log.error(u'Unable to render top file: ' + str(err.error)) return {} return self.merge_tops(tops) @@ -3086,11 +3172,11 @@ class BaseHighState(object): Returns: {'saltenv': ['state1', 'state2', ...]} ''' - matches = {} + matches = DefaultOrderedDict(OrderedDict) # pylint: disable=cell-var-from-loop for saltenv, body in six.iteritems(top): - if self.opts['environment']: - if saltenv != self.opts['environment']: + if self.opts[u'environment']: + if saltenv != self.opts[u'environment']: continue for match, data in six.iteritems(body): def _filter_matches(_match, _data, _opts): @@ -3104,8 +3190,8 @@ class BaseHighState(object): if saltenv not in matches: matches[saltenv] = [] for item in _data: - if 'subfilter' in item: - _tmpdata = item.pop('subfilter') + if u'subfilter' in item: + _tmpdata = item.pop(u'subfilter') for match, data in six.iteritems(_tmpdata): _filter_matches(match, data, _opts) if isinstance(item, six.string_types): @@ -3117,14 +3203,18 @@ class BaseHighState(object): if env_key not in matches: matches[env_key] = [] matches[env_key].append(inc_sls) - _filter_matches(match, data, self.opts['nodegroups']) + _filter_matches(match, data, self.opts[u'nodegroups']) ext_matches = self._master_tops() for saltenv in ext_matches: - if saltenv in matches: - matches[saltenv] = list( - set(ext_matches[saltenv]).union(matches[saltenv])) + top_file_matches = matches.get(saltenv, []) + if self.opts[u'master_tops_first']: + first = ext_matches[saltenv] + second = top_file_matches else: - matches[saltenv] = ext_matches[saltenv] + first = top_file_matches + second = ext_matches[saltenv] + matches[saltenv] = first + [x for x in second if x not in first] + # pylint: enable=cell-var-from-loop return matches @@ -3140,13 +3230,13 @@ class BaseHighState(object): If autoload_dynamic_modules is True then automatically load the dynamic modules ''' - if not self.opts['autoload_dynamic_modules']: + if not self.opts[u'autoload_dynamic_modules']: return - syncd = self.state.functions['saltutil.sync_all'](list(matches), + syncd = self.state.functions[u'saltutil.sync_all'](list(matches), refresh=False) - if syncd['grains']: - self.opts['grains'] = salt.loader.grains(self.opts) - self.state.opts['pillar'] = self.state._gather_pillar() + if syncd[u'grains']: + self.opts[u'grains'] = salt.loader.grains(self.opts) + self.state.opts[u'pillar'] = self.state._gather_pillar() self.state.module_refresh() def render_state(self, sls, saltenv, mods, matches, local=False): @@ -3156,39 +3246,39 @@ class BaseHighState(object): errors = [] if not local: state_data = self.client.get_state(sls, saltenv) - fn_ = state_data.get('dest', False) + fn_ = state_data.get(u'dest', False) else: fn_ = sls if not os.path.isfile(fn_): errors.append( - 'Specified SLS {0} on local filesystem cannot ' - 'be found.'.format(sls) + u'Specified SLS {0} on local filesystem cannot ' + u'be found.'.format(sls) ) if not fn_: errors.append( - 'Specified SLS {0} in saltenv {1} is not ' - 'available on the salt master or through a configured ' - 'fileserver'.format(sls, saltenv) + u'Specified SLS {0} in saltenv {1} is not ' + u'available on the salt master or through a configured ' + u'fileserver'.format(sls, saltenv) ) state = None try: state = compile_template(fn_, self.state.rend, - self.state.opts['renderer'], - self.state.opts['renderer_blacklist'], - self.state.opts['renderer_whitelist'], + self.state.opts[u'renderer'], + self.state.opts[u'renderer_blacklist'], + self.state.opts[u'renderer_whitelist'], saltenv, sls, rendered_sls=mods ) except SaltRenderError as exc: - msg = 'Rendering SLS \'{0}:{1}\' failed: {2}'.format( + msg = u'Rendering SLS \'{0}:{1}\' failed: {2}'.format( saltenv, sls, exc ) log.critical(msg) errors.append(msg) except Exception as exc: - msg = 'Rendering SLS {0} failed, render error: {1}'.format( + msg = u'Rendering SLS {0} failed, render error: {1}'.format( sls, exc ) log.critical( @@ -3196,25 +3286,25 @@ class BaseHighState(object): # Show the traceback if the debug logging level is enabled exc_info_on_loglevel=logging.DEBUG ) - errors.append('{0}\n{1}'.format(msg, traceback.format_exc())) + errors.append(u'{0}\n{1}'.format(msg, traceback.format_exc())) try: - mods.add('{0}:{1}'.format(saltenv, sls)) + mods.add(u'{0}:{1}'.format(saltenv, sls)) except AttributeError: pass if state: if not isinstance(state, dict): errors.append( - 'SLS {0} does not render to a dictionary'.format(sls) + u'SLS {0} does not render to a dictionary'.format(sls) ) else: include = [] - if 'include' in state: - if not isinstance(state['include'], list): - err = ('Include Declaration in SLS {0} is not formed ' - 'as a list'.format(sls)) + if u'include' in state: + if not isinstance(state[u'include'], list): + err = (u'Include Declaration in SLS {0} is not formed ' + u'as a list'.format(sls)) errors.append(err) else: - include = state.pop('include') + include = state.pop(u'include') self._handle_extend(state, sls, saltenv, errors) self._handle_exclude(state, sls, saltenv, errors) @@ -3225,7 +3315,7 @@ class BaseHighState(object): # 'sls.to.include' <- same as {: 'sls.to.include'} # {: 'sls.to.include'} # {'_xenv': 'sls.to.resolve'} - xenv_key = '_xenv' + xenv_key = u'_xenv' if isinstance(inc_sls, dict): env_key, inc_sls = inc_sls.popitem() @@ -3233,37 +3323,37 @@ class BaseHighState(object): env_key = saltenv if env_key not in self.avail: - msg = ('Nonexistent saltenv \'{0}\' found in include ' - 'of \'{1}\' within SLS \'{2}:{3}\'' + msg = (u'Nonexistent saltenv \'{0}\' found in include ' + u'of \'{1}\' within SLS \'{2}:{3}\'' .format(env_key, inc_sls, saltenv, sls)) log.error(msg) errors.append(msg) continue - if inc_sls.startswith('.'): - match = re.match(r'^(\.+)(.*)$', inc_sls) + if inc_sls.startswith(u'.'): + match = re.match(r'^(\.+)(.*)$', inc_sls) # future lint: disable=non-unicode-string if match: levels, include = match.groups() else: - msg = ('Badly formatted include {0} found in include ' - 'in SLS \'{2}:{3}\'' + msg = (u'Badly formatted include {0} found in include ' + u'in SLS \'{2}:{3}\'' .format(inc_sls, saltenv, sls)) log.error(msg) errors.append(msg) continue level_count = len(levels) - p_comps = sls.split('.') - if state_data.get('source', '').endswith('/init.sls'): - p_comps.append('init') + p_comps = sls.split(u'.') + if state_data.get(u'source', u'').endswith(u'/init.sls'): + p_comps.append(u'init') if level_count > len(p_comps): - msg = ('Attempted relative include of \'{0}\' ' - 'within SLS \'{1}:{2}\' ' - 'goes beyond top level package ' + msg = (u'Attempted relative include of \'{0}\' ' + u'within SLS \'{1}:{2}\' ' + u'goes beyond top level package ' .format(inc_sls, saltenv, sls)) log.error(msg) errors.append(msg) continue - inc_sls = '.'.join(p_comps[:-level_count] + [include]) + inc_sls = u'.'.join(p_comps[:-level_count] + [include]) if env_key != xenv_key: if matches is None: @@ -3295,7 +3385,7 @@ class BaseHighState(object): for sls_target in sls_targets: r_env = resolved_envs[0] if len(resolved_envs) == 1 else saltenv - mod_tgt = '{0}:{1}'.format(r_env, sls_target) + mod_tgt = u'{0}:{1}'.format(r_env, sls_target) if mod_tgt not in mods: nstate, err = self.render_state( sls_target, @@ -3309,25 +3399,25 @@ class BaseHighState(object): if err: errors.extend(err) else: - msg = '' + msg = u'' if not resolved_envs: - msg = ('Unknown include: Specified SLS {0}: {1} is not available on the salt ' - 'master in saltenv(s): {2} ' + msg = (u'Unknown include: Specified SLS {0}: {1} is not available on the salt ' + u'master in saltenv(s): {2} ' ).format(env_key, inc_sls, - ', '.join(matches) if env_key == xenv_key else env_key) + u', '.join(matches) if env_key == xenv_key else env_key) elif len(resolved_envs) > 1: - msg = ('Ambiguous include: Specified SLS {0}: {1} is available on the salt master ' - 'in multiple available saltenvs: {2}' + msg = (u'Ambiguous include: Specified SLS {0}: {1} is available on the salt master ' + u'in multiple available saltenvs: {2}' ).format(env_key, inc_sls, - ', '.join(resolved_envs)) + u', '.join(resolved_envs)) log.critical(msg) errors.append(msg) try: self._handle_iorder(state) except TypeError: - log.critical('Could not render SLS {0}. Syntax error detected.'.format(sls)) + log.critical(u'Could not render SLS %s. Syntax error detected.', sls) else: state = {} return state, errors @@ -3336,7 +3426,7 @@ class BaseHighState(object): ''' Take a state and apply the iorder system ''' - if self.opts['state_auto_order']: + if self.opts[u'state_auto_order']: for name in state: for s_dec in state[name]: if not isinstance(s_dec, six.string_types): @@ -3351,20 +3441,20 @@ class BaseHighState(object): continue found = False - if s_dec.startswith('_'): + if s_dec.startswith(u'_'): continue for arg in state[name][s_dec]: if isinstance(arg, dict): if len(arg) > 0: - if next(six.iterkeys(arg)) == 'order': + if next(six.iterkeys(arg)) == u'order': found = True if not found: if not isinstance(state[name][s_dec], list): # quite certainly a syntax error, managed elsewhere continue state[name][s_dec].append( - {'order': self.iorder} + {u'order': self.iorder} ) self.iorder += 1 return state @@ -3375,31 +3465,31 @@ class BaseHighState(object): ''' for name in state: if not isinstance(state[name], dict): - if name == '__extend__': + if name == u'__extend__': continue - if name == '__exclude__': + if name == u'__exclude__': continue if isinstance(state[name], six.string_types): # Is this is a short state, it needs to be padded - if '.' in state[name]: - comps = state[name].split('.') - state[name] = {'__sls__': sls, - '__env__': saltenv, + if u'.' in state[name]: + comps = state[name].split(u'.') + state[name] = {u'__sls__': sls, + u'__env__': saltenv, comps[0]: [comps[1]]} continue errors.append( - 'ID {0} in SLS {1} is not a dictionary'.format(name, sls) + u'ID {0} in SLS {1} is not a dictionary'.format(name, sls) ) continue skeys = set() for key in list(state[name]): - if key.startswith('_'): + if key.startswith(u'_'): continue if not isinstance(state[name][key], list): continue - if '.' in key: - comps = key.split('.') + if u'.' in key: + comps = key.split(u'.') # Salt doesn't support state files such as: # # /etc/redis/redis.conf: @@ -3412,8 +3502,8 @@ class BaseHighState(object): # - regex: ^requirepass if comps[0] in skeys: errors.append( - 'ID \'{0}\' in SLS \'{1}\' contains multiple state ' - 'declarations of the same type'.format(name, sls) + u'ID \'{0}\' in SLS \'{1}\' contains multiple state ' + u'declarations of the same type'.format(name, sls) ) continue state[name][comps[0]] = state[name].pop(key) @@ -3421,55 +3511,55 @@ class BaseHighState(object): skeys.add(comps[0]) continue skeys.add(key) - if '__sls__' not in state[name]: - state[name]['__sls__'] = sls - if '__env__' not in state[name]: - state[name]['__env__'] = saltenv + if u'__sls__' not in state[name]: + state[name][u'__sls__'] = sls + if u'__env__' not in state[name]: + state[name][u'__env__'] = saltenv def _handle_extend(self, state, sls, saltenv, errors): ''' Take the extend dec out of state and apply to the highstate global dec ''' - if 'extend' in state: - ext = state.pop('extend') + if u'extend' in state: + ext = state.pop(u'extend') if not isinstance(ext, dict): - errors.append(('Extension value in SLS \'{0}\' is not a ' - 'dictionary').format(sls)) + errors.append((u'Extension value in SLS \'{0}\' is not a ' + u'dictionary').format(sls)) return for name in ext: if not isinstance(ext[name], dict): - errors.append(('Extension name \'{0}\' in SLS \'{1}\' is ' - 'not a dictionary' + errors.append((u'Extension name \'{0}\' in SLS \'{1}\' is ' + u'not a dictionary' .format(name, sls))) continue - if '__sls__' not in ext[name]: - ext[name]['__sls__'] = sls - if '__env__' not in ext[name]: - ext[name]['__env__'] = saltenv + if u'__sls__' not in ext[name]: + ext[name][u'__sls__'] = sls + if u'__env__' not in ext[name]: + ext[name][u'__env__'] = saltenv for key in list(ext[name]): - if key.startswith('_'): + if key.startswith(u'_'): continue if not isinstance(ext[name][key], list): continue - if '.' in key: - comps = key.split('.') + if u'.' in key: + comps = key.split(u'.') ext[name][comps[0]] = ext[name].pop(key) ext[name][comps[0]].append(comps[1]) - state.setdefault('__extend__', []).append(ext) + state.setdefault(u'__extend__', []).append(ext) def _handle_exclude(self, state, sls, saltenv, errors): ''' Take the exclude dec out of the state and apply it to the highstate global dec ''' - if 'exclude' in state: - exc = state.pop('exclude') + if u'exclude' in state: + exc = state.pop(u'exclude') if not isinstance(exc, list): - err = ('Exclude Declaration in SLS {0} is not formed ' - 'as a list'.format(sls)) + err = (u'Exclude Declaration in SLS {0} is not formed ' + u'as a list'.format(sls)) errors.append(err) - state.setdefault('__exclude__', []).extend(exc) + state.setdefault(u'__exclude__', []).extend(exc) def render_highstate(self, matches): ''' @@ -3486,8 +3576,8 @@ class BaseHighState(object): statefiles = fnmatch.filter(self.avail[saltenv], sls_match) except KeyError: all_errors.extend( - ['No matching salt environment for environment ' - '\'{0}\' found'.format(saltenv)] + [u'No matching salt environment for environment ' + u'\'{0}\' found'.format(saltenv)] ) # if we did not found any sls in the fileserver listing, this # may be because the sls was generated or added later, we can @@ -3497,7 +3587,7 @@ class BaseHighState(object): statefiles = [sls_match] for sls in statefiles: - r_env = '{0}:{1}'.format(saltenv, sls) + r_env = u'{0}:{1}'.format(saltenv, sls) if r_env in mods: continue state, errors = self.render_state( @@ -3505,55 +3595,55 @@ class BaseHighState(object): if state: self.merge_included_states(highstate, state, errors) for i, error in enumerate(errors[:]): - if 'is not available' in error: + if u'is not available' in error: # match SLS foobar in environment - this_sls = 'SLS {0} in saltenv'.format( + this_sls = u'SLS {0} in saltenv'.format( sls_match) if this_sls in error: errors[i] = ( - 'No matching sls found for \'{0}\' ' - 'in env \'{1}\''.format(sls_match, saltenv)) + u'No matching sls found for \'{0}\' ' + u'in env \'{1}\''.format(sls_match, saltenv)) all_errors.extend(errors) self.clean_duplicate_extends(highstate) return highstate, all_errors def clean_duplicate_extends(self, highstate): - if '__extend__' in highstate: + if u'__extend__' in highstate: highext = [] - for items in (six.iteritems(ext) for ext in highstate['__extend__']): + for items in (six.iteritems(ext) for ext in highstate[u'__extend__']): for item in items: if item not in highext: highext.append(item) - highstate['__extend__'] = [{t[0]: t[1]} for t in highext] + highstate[u'__extend__'] = [{t[0]: t[1]} for t in highext] def merge_included_states(self, highstate, state, errors): # The extend members can not be treated as globally unique: - if '__extend__' in state: - highstate.setdefault('__extend__', - []).extend(state.pop('__extend__')) - if '__exclude__' in state: - highstate.setdefault('__exclude__', - []).extend(state.pop('__exclude__')) + if u'__extend__' in state: + highstate.setdefault(u'__extend__', + []).extend(state.pop(u'__extend__')) + if u'__exclude__' in state: + highstate.setdefault(u'__exclude__', + []).extend(state.pop(u'__exclude__')) for id_ in state: if id_ in highstate: if highstate[id_] != state[id_]: errors.append(( - 'Detected conflicting IDs, SLS' - ' IDs need to be globally unique.\n The' - ' conflicting ID is \'{0}\' and is found in SLS' - ' \'{1}:{2}\' and SLS \'{3}:{4}\'').format( + u'Detected conflicting IDs, SLS' + u' IDs need to be globally unique.\n The' + u' conflicting ID is \'{0}\' and is found in SLS' + u' \'{1}:{2}\' and SLS \'{3}:{4}\'').format( id_, - highstate[id_]['__env__'], - highstate[id_]['__sls__'], - state[id_]['__env__'], - state[id_]['__sls__']) + highstate[id_][u'__env__'], + highstate[id_][u'__sls__'], + state[id_][u'__env__'], + state[id_][u'__sls__']) ) try: highstate.update(state) except ValueError: errors.append( - 'Error when rendering state with contents: {0}'.format(state) + u'Error when rendering state with contents: {0}'.format(state) ) def _check_pillar(self, force=False): @@ -3563,7 +3653,7 @@ class BaseHighState(object): ''' if force: return True - if '_errors' in self.state.opts['pillar']: + if u'_errors' in self.state.opts[u'pillar']: return False return True @@ -3576,7 +3666,7 @@ class BaseHighState(object): return matches ret_matches = {} if not isinstance(whitelist, list): - whitelist = whitelist.split(',') + whitelist = whitelist.split(u',') for env in matches: for sls in matches[env]: if sls in whitelist: @@ -3584,28 +3674,28 @@ class BaseHighState(object): ret_matches[env].append(sls) return ret_matches - def call_highstate(self, exclude=None, cache=None, cache_name='highstate', + def call_highstate(self, exclude=None, cache=None, cache_name=u'highstate', force=False, whitelist=None, orchestration_jid=None): ''' Run the sequence to execute the salt highstate for this minion ''' # Check that top file exists - tag_name = 'no_|-states_|-states_|-None' + tag_name = u'no_|-states_|-states_|-None' ret = {tag_name: { - 'result': False, - 'comment': 'No states found for this minion', - 'name': 'No States', - 'changes': {}, - '__run_num__': 0, + u'result': False, + u'comment': u'No states found for this minion', + u'name': u'No States', + u'changes': {}, + u'__run_num__': 0, }} cfn = os.path.join( - self.opts['cachedir'], - '{0}.cache.p'.format(cache_name) + self.opts[u'cachedir'], + u'{0}.cache.p'.format(cache_name) ) if cache: if os.path.isfile(cfn): - with salt.utils.fopen(cfn, 'rb') as fp_: + with salt.utils.files.fopen(cfn, u'rb') as fp_: high = self.serial.load(fp_) return self.state.call_high(high, orchestration_jid) # File exists so continue @@ -3613,8 +3703,8 @@ class BaseHighState(object): try: top = self.get_top() except SaltRenderError as err: - ret[tag_name]['comment'] = 'Unable to render top file: ' - ret[tag_name]['comment'] += str(err.error) + ret[tag_name][u'comment'] = u'Unable to render top file: ' + ret[tag_name][u'comment'] += str(err.error) return ret except Exception: trb = traceback.format_exc() @@ -3623,23 +3713,23 @@ class BaseHighState(object): err += self.verify_tops(top) matches = self.top_matches(top) if not matches: - msg = 'No Top file or external nodes data matches found.' - ret[tag_name]['comment'] = msg + msg = u'No Top file or master_tops data matches found.' + ret[tag_name][u'comment'] = msg return ret matches = self.matches_whitelist(matches, whitelist) self.load_dynamic(matches) if not self._check_pillar(force): - err += ['Pillar failed to render with the following messages:'] - err += self.state.opts['pillar']['_errors'] + err += [u'Pillar failed to render with the following messages:'] + err += self.state.opts[u'pillar'][u'_errors'] else: high, errors = self.render_highstate(matches) if exclude: - if isinstance(exclude, str): - exclude = exclude.split(',') - if '__exclude__' in high: - high['__exclude__'].extend(exclude) + if isinstance(exclude, six.string_types): + exclude = exclude.split(u',') + if u'__exclude__' in high: + high[u'__exclude__'].extend(exclude) else: - high['__exclude__'] = exclude + high[u'__exclude__'] = exclude err += errors if err: return err @@ -3647,18 +3737,20 @@ class BaseHighState(object): return ret cumask = os.umask(0o77) try: - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): # Make sure cache file isn't read-only - self.state.functions['cmd.run']('attrib -R "{0}"'.format(cfn), output_loglevel='quiet') - with salt.utils.fopen(cfn, 'w+b') as fp_: + self.state.functions[u'cmd.run']( + [u'attrib', u'-R', cfn], + python_shell=False, + output_loglevel=u'quiet') + with salt.utils.files.fopen(cfn, u'w+b') as fp_: try: self.serial.dump(high, fp_) except TypeError: # Can't serialize pydsl pass except (IOError, OSError): - msg = 'Unable to write to "state.highstate" cache file {0}' - log.error(msg.format(cfn)) + log.error(u'Unable to write to "state.highstate" cache file %s', cfn) os.umask(cumask) return self.state.call_high(high, orchestration_jid) @@ -3722,23 +3814,23 @@ class BaseHighState(object): for saltenv, states in self.avail.items(): env_usage = { - 'used': [], - 'unused': [], - 'count_all': 0, - 'count_used': 0, - 'count_unused': 0 + u'used': [], + u'unused': [], + u'count_all': 0, + u'count_used': 0, + u'count_unused': 0 } env_matches = matches.get(saltenv) for state in states: - env_usage['count_all'] += 1 + env_usage[u'count_all'] += 1 if state in env_matches: - env_usage['count_used'] += 1 - env_usage['used'].append(state) + env_usage[u'count_used'] += 1 + env_usage[u'used'].append(state) else: - env_usage['count_unused'] += 1 - env_usage['unused'].append(state) + env_usage[u'count_unused'] += 1 + env_usage[u'unused'].append(state) state_usage[saltenv] = env_usage @@ -3757,24 +3849,26 @@ class HighState(BaseHighState): def __init__( self, opts, - pillar=None, + pillar_override=None, jid=None, pillar_enc=None, proxy=None, context=None, mocked=False, - loader='states'): + loader=u'states', + initial_pillar=None): self.opts = opts self.client = salt.fileclient.get_file_client(self.opts) BaseHighState.__init__(self, opts) self.state = State(self.opts, - pillar, + pillar_override, jid, pillar_enc, proxy=proxy, context=context, mocked=mocked, - loader=loader) + loader=loader, + initial_pillar=initial_pillar) self.matcher = salt.minion.Matcher(self.opts) self.proxy = proxy @@ -3819,13 +3913,13 @@ class MasterState(State): ''' Load the modules into the state ''' - log.info('Loading fresh modules for state activity') + log.info(u'Loading fresh modules for state activity') # Load a modified client interface that looks like the interface used # from the minion, but uses remote execution # self.functions = salt.client.FunctionWrapper( self.opts, - self.opts['id'] + self.opts[u'id'] ) # Load the states, but they should not be used in this class apart # from inspection @@ -3843,12 +3937,12 @@ class MasterHighState(HighState): saltenv=None): # Force the fileclient to be local opts = copy.deepcopy(minion_opts) - opts['file_client'] = 'local' - opts['file_roots'] = master_opts['master_roots'] - opts['renderer'] = master_opts['renderer'] - opts['state_top'] = master_opts['state_top'] - opts['id'] = id_ - opts['grains'] = grains + opts[u'file_client'] = u'local' + opts[u'file_roots'] = master_opts[u'master_roots'] + opts[u'renderer'] = master_opts[u'renderer'] + opts[u'state_top'] = master_opts[u'state_top'] + opts[u'id'] = id_ + opts[u'grains'] = grains HighState.__init__(self, opts) @@ -3861,15 +3955,15 @@ class RemoteHighState(object): self.grains = grains self.serial = salt.payload.Serial(self.opts) # self.auth = salt.crypt.SAuth(opts) - self.channel = salt.transport.Channel.factory(self.opts['master_uri']) + self.channel = salt.transport.Channel.factory(self.opts[u'master_uri']) def compile_master(self): ''' Return the state data from the master ''' - load = {'grains': self.grains, - 'opts': self.opts, - 'cmd': '_master_state'} + load = {u'grains': self.grains, + u'opts': self.opts, + u'cmd': u'_master_state'} try: return self.channel.send(load, tries=3, timeout=72000) except SaltReqTimeoutError: diff --git a/salt/states/acme.py b/salt/states/acme.py index 1ab6b57dfb4..43649a64262 100644 --- a/salt/states/acme.py +++ b/salt/states/acme.py @@ -116,9 +116,14 @@ def cert(name, if res['result'] is None: ret['changes'] = {} else: + if not __salt__['acme.has'](name): + new = None + else: + new = __salt__['acme.info'](name) + ret['changes'] = { 'old': old, - 'new': __salt__['acme.info'](name) + 'new': new } return ret diff --git a/salt/states/apache.py b/salt/states/apache.py index a3f4b862288..b732e71df39 100644 --- a/salt/states/apache.py +++ b/salt/states/apache.py @@ -45,7 +45,7 @@ from __future__ import absolute_import import os.path # Import Salt libs -import salt.utils +import salt.utils.files def __virtual__(): @@ -61,7 +61,7 @@ def configfile(name, config): configs = __salt__['apache.config'](name, config, edit=False) current_configs = '' if os.path.exists(name): - with salt.utils.fopen(name) as config_file: + with salt.utils.files.fopen(name) as config_file: current_configs = config_file.read() if configs == current_configs.strip(): @@ -78,7 +78,7 @@ def configfile(name, config): return ret try: - with salt.utils.fopen(name, 'w') as config_file: + with salt.utils.files.fopen(name, 'w') as config_file: print(configs, file=config_file) ret['changes'] = { 'old': current_configs, diff --git a/salt/states/apache_conf.py b/salt/states/apache_conf.py index 17cef65e1ee..e4494ab205b 100644 --- a/salt/states/apache_conf.py +++ b/salt/states/apache_conf.py @@ -17,17 +17,17 @@ Enable and disable apache confs. - name: security ''' from __future__ import absolute_import -from salt.ext.six import string_types +from salt.ext import six # Import salt libs -import salt.utils +import salt.utils.path def __virtual__(): ''' Only load if a2enconf is available. ''' - return 'apache_conf' if 'apache.a2enconf' in __salt__ and salt.utils.which('a2enconf') else False + return 'apache_conf' if 'apache.a2enconf' in __salt__ and salt.utils.path.which('a2enconf') else False def enabled(name): @@ -49,14 +49,14 @@ def enabled(name): ret['result'] = None return ret status = __salt__['apache.a2enconf'](name)['Status'] - if isinstance(status, string_types) and 'enabled' in status: + if isinstance(status, six.string_types) and 'enabled' in status: ret['result'] = True ret['changes']['old'] = None ret['changes']['new'] = name else: ret['result'] = False ret['comment'] = 'Failed to enable {0} Apache conf'.format(name) - if isinstance(status, string_types): + if isinstance(status, six.string_types): ret['comment'] = ret['comment'] + ' ({0})'.format(status) return ret else: @@ -83,14 +83,14 @@ def disabled(name): ret['result'] = None return ret status = __salt__['apache.a2disconf'](name)['Status'] - if isinstance(status, string_types) and 'disabled' in status: + if isinstance(status, six.string_types) and 'disabled' in status: ret['result'] = True ret['changes']['old'] = name ret['changes']['new'] = None else: ret['result'] = False ret['comment'] = 'Failed to disable {0} Apache conf'.format(name) - if isinstance(status, string_types): + if isinstance(status, six.string_types): ret['comment'] = ret['comment'] + ' ({0})'.format(status) return ret else: diff --git a/salt/states/archive.py b/salt/states/archive.py index f61a8084dd7..c2308cbbd0e 100644 --- a/salt/states/archive.py +++ b/salt/states/archive.py @@ -17,13 +17,16 @@ import tarfile from contextlib import closing # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import shlex_quote as _cmd_quote from salt.ext.six.moves.urllib.parse import urlparse as _urlparse # pylint: disable=no-name-in-module -# Import salt libs +# Import Salt libs import salt.utils +import salt.utils.args import salt.utils.files +import salt.utils.path +import salt.utils.platform import salt.utils.url from salt.exceptions import CommandExecutionError, CommandNotFoundError @@ -70,7 +73,7 @@ def _update_checksum(cached_source): lines = [] try: try: - with salt.utils.fopen(cached_source_sum, 'r') as fp_: + with salt.utils.files.fopen(cached_source_sum, 'r') as fp_: for line in fp_: try: lines.append(line.rstrip('\n').split(':', 1)) @@ -80,7 +83,7 @@ def _update_checksum(cached_source): if exc.errno != errno.ENOENT: raise - with salt.utils.fopen(cached_source_sum, 'w') as fp_: + with salt.utils.files.fopen(cached_source_sum, 'w') as fp_: for line in lines: if line[0] == hash_type: line[1] = hsum @@ -99,7 +102,7 @@ def _read_cached_checksum(cached_source, form=None): form = __opts__['hash_type'] path = '.'.join((cached_source, 'hash')) try: - with salt.utils.fopen(path, 'r') as fp_: + with salt.utils.files.fopen(path, 'r') as fp_: for line in fp_: # Should only be one line in this file but just in case it # isn't, read only a single line to avoid overuse of memory. @@ -375,7 +378,7 @@ def extracted(name, .. versionadded:: 2016.11.0 - source_hash_update + source_hash_update : False Set this to ``True`` if archive should be extracted if source_hash has changed. This would extract regardless of the ``if_missing`` parameter. @@ -621,7 +624,7 @@ def extracted(name, ret = {'name': name, 'result': False, 'changes': {}, 'comment': ''} # Remove pub kwargs as they're irrelevant here. - kwargs = salt.utils.clean_kwargs(**kwargs) + kwargs = salt.utils.args.clean_kwargs(**kwargs) if not _path_is_abs(name): ret['comment'] = '{0} is not an absolute path'.format(name) @@ -674,7 +677,7 @@ def extracted(name, return ret if user or group: - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): ret['comment'] = \ 'User/group ownership cannot be enforced on Windows minions' return ret @@ -844,10 +847,10 @@ def extracted(name, if source_hash: try: source_sum = __salt__['file.get_source_sum']( - source=source_match, - source_hash=source_hash, - source_hash_name=source_hash_name, - saltenv=__env__) + source=source_match, + source_hash=source_hash, + source_hash_name=source_hash_name, + saltenv=__env__) except CommandExecutionError as exc: ret['comment'] = exc.strerror return ret @@ -866,9 +869,9 @@ def extracted(name, if os.path.isdir(cached_source): # Prevent a traceback from attempting to read from a directory path - salt.utils.rm_rf(cached_source) + salt.utils.files.rm_rf(cached_source) - existing_cached_source_sum = _read_cached_checksum(cached_source) \ + existing_cached_source_sum = _read_cached_checksum(cached_source) if source_is_local: # No need to download archive, it's local to the minion @@ -935,15 +938,16 @@ def extracted(name, ) return file_result - if source_hash: - _update_checksum(cached_source) - else: log.debug( 'Archive %s is already in cache', salt.utils.url.redact_http_basic_auth(source_match) ) + if source_hash and source_hash_update and not skip_verify: + # Create local hash sum file if we're going to track sum update + _update_checksum(cached_source) + if archive_format == 'zip' and not password: log.debug('Checking %s to see if it is password-protected', source_match) @@ -1116,7 +1120,7 @@ def extracted(name, for path in incorrect_type: full_path = os.path.join(name, path) try: - salt.utils.rm_rf(full_path.rstrip(os.sep)) + salt.utils.files.rm_rf(full_path.rstrip(os.sep)) ret['changes'].setdefault( 'removed', []).append(full_path) extraction_needed = True @@ -1147,6 +1151,15 @@ def extracted(name, created_destdir = False if extraction_needed: + if source_is_local and source_hash and not skip_verify: + ret['result'] = __salt__['file.check_hash'](source_match, source_sum['hsum']) + if not ret['result']: + ret['comment'] = \ + '{0} does not match the desired source_hash {1}'.format( + source_match, source_sum['hsum'] + ) + return ret + if __opts__['test']: ret['result'] = None ret['comment'] = \ @@ -1166,7 +1179,7 @@ def extracted(name, full_path = os.path.join(name, path) try: log.debug('Removing %s', full_path) - salt.utils.rm_rf(full_path.rstrip(os.sep)) + salt.utils.files.rm_rf(full_path.rstrip(os.sep)) ret['changes'].setdefault( 'removed', []).append(full_path) except OSError as exc: @@ -1184,7 +1197,7 @@ def extracted(name, return ret if not os.path.isdir(name): - __salt__['file.makedirs'](name, user=user) + __states__['file.directory'](name, user=user, makedirs=True) created_destdir = True log.debug('Extracting {0} to {1}'.format(cached_source, name)) @@ -1208,6 +1221,7 @@ def extracted(name, options=options, trim_output=trim_output, password=password, + extract_perms=extract_perms, **kwargs) elif archive_format == 'rar': try: @@ -1227,7 +1241,7 @@ def extracted(name, if trim_output: files = files[:trim_output] except tarfile.ReadError: - if salt.utils.which('xz'): + if salt.utils.path.which('xz'): if __salt__['cmd.retcode']( ['xz', '-t', cached_source], python_shell=False, @@ -1285,7 +1299,7 @@ def extracted(name, ) return ret else: - if not salt.utils.which('tar'): + if not salt.utils.path.which('tar'): ret['comment'] = ( 'tar command not available, it might not be ' 'installed on minion' diff --git a/salt/states/artifactory.py b/salt/states/artifactory.py index ddcc7a6707d..cf6068824a4 100644 --- a/salt/states/artifactory.py +++ b/salt/states/artifactory.py @@ -11,10 +11,10 @@ import logging log = logging.getLogger(__name__) -def downloaded(name, artifact, target_dir='/tmp', target_file=None): +def downloaded(name, artifact, target_dir='/tmp', target_file=None, use_literal_group_id=False): ''' Ensures that the artifact from artifactory exists at given location. If it doesn't exist, then - it will be downloaded. It it already exists then the checksum of existing file is checked against checksum + it will be downloaded. If it already exists then the checksum of existing file is checked against checksum in artifactory. If it is different then the step will fail. artifact @@ -84,7 +84,7 @@ def downloaded(name, artifact, target_dir='/tmp', target_file=None): 'comment': ''} try: - fetch_result = __fetch_from_artifactory(artifact, target_dir, target_file) + fetch_result = __fetch_from_artifactory(artifact, target_dir, target_file, use_literal_group_id) except Exception as exc: ret['result'] = False ret['comment'] = str(exc) @@ -100,7 +100,7 @@ def downloaded(name, artifact, target_dir='/tmp', target_file=None): return ret -def __fetch_from_artifactory(artifact, target_dir, target_file): +def __fetch_from_artifactory(artifact, target_dir, target_file, use_literal_group_id): if ('latest_snapshot' in artifact and artifact['latest_snapshot']) or artifact['version'] == 'latest_snapshot': fetch_result = __salt__['artifactory.get_latest_snapshot'](artifactory_url=artifact['artifactory_url'], repository=artifact['repository'], @@ -111,7 +111,8 @@ def __fetch_from_artifactory(artifact, target_dir, target_file): target_dir=target_dir, target_file=target_file, username=artifact['username'] if 'username' in artifact else None, - password=artifact['password'] if 'password' in artifact else None) + password=artifact['password'] if 'password' in artifact else None, + use_literal_group_id=use_literal_group_id) elif artifact['version'].endswith('SNAPSHOT'): fetch_result = __salt__['artifactory.get_snapshot'](artifactory_url=artifact['artifactory_url'], repository=artifact['repository'], @@ -123,7 +124,8 @@ def __fetch_from_artifactory(artifact, target_dir, target_file): target_dir=target_dir, target_file=target_file, username=artifact['username'] if 'username' in artifact else None, - password=artifact['password'] if 'password' in artifact else None) + password=artifact['password'] if 'password' in artifact else None, + use_literal_group_id=use_literal_group_id) elif artifact['version'] == 'latest': fetch_result = __salt__['artifactory.get_latest_release'](artifactory_url=artifact['artifactory_url'], repository=artifact['repository'], @@ -134,7 +136,8 @@ def __fetch_from_artifactory(artifact, target_dir, target_file): target_dir=target_dir, target_file=target_file, username=artifact['username'] if 'username' in artifact else None, - password=artifact['password'] if 'password' in artifact else None) + password=artifact['password'] if 'password' in artifact else None, + use_literal_group_id=use_literal_group_id) else: fetch_result = __salt__['artifactory.get_release'](artifactory_url=artifact['artifactory_url'], repository=artifact['repository'], @@ -146,5 +149,6 @@ def __fetch_from_artifactory(artifact, target_dir, target_file): target_dir=target_dir, target_file=target_file, username=artifact['username'] if 'username' in artifact else None, - password=artifact['password'] if 'password' in artifact else None) + password=artifact['password'] if 'password' in artifact else None, + use_literal_group_id=use_literal_group_id) return fetch_result diff --git a/salt/states/augeas.py b/salt/states/augeas.py index 8321f5ed959..60759a79afe 100644 --- a/salt/states/augeas.py +++ b/salt/states/augeas.py @@ -36,7 +36,8 @@ import logging import difflib # Import Salt libs -import salt.utils +import salt.utils.args +import salt.utils.files from salt.modules.augeas_cfg import METHOD_MAP @@ -74,7 +75,7 @@ def _check_filepath(changes): error = 'Command {0} is not supported (yet)'.format(cmd) raise ValueError(error) method = METHOD_MAP[cmd] - parts = salt.utils.shlex_split(arg) + parts = salt.utils.args.shlex_split(arg) if method in ['set', 'setm', 'move', 'remove']: filename_ = parts[0] else: @@ -200,7 +201,7 @@ def change(name, context=None, changes=None, lens=None, redis-conf: augeas.change: - - lens: redis + - lens: redis.lns - context: /files/etc/redis/redis.conf - changes: - set bind 0.0.0.0 @@ -224,21 +225,34 @@ def change(name, context=None, changes=None, lens=None, zabbix-service: augeas.change: - - lens: services + - lens: services.lns - context: /files/etc/services - changes: - ins service-name after service-name[last()] - - set service-name[last()] zabbix-agent - - set service-name[. = 'zabbix-agent']/#comment "Zabbix Agent service" - - set service-name[. = 'zabbix-agent']/port 10050 - - set service-name[. = 'zabbix-agent']/protocol tcp - - rm service-name[. = 'im-obsolete'] + - set service-name[last()] "zabbix-agent" + - set "service-name[. = 'zabbix-agent']/port" 10050 + - set "service-name[. = 'zabbix-agent']/protocol" tcp + - set "service-name[. = 'zabbix-agent']/#comment" "Zabbix Agent service" + - rm "service-name[. = 'im-obsolete']" - unless: grep "zabbix-agent" /etc/services .. warning:: - Don't forget the ``unless`` here, otherwise a new entry will be added - every time this state is run. + Don't forget the ``unless`` here, otherwise it will fail on next runs + because the service is already defined. Additionally you have to quote + lines containing ``service-name[. = 'zabbix-agent']`` otherwise + :mod:`augeas_cfg ` execute will fail because + it will receive more parameters than expected. + + .. note:: + + Order is important when defining a service with Augeas, in this case + it's ``port``, ``protocol`` and ``#comment``. For more info about + the lens check `services lens documentation`_. + + .. _services lens documentation: + + http://augeas.net/docs/references/lenses/files/services-aug.html#Services.record ''' ret = {'name': name, 'result': False, 'comment': '', 'changes': {}} @@ -275,7 +289,7 @@ def change(name, context=None, changes=None, lens=None, old_file = [] if filename is not None: if os.path.isfile(filename): - with salt.utils.fopen(filename, 'r') as file_: + with salt.utils.files.fopen(filename, 'r') as file_: old_file = file_.readlines() result = __salt__['augeas.execute']( @@ -288,7 +302,7 @@ def change(name, context=None, changes=None, lens=None, return ret if old_file: - with salt.utils.fopen(filename, 'r') as file_: + with salt.utils.files.fopen(filename, 'r') as file_: diff = ''.join( difflib.unified_diff(old_file, file_.readlines(), n=0)) diff --git a/salt/states/beacon.py b/salt/states/beacon.py index 5b9730c2b97..64d1905dc29 100644 --- a/salt/states/beacon.py +++ b/salt/states/beacon.py @@ -9,24 +9,27 @@ Management of the Salt beacons ps: beacon.present: + - save: True - enable: False - - salt-master: running - - apache2: stopped + - services: + salt-master: running + apache2: stopped sh: - beacon.present: + beacon.present: [] load: beacon.present: - - 1m: - - 0.0 - - 2.0 - - 5m: - - 0.0 - - 1.5 - - 15m: - - 0.1 - - 1.0 + - averages: + 1m: + - 0.0 + - 2.0 + 5m: + - 0.0 + - 1.5 + 15m: + - 0.1 + - 1.0 ''' from __future__ import absolute_import @@ -35,12 +38,15 @@ log = logging.getLogger(__name__) def present(name, + save=False, **kwargs): ''' Ensure beacon is configured with the included beacon data. name The name of the beacon ensure is configured. + save + True/False, if True the beacons.conf file be updated too. Default is False. ''' @@ -50,7 +56,7 @@ def present(name, 'comment': []} current_beacons = __salt__['beacons.list'](return_yaml=False) - beacon_data = kwargs + beacon_data = [kwargs] if name in current_beacons: @@ -59,11 +65,11 @@ def present(name, else: if 'test' in __opts__ and __opts__['test']: kwargs['test'] = True - result = __salt__['beacons.modify'](name, beacon_data, **kwargs) + result = __salt__['beacons.modify'](name, beacon_data) ret['comment'].append(result['comment']) ret['changes'] = result['changes'] else: - result = __salt__['beacons.modify'](name, beacon_data, **kwargs) + result = __salt__['beacons.modify'](name, beacon_data) if not result['result']: ret['result'] = result['result'] ret['comment'] = result['comment'] @@ -74,13 +80,14 @@ def present(name, ret['changes'] = result['changes'] else: ret['comment'].append(result['comment']) + else: if 'test' in __opts__ and __opts__['test']: kwargs['test'] = True result = __salt__['beacons.add'](name, beacon_data, **kwargs) ret['comment'].append(result['comment']) else: - result = __salt__['beacons.add'](name, beacon_data, **kwargs) + result = __salt__['beacons.add'](name, beacon_data) if not result['result']: ret['result'] = result['result'] ret['comment'] = result['comment'] @@ -88,16 +95,24 @@ def present(name, else: ret['comment'].append('Adding {0} to beacons'.format(name)) + if save: + result = __salt__['beacons.save']() + ret['comment'].append('Beacon {0} saved'.format(name)) + ret['comment'] = '\n'.join(ret['comment']) return ret -def absent(name, **kwargs): +def absent(name, + save=False, + **kwargs): ''' Ensure beacon is absent. name The name of the beacon ensured absent. + save + True/False, if True the beacons.conf file be updated too. Default is False. ''' ### NOTE: The keyword arguments in **kwargs are ignored in this state, but @@ -126,6 +141,10 @@ def absent(name, **kwargs): else: ret['comment'].append('{0} not configured in beacons'.format(name)) + if save: + result = __salt__['beacons.save']() + ret['comment'].append('Beacon {0} saved'.format(name)) + ret['comment'] = '\n'.join(ret['comment']) return ret diff --git a/salt/states/bigip.py b/salt/states/bigip.py index 8a575b0d1d6..11247b3618e 100644 --- a/salt/states/bigip.py +++ b/salt/states/bigip.py @@ -11,7 +11,7 @@ from __future__ import absolute_import import json # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six #set up virtual function @@ -85,7 +85,7 @@ def _check_for_changes(entity_type, ret, existing, modified): if 'generation' in existing['content'].keys(): del existing['content']['generation'] - if cmp(modified['content'], existing['content']) == 0: + if modified['content'] == existing['content']: ret['comment'] = '{entity_type} is currently enforced to the desired state. No changes made.'.format(entity_type=entity_type) else: ret['comment'] = '{entity_type} was enforced to the desired state. Note: Only parameters specified ' \ @@ -94,7 +94,7 @@ def _check_for_changes(entity_type, ret, existing, modified): ret['changes']['new'] = modified['content'] else: - if cmp(modified, existing) == 0: + if modified == existing: ret['comment'] = '{entity_type} is currently enforced to the desired state. No changes made.'.format(entity_type=entity_type) else: ret['comment'] = '{entity_type} was enforced to the desired state. Note: Only parameters specified ' \ diff --git a/salt/states/blockdev.py b/salt/states/blockdev.py index 4b0dc5ca81d..6866c0c26b6 100644 --- a/salt/states/blockdev.py +++ b/salt/states/blockdev.py @@ -29,7 +29,7 @@ import time import logging # Import salt libs -import salt.utils +import salt.utils.path from salt.ext.six.moves import range __virtualname__ = 'blockdev' @@ -150,7 +150,7 @@ def formatted(name, fs_type='ext4', force=False, **kwargs): if current_fs == fs_type: ret['result'] = True return ret - elif not salt.utils.which('mkfs.{0}'.format(fs_type)): + elif not salt.utils.path.which('mkfs.{0}'.format(fs_type)): ret['comment'] = 'Invalid fs_type: {0}'.format(fs_type) ret['result'] = False return ret diff --git a/salt/states/boto3_route53.py b/salt/states/boto3_route53.py index 0a49dcc3fef..d55ca48bb4b 100644 --- a/salt/states/boto3_route53.py +++ b/salt/states/boto3_route53.py @@ -70,7 +70,8 @@ import uuid # Import Salt Libs import salt.utils.dictupdate as dictupdate -from salt.utils import SaltInvocationError, exactly_one +from salt.utils import exactly_one +from salt.exceptions import SaltInvocationError import logging log = logging.getLogger(__name__) # pylint: disable=W1699 diff --git a/salt/states/boto3_sns.py b/salt/states/boto3_sns.py index 47f88fc3777..720277e5384 100644 --- a/salt/states/boto3_sns.py +++ b/salt/states/boto3_sns.py @@ -60,7 +60,7 @@ import re import logging import json import copy -import salt.ext.six as six +from salt.ext import six log = logging.getLogger(__name__) diff --git a/salt/states/boto_apigateway.py b/salt/states/boto_apigateway.py index 0f1ffc33fd8..7288230144d 100644 --- a/salt/states/boto_apigateway.py +++ b/salt/states/boto_apigateway.py @@ -56,8 +56,8 @@ import re import json import yaml # Import Salt Libs -import salt.ext.six as six -import salt.utils +from salt.ext import six +import salt.utils.files from salt.utils.yamlloader import SaltYamlSafeLoader log = logging.getLogger(__name__) @@ -432,7 +432,7 @@ def _gen_md5_filehash(fname, *args): and participates in the hash calculation ''' _hash = hashlib.md5() - with salt.utils.fopen(fname, 'rb') as f: + with salt.utils.files.fopen(fname, 'rb') as f: for chunk in iter(lambda: f.read(4096), b''): _hash.update(chunk) @@ -713,7 +713,7 @@ class _Swagger(object): self._md5_filehash = _gen_md5_filehash(self._swagger_file, error_response_template, response_template) - with salt.utils.fopen(self._swagger_file, 'rb') as sf: + with salt.utils.files.fopen(self._swagger_file, 'rb') as sf: self._cfg = yaml.load( sf, Loader=SaltYamlSafeLoader diff --git a/salt/states/boto_asg.py b/salt/states/boto_asg.py index e8260a688d3..73305ae44e6 100644 --- a/salt/states/boto_asg.py +++ b/salt/states/boto_asg.py @@ -200,7 +200,7 @@ import copy # Import Salt libs import salt.utils.dictupdate as dictupdate -import salt.ext.six as six +from salt.ext import six from salt.exceptions import SaltInvocationError log = logging.getLogger(__name__) diff --git a/salt/states/boto_cfn.py b/salt/states/boto_cfn.py index fcc73754649..3db51d75792 100644 --- a/salt/states/boto_cfn.py +++ b/salt/states/boto_cfn.py @@ -37,13 +37,16 @@ Connection module for Amazon Cloud Formation - name: mystack ''' -from __future__ import absolute_import - # Import Python libs +from __future__ import absolute_import import logging import json -# Import 3rd party libs +# Import Salt libs +import salt.utils.compat +from salt.ext import six + +# Import 3rd-party libs try: from salt._compat import ElementTree as ET HAS_ELEMENT_TREE = True @@ -141,10 +144,14 @@ def present(name, template_body=None, template_url=None, parameters=None, notifi stack_policy_body = _get_template(stack_policy_body, name) stack_policy_during_update_body = _get_template(stack_policy_during_update_body, name) + for i in [template_body, stack_policy_body, stack_policy_during_update_body]: + if isinstance(i, dict): + return i + _valid = _validate(template_body, template_url, region, key, keyid, profile) log.debug('Validate is : {0}.'.format(_valid)) if _valid is not True: - code, message = _get_error(_valid) + code, message = _valid ret['result'] = False ret['comment'] = 'Template could not be validated.\n{0} \n{1}'.format(code, message) return ret @@ -154,7 +161,7 @@ def present(name, template_body=None, template_url=None, parameters=None, notifi template = template['GetTemplateResponse']['GetTemplateResult']['TemplateBody'].encode('ascii', 'ignore') template = json.loads(template) _template_body = json.loads(template_body) - compare = cmp(template, _template_body) + compare = salt.utils.compat.cmp(template, _template_body) if compare != 0: log.debug('Templates are not the same. Compare value is {0}'.format(compare)) # At this point we should be able to run update safely since we already validated the template @@ -169,7 +176,7 @@ def present(name, template_body=None, template_url=None, parameters=None, notifi stack_policy_during_update_url, stack_policy_body, stack_policy_url, region, key, keyid, profile) - if isinstance(updated, str): + if isinstance(updated, six.string_types): code, message = _get_error(updated) log.debug('Update error is {0} and message is {1}'.format(code, message)) ret['result'] = False @@ -221,7 +228,7 @@ def absent(name, region=None, key=None, keyid=None, profile=None): ret['result'] = None return ret deleted = __salt__['boto_cfn.delete'](name, region, key, keyid, profile) - if isinstance(deleted, str): + if isinstance(deleted, six.string_types): code, message = _get_error(deleted) ret['comment'] = 'Stack {0} could not be deleted.\n{1}\n{2}'.format(name, code, message) ret['result'] = False @@ -250,8 +257,8 @@ def _get_template(template, name): def _validate(template_body=None, template_url=None, region=None, key=None, keyid=None, profile=None): # Validates template. returns true if template syntax is correct. validate = __salt__['boto_cfn.validate_template'](template_body, template_url, region, key, keyid, profile) - log.debug('Validate is result is {0}.'.format(str(validate))) - if isinstance(validate, str): + log.debug('Validate result is {0}.'.format(str(validate))) + if isinstance(validate, six.string_types): code, message = _get_error(validate) log.debug('Validate error is {0} and message is {1}.'.format(code, message)) return code, message diff --git a/salt/states/boto_cloudfront.py b/salt/states/boto_cloudfront.py new file mode 100644 index 00000000000..eb4d2ab940d --- /dev/null +++ b/salt/states/boto_cloudfront.py @@ -0,0 +1,229 @@ +# -*- coding: utf-8 -*- +''' +Manage CloudFront distributions + +.. versionadded:: Oxygen + +Create, update and destroy CloudFront distributions. + +This module accepts explicit AWS credentials but can also utilize +IAM roles assigned to the instance through Instance Profiles. +Dynamic credentials are then automatically obtained from AWS API +and no further configuration is necessary. +More information available `here +`_. + +If IAM roles are not used you need to specify them, +either in a pillar file or in the minion's config file: + +.. code-block:: yaml + + cloudfront.keyid: GKTADJGHEIQSXMKKRBJ08H + cloudfront.key: askdjghsdfjkghWupUjasdflkdfklgjsdfjajkghs + +It's also possible to specify ``key``, ``keyid``, and ``region`` via a profile, +either passed in as a dict, or a string to pull from pillars or minion config: + +.. code-block:: yaml + + myprofile: + keyid: GKTADJGHEIQSXMKKRBJ08H + key: askdjghsdfjkghWupUjasdflkdfklgjsdfjajkghs + region: us-east-1 + +.. code-block:: yaml + + aws: + region: + us-east-1: + profile: + keyid: GKTADJGHEIQSXMKKRBJ08H + key: askdjghsdfjkghWupUjasdflkdfklgjsdfjajkghs + region: us-east-1 + +:depends: boto3 +''' + +# Import Python Libs +from __future__ import absolute_import +import difflib +import logging + +import yaml + +log = logging.getLogger(__name__) + + +def __virtual__(): + ''' + Only load if boto is available. + ''' + if 'boto_cloudfront.get_distribution' not in __salt__: + msg = 'The boto_cloudfront state module could not be loaded: {}.' + return (False, msg.format('boto_cloudfront exec module unavailable.')) + return 'boto_cloudfront' + + +def present( + name, + config, + tags, + region=None, + key=None, + keyid=None, + profile=None, +): + ''' + Ensure the CloudFront distribution is present. + + name (string) + Name of the CloudFront distribution + + config (dict) + Configuration for the distribution + + tags (dict) + Tags to associate with the distribution + + region (string) + Region to connect to + + key (string) + Secret key to use + + keyid (string) + Access key to use + + profile (dict or string) + A dict with region, key, and keyid, + or a pillar key (string) that contains such a dict. + + Example: + + .. code-block:: yaml + + Manage my_distribution CloudFront distribution: + boto_cloudfront.present: + - name: my_distribution + - config: + Comment: 'partial config shown, most parameters elided' + Enabled: True + - tags: + testing_key: testing_value + ''' + ret = { + 'name': name, + 'comment': '', + 'changes': {}, + } + + res = __salt__['boto_cloudfront.get_distribution']( + name, + region=region, + key=key, + keyid=keyid, + profile=profile, + ) + if 'error' in res: + ret['result'] = False + ret['comment'] = 'Error checking distribution {0}: {1}'.format( + name, + res['error'], + ) + return ret + + old = res['result'] + if old is None: + if __opts__['test']: + ret['result'] = None + ret['comment'] = 'Distribution {0} set for creation.'.format(name) + ret['pchanges'] = {'old': None, 'new': name} + return ret + + res = __salt__['boto_cloudfront.create_distribution']( + name, + config, + tags, + region=region, + key=key, + keyid=keyid, + profile=profile, + ) + if 'error' in res: + ret['result'] = False + ret['comment'] = 'Error creating distribution {0}: {1}'.format( + name, + res['error'], + ) + return ret + + ret['result'] = True + ret['comment'] = 'Created distribution {0}.'.format(name) + ret['changes'] = {'old': None, 'new': name} + return ret + else: + full_config_old = { + 'config': old['distribution']['DistributionConfig'], + 'tags': old['tags'], + } + full_config_new = { + 'config': config, + 'tags': tags, + } + diffed_config = __utils__['dictdiffer.deep_diff']( + full_config_old, + full_config_new, + ) + + def _yaml_safe_dump(attrs): + '''Safely dump YAML using a readable flow style''' + dumper_name = 'IndentedSafeOrderedDumper' + dumper = __utils__['yamldumper.get_dumper'](dumper_name) + return yaml.dump( + attrs, + default_flow_style=False, + Dumper=dumper, + ) + changes_diff = ''.join(difflib.unified_diff( + _yaml_safe_dump(full_config_old).splitlines(True), + _yaml_safe_dump(full_config_new).splitlines(True), + )) + + any_changes = bool('old' in diffed_config or 'new' in diffed_config) + if not any_changes: + ret['result'] = True + ret['comment'] = 'Distribution {0} has correct config.'.format( + name, + ) + return ret + + if __opts__['test']: + ret['result'] = None + ret['comment'] = '\n'.join([ + 'Distribution {0} set for new config:'.format(name), + changes_diff, + ]) + ret['pchanges'] = {'diff': changes_diff} + return ret + + res = __salt__['boto_cloudfront.update_distribution']( + name, + config, + tags, + region=region, + key=key, + keyid=keyid, + profile=profile, + ) + if 'error' in res: + ret['result'] = False + ret['comment'] = 'Error updating distribution {0}: {1}'.format( + name, + res['error'], + ) + return ret + + ret['result'] = True + ret['comment'] = 'Updated distribution {0}.'.format(name) + ret['changes'] = {'diff': changes_diff} + return ret diff --git a/salt/states/boto_cloudtrail.py b/salt/states/boto_cloudtrail.py index e79823faf98..589c1d6a5b1 100644 --- a/salt/states/boto_cloudtrail.py +++ b/salt/states/boto_cloudtrail.py @@ -59,7 +59,7 @@ import os import os.path # Import Salt Libs -import salt.ext.six as six +from salt.ext import six import salt.utils log = logging.getLogger(__name__) diff --git a/salt/states/boto_cloudwatch_alarm.py b/salt/states/boto_cloudwatch_alarm.py index 19e48256945..a5433963351 100644 --- a/salt/states/boto_cloudwatch_alarm.py +++ b/salt/states/boto_cloudwatch_alarm.py @@ -57,7 +57,7 @@ as a passed in dict, or as a string to pull from pillars or minion config: from __future__ import absolute_import # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six def __virtual__(): diff --git a/salt/states/boto_cloudwatch_event.py b/salt/states/boto_cloudwatch_event.py index 6a799e776c5..3faddf7347a 100644 --- a/salt/states/boto_cloudwatch_event.py +++ b/salt/states/boto_cloudwatch_event.py @@ -60,7 +60,7 @@ import os.path import json # Import Salt Libs -import salt.ext.six as six +from salt.ext import six log = logging.getLogger(__name__) diff --git a/salt/states/boto_datapipeline.py b/salt/states/boto_datapipeline.py index 85f97882cd7..f29c3bcbc45 100644 --- a/salt/states/boto_datapipeline.py +++ b/salt/states/boto_datapipeline.py @@ -57,7 +57,7 @@ import difflib import json # Import Salt lobs -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import zip diff --git a/salt/states/boto_dynamodb.py b/salt/states/boto_dynamodb.py index 8ac818c99f7..5a2d2f7289f 100644 --- a/salt/states/boto_dynamodb.py +++ b/salt/states/boto_dynamodb.py @@ -163,7 +163,7 @@ import logging import copy # Import salt libs -import salt.ext.six as six +from salt.ext import six import salt.utils.dictupdate as dictupdate logging.basicConfig( diff --git a/salt/states/boto_ec2.py b/salt/states/boto_ec2.py index 5a543a9e255..1cacd624d97 100644 --- a/salt/states/boto_ec2.py +++ b/salt/states/boto_ec2.py @@ -57,7 +57,7 @@ import logging from time import time, sleep # Import salt libs -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import range # pylint: disable=import-error,no-name-in-module,redefined-builtin import salt.utils import salt.utils.dictupdate as dictupdate diff --git a/salt/states/boto_elasticsearch_domain.py b/salt/states/boto_elasticsearch_domain.py index af4537ffd58..2e544a655b8 100644 --- a/salt/states/boto_elasticsearch_domain.py +++ b/salt/states/boto_elasticsearch_domain.py @@ -86,7 +86,7 @@ import os.path import json # Import Salt Libs -import salt.ext.six as six +from salt.ext import six log = logging.getLogger(__name__) diff --git a/salt/states/boto_elb.py b/salt/states/boto_elb.py index b93c9dd6eb9..6163c09dfda 100644 --- a/salt/states/boto_elb.py +++ b/salt/states/boto_elb.py @@ -245,7 +245,7 @@ import re import salt.utils.dictupdate as dictupdate from salt.utils import exactly_one from salt.exceptions import SaltInvocationError -import salt.ext.six as six +from salt.ext import six log = logging.getLogger(__name__) @@ -1312,7 +1312,7 @@ def _tags_present(name, tags, region, key, keyid, profile): tags_to_add = tags tags_to_update = {} tags_to_remove = [] - if lb['tags']: + if lb.get('tags'): for _tag in lb['tags']: if _tag not in tags.keys(): if _tag not in tags_to_remove: diff --git a/salt/states/boto_elbv2.py b/salt/states/boto_elbv2.py index 417de4e1826..cd66b18d9a9 100644 --- a/salt/states/boto_elbv2.py +++ b/salt/states/boto_elbv2.py @@ -2,7 +2,7 @@ ''' Manage AWS Application Load Balancer -.. versionadded:: TBD +.. versionadded:: 2017.7.0 Add and remove targets from an ALB target group. @@ -40,7 +40,9 @@ from __future__ import absolute_import import logging import copy -# Import Salt Libs +# Import 3rd-party libs +from salt.ext import six + log = logging.getLogger(__name__) @@ -48,12 +50,159 @@ def __virtual__(): ''' Only load if boto is available. ''' - return 'boto_elbv2' if 'boto_elbv2.target_group_exists' in __salt__ else False + if 'boto_elbv2.target_group_exists' in __salt__: + return 'boto_elbv2' + return (False, "The boto_elbv2 module cannot be loaded: boto3 library not found") + + +def create_target_group(name, protocol, port, vpc_id, + region=None, key=None, keyid=None, profile=None, + health_check_protocol='HTTP', health_check_port='traffic-port', + health_check_path='/', health_check_interval_seconds=30, + health_check_timeout_seconds=5, healthy_threshold_count=5, + unhealthy_threshold_count=2, **kwargs): + + ''' + .. versionadded:: 2017.11.0 + + Create target group if not present. + + name + (string) - The name of the target group. + protocol + (string) - The protocol to use for routing traffic to the targets + port + (int) - The port on which the targets receive traffic. This port is used unless + you specify a port override when registering the traffic. + vpc_id + (string) - The identifier of the virtual private cloud (VPC). + health_check_protocol + (string) - The protocol the load balancer uses when performing health check on + targets. The default is the HTTP protocol. + health_check_port + (string) - The port the load balancer uses when performing health checks on + targets. The default is 'traffic-port', which indicates the port on which each + target receives traffic from the load balancer. + health_check_path + (string) - The ping path that is the destination on the targets for health + checks. The default is /. + health_check_interval_seconds + (integer) - The approximate amount of time, in seconds, between health checks + of an individual target. The default is 30 seconds. + health_check_timeout_seconds + (integer) - The amount of time, in seconds, during which no response from a + target means a failed health check. The default is 5 seconds. + healthy_threshold_count + (integer) - The number of consecutive health checks successes required before + considering an unhealthy target healthy. The default is 5. + unhealthy_threshold_count + (integer) - The number of consecutive health check failures required before + considering a target unhealthy. The default is 2. + + returns + (bool) - True on success, False on failure. + + CLI example: + .. code-block:: yaml + + create-target: + boto_elb2.create_targets_group: + - name: myALB + - protocol: https + - port: 443 + - vpc_id: myVPC + ''' + ret = {'name': name, 'result': None, 'comment': '', 'changes': {}} + + if __salt__['boto_elbv2.target_group_exists'](name, region, key, keyid, profile): + ret['result'] = True + ret['comment'] = 'Target Group {0} already exists'.format(name) + return ret + + if __opts__['test']: + ret['comment'] = 'Target Group {0} will be created'.format(name) + return ret + + state = __salt__['boto_elbv2.create_target_group'](name, + protocol, + port, + vpc_id, + region=region, + key=key, + keyid=keyid, + profile=profile, + health_check_protocol=health_check_protocol, + health_check_port=health_check_port, + health_check_path=health_check_path, + health_check_interval_seconds=health_check_interval_seconds, + health_check_timeout_seconds=health_check_timeout_seconds, + healthy_threshold_count=healthy_threshold_count, + unhealthy_threshold_count=unhealthy_threshold_count, + **kwargs) + + if state: + ret['changes']['target_group'] = name + ret['result'] = True + ret['comment'] = 'Target Group {0} created'.format(name) + else: + ret['result'] = False + ret['comment'] = 'Target Group {0} creation failed'.format(name) + return ret + + +def delete_target_group(name, region=None, key=None, keyid=None, profile=None): + ''' + Delete target group. + + name + (string) - The Amazon Resource Name (ARN) of the resource. + + returns + (bool) - True on success, False on failure. + + CLI example: + + .. code-block:: bash + + check-target: + boto_elb2.delete_targets_group: + - name: myALB + - protocol: https + - port: 443 + - vpc_id: myVPC + ''' + ret = {'name': name, 'result': None, 'comment': '', 'changes': {}} + + if not __salt__['boto_elbv2.target_group_exists'](name, region, key, keyid, profile): + ret['result'] = True + ret['comment'] = 'Target Group {0} does not exists'.format(name) + return ret + + if __opts__['test']: + ret['comment'] = 'Target Group {0} will be deleted'.format(name) + return ret + + state = __salt__['boto_elbv2.delete_target_group'](name, + region=region, + key=key, + keyid=keyid, + profile=profile) + + if state: + ret['result'] = True + ret['changes']['target_group'] = name + ret['comment'] = 'Target Group {0} deleted'.format(name) + else: + ret['result'] = False + ret['comment'] = 'Target Group {0} deletion failed'.format(name) + return ret def targets_registered(name, targets, region=None, key=None, keyid=None, profile=None, **kwargs): ''' + .. versionadded:: 2017.7.0 + Add targets to an Application Load Balancer target group. This state will not remove targets. name @@ -63,8 +212,6 @@ def targets_registered(name, targets, region=None, key=None, keyid=None, A list of target IDs or a string of a single target that this target group should distribute traffic to. - .. versionadded:: TBD - .. code-block:: yaml add-targets: @@ -75,15 +222,18 @@ def targets_registered(name, targets, region=None, key=None, keyid=None, - instance-id2 ''' ret = {'name': name, 'result': None, 'comment': '', 'changes': {}} - tg = __salt__['boto_elbv2.target_group_exists'](name, region, key, keyid, profile) - if tg: - health = __salt__['boto_elbv2.describe_target_health'](name, region=region, key=key, keyid=keyid, profile=profile) + if __salt__['boto_elbv2.target_group_exists'](name, region, key, keyid, profile): + health = __salt__['boto_elbv2.describe_target_health'](name, + region=region, + key=key, + keyid=keyid, + profile=profile) failure = False changes = False newhealth_mock = copy.copy(health) - if isinstance(targets, str): + if isinstance(targets, six.string_types): targets = [targets] for target in targets: @@ -97,10 +247,10 @@ def targets_registered(name, targets, region=None, key=None, keyid=None, else: state = __salt__['boto_elbv2.register_targets'](name, targets, - region, - key, - keyid, - profile) + region=region, + key=key, + keyid=keyid, + profile=profile) if state: changes = True ret['result'] = True @@ -117,7 +267,11 @@ def targets_registered(name, targets, region=None, key=None, keyid=None, ret['changes']['new'] = newhealth_mock else: ret['comment'] = 'Target Group {0} has been changed'.format(name) - newhealth = __salt__['boto_elbv2.describe_target_health'](name, region=region, key=key, keyid=keyid, profile=profile) + newhealth = __salt__['boto_elbv2.describe_target_health'](name, + region=region, + key=key, + keyid=keyid, + profile=profile) ret['changes']['new'] = newhealth return ret else: @@ -126,7 +280,7 @@ def targets_registered(name, targets, region=None, key=None, keyid=None, def targets_deregistered(name, targets, region=None, key=None, keyid=None, - profile=None, **kwargs): + profile=None, **kwargs): ''' Remove targets to an Application Load Balancer target group. @@ -148,13 +302,16 @@ def targets_deregistered(name, targets, region=None, key=None, keyid=None, - instance-id2 ''' ret = {'name': name, 'result': None, 'comment': '', 'changes': {}} - tg = __salt__['boto_elbv2.target_group_exists'](name, region, key, keyid, profile) - if tg: - health = __salt__['boto_elbv2.describe_target_health'](name, region=region, key=key, keyid=keyid, profile=profile) + if __salt__['boto_elbv2.target_group_exists'](name, region, key, keyid, profile): + health = __salt__['boto_elbv2.describe_target_health'](name, + region=region, + key=key, + keyid=keyid, + profile=profile) failure = False changes = False newhealth_mock = copy.copy(health) - if isinstance(targets, str): + if isinstance(targets, six.string_types): targets = [targets] for target in targets: if target not in health or health.get(target) == "draining": @@ -166,11 +323,11 @@ def targets_deregistered(name, targets, region=None, key=None, keyid=None, newhealth_mock.update({target: "draining"}) else: state = __salt__['boto_elbv2.deregister_targets'](name, - targets, - region, - key, - keyid, - profile) + targets, + region=region, + key=key, + keyid=keyid, + profile=profile) if state: changes = True ret['result'] = True @@ -187,7 +344,11 @@ def targets_deregistered(name, targets, region=None, key=None, keyid=None, ret['changes']['new'] = newhealth_mock else: ret['comment'] = 'Target Group {0} has been changed'.format(name) - newhealth = __salt__['boto_elbv2.describe_target_health'](name, region, key, keyid, profile) + newhealth = __salt__['boto_elbv2.describe_target_health'](name, + region=region, + key=key, + keyid=keyid, + profile=profile) ret['changes']['new'] = newhealth return ret else: diff --git a/salt/states/boto_iam.py b/salt/states/boto_iam.py index ed20500d125..f0f6f4402b8 100644 --- a/salt/states/boto_iam.py +++ b/salt/states/boto_iam.py @@ -139,9 +139,10 @@ import os # Import Salt Libs import salt.utils +import salt.utils.files import salt.utils.odict as odict import salt.utils.dictupdate as dictupdate -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import range # pylint: disable=import-error,redefined-builtin # Import 3rd party libs @@ -273,7 +274,7 @@ def user_absent(name, delete_keys=True, delete_mfa_devices=True, delete_profile= if profile_deleted: ret['comment'] = ' '.join([ret['comment'], 'IAM user {0} login profile is deleted.'.format(name)]) if __opts__['test']: - ret['comment'] = ' '.join([ret['comment'], 'IAM user {0} policies are set to be deleted.'.format(name)]) + ret['comment'] = ' '.join([ret['comment'], 'IAM user {0} managed policies are set to be detached.'.format(name)]) ret['result'] = None else: _ret = _user_policies_detached(name, region, key, keyid, profile) @@ -282,6 +283,16 @@ def user_absent(name, delete_keys=True, delete_mfa_devices=True, delete_profile= ret['result'] = _ret['result'] if ret['result'] is False: return ret + if __opts__['test']: + ret['comment'] = ' '.join([ret['comment'], 'IAM user {0} inline policies are set to be deleted.'.format(name)]) + ret['result'] = None + else: + _ret = _user_policies_deleted(name, region, key, keyid, profile) + ret['comment'] = ' '.join([ret['comment'], _ret['comment']]) + if not _ret['result']: + ret['result'] = _ret['result'] + if ret['result'] is False: + return ret # finally, actually delete the user if __opts__['test']: ret['comment'] = ' '.join([ret['comment'], 'IAM user {0} is set to be deleted.'.format(name)]) @@ -349,7 +360,7 @@ def keys_present(name, number, save_dir, region=None, key=None, keyid=None, prof return ret keys = __salt__['boto_iam.get_all_access_keys'](user_name=name, region=region, key=key, keyid=keyid, profile=profile) - if isinstance(keys, str): + if isinstance(keys, six.string_types): log.debug('keys are : false {0}'.format(keys)) error, message = _get_error(keys) ret['comment'] = 'Could not get keys.\n{0}\n{1}'.format(error, message) @@ -368,7 +379,7 @@ def keys_present(name, number, save_dir, region=None, key=None, keyid=None, prof new_keys = {} for i in range(number-len(keys)): created = __salt__['boto_iam.create_access_key'](name, region, key, keyid, profile) - if isinstance(created, str): + if isinstance(created, six.string_types): error, message = _get_error(created) ret['comment'] = 'Could not create keys.\n{0}\n{1}'.format(error, message) ret['result'] = False @@ -380,7 +391,7 @@ def keys_present(name, number, save_dir, region=None, key=None, keyid=None, prof new_keys[str(i)]['key_id'] = created[response][result]['access_key']['access_key_id'] new_keys[str(i)]['secret_key'] = created[response][result]['access_key']['secret_access_key'] try: - with salt.utils.fopen('{0}/{1}'.format(save_dir, name), 'a') as _wrf: + with salt.utils.files.fopen('{0}/{1}'.format(save_dir, name), 'a') as _wrf: for key_num, key in new_keys.items(): key_id = key['key_id'] secret_key = key['secret_key'] @@ -435,7 +446,7 @@ def _delete_key(ret, access_key_id, user_name, region=None, key=None, keyid=None keys = __salt__['boto_iam.get_all_access_keys'](user_name=user_name, region=region, key=key, keyid=keyid, profile=profile) log.debug('Keys for user {1} are : {0}.'.format(keys, user_name)) - if isinstance(keys, str): + if isinstance(keys, six.string_types): log.debug('Keys {0} are a string. Something went wrong.'.format(keys)) ret['comment'] = ' '.join([ret['comment'], 'Key {0} could not be deleted.'.format(access_key_id)]) return ret @@ -737,7 +748,49 @@ def _user_policies_detached( newpolicies = [x.get('policy_arn') for x in _list] ret['changes']['new'] = {'managed_policies': newpolicies} msg = '{0} policies detached from user {1}.' - ret['comment'] = msg.format(', '.join(newpolicies), name) + ret['comment'] = msg.format(', '.join(oldpolicies), name) + return ret + + +def _user_policies_deleted( + name, + region=None, + key=None, + keyid=None, + profile=None): + ret = {'result': True, 'comment': '', 'changes': {}} + oldpolicies = __salt__['boto_iam.get_all_user_policies'](user_name=name, + region=region, key=key, keyid=keyid, profile=profile) + if not oldpolicies: + msg = 'No inline policies in user {0}.'.format(name) + ret['comment'] = msg + return ret + if __opts__['test']: + msg = '{0} policies to be deleted from user {1}.' + ret['comment'] = msg.format(', '.join(oldpolicies), name) + ret['result'] = None + return ret + ret['changes']['old'] = {'inline_policies': oldpolicies} + for policy_name in oldpolicies: + policy_deleted = __salt__['boto_iam.delete_user_policy'](name, + policy_name, + region=region, key=key, + keyid=keyid, + profile=profile) + if not policy_deleted: + newpolicies = __salt__['boto_iam.get_all_user_policies'](name, region=region, + key=key, keyid=keyid, + profile=profile) + ret['changes']['new'] = {'inline_policies': newpolicies} + ret['result'] = False + msg = 'Failed to detach {0} from user {1}' + ret['comment'] = msg.format(policy_name, name) + return ret + newpolicies = __salt__['boto_iam.get_all_user_policies'](name, region=region, key=key, + keyid=keyid, profile=profile) + ret['changes']['new'] = {'inline_policies': newpolicies} + msg = '{0} policies deleted from user {1}.' + ret['comment'] = msg.format(', '.join(oldpolicies), name) return ret @@ -789,7 +842,7 @@ def group_absent(name, region=None, key=None, keyid=None, profile=None): ret['comment'] = 'IAM Group {0} does not exist.'.format(name) return ret if __opts__['test']: - ret['comment'] = ' '.join([ret['comment'], 'IAM group {0} policies are set to be deleted.'.format(name)]) + ret['comment'] = ' '.join([ret['comment'], 'IAM group {0} managed policies are set to be detached.'.format(name)]) ret['result'] = None else: _ret = _group_policies_detached(name, region, key, keyid, profile) @@ -798,6 +851,16 @@ def group_absent(name, region=None, key=None, keyid=None, profile=None): ret['result'] = _ret['result'] if ret['result'] is False: return ret + if __opts__['test']: + ret['comment'] = ' '.join([ret['comment'], 'IAM group {0} inline policies are set to be deleted.'.format(name)]) + ret['result'] = None + else: + _ret = _group_policies_deleted(name, region, key, keyid, profile) + ret['comment'] = ' '.join([ret['comment'], _ret['comment']]) + if not _ret['result']: + ret['result'] = _ret['result'] + if ret['result'] is False: + return ret ret['comment'] = ' '.join([ret['comment'], 'IAM group {0} users are set to be removed.'.format(name)]) existing_users = __salt__['boto_iam.get_group_members'](group_name=name, region=region, key=key, keyid=keyid, profile=profile) _ret = _case_group(ret, [], name, existing_users, region, key, keyid, profile) @@ -847,7 +910,7 @@ def group_present(name, policies=None, policies_from_pillars=None, managed_polic in the policies argument will override the keys defined in policies_from_pillars. - manaaged_policies (list) + managed_policies (list) A list of policy names or ARNs that should be attached to this group. users (list) @@ -1151,6 +1214,48 @@ def _group_policies_detached( return ret +def _group_policies_deleted( + name, + region=None, + key=None, + keyid=None, + profile=None): + ret = {'result': True, 'comment': '', 'changes': {}} + oldpolicies = __salt__['boto_iam.get_all_group_policies'](group_name=name, + region=region, key=key, keyid=keyid, profile=profile) + if not oldpolicies: + msg = 'No inline policies in group {0}.'.format(name) + ret['comment'] = msg + return ret + if __opts__['test']: + msg = '{0} policies to be deleted from group {1}.' + ret['comment'] = msg.format(', '.join(oldpolicies), name) + ret['result'] = None + return ret + ret['changes']['old'] = {'inline_policies': oldpolicies} + for policy_name in oldpolicies: + policy_deleted = __salt__['boto_iam.delete_group_policy'](name, + policy_name, + region=region, key=key, + keyid=keyid, + profile=profile) + if not policy_deleted: + newpolicies = __salt__['boto_iam.get_all_group_policies'](name, region=region, + key=key, keyid=keyid, + profile=profile) + ret['changes']['new'] = {'inline_policies': newpolicies} + ret['result'] = False + msg = 'Failed to detach {0} from group {1}' + ret['comment'] = msg.format(policy_name, name) + return ret + newpolicies = __salt__['boto_iam.get_all_group_policies'](name, region=region, key=key, + keyid=keyid, profile=profile) + ret['changes']['new'] = {'inline_policies': newpolicies} + msg = '{0} policies deleted from group {1}.' + ret['comment'] = msg.format(', '.join(oldpolicies), name) + return ret + + def account_policy(name=None, allow_users_to_change_password=None, hard_expiry=None, max_password_age=None, minimum_password_length=None, password_reuse_prevention=None, diff --git a/salt/states/boto_iam_role.py b/salt/states/boto_iam_role.py index 84b0a243432..26faff5e36b 100644 --- a/salt/states/boto_iam_role.py +++ b/salt/states/boto_iam_role.py @@ -89,7 +89,7 @@ on the IAM role to be persistent. This functionality was added in 2015.8.0. from __future__ import absolute_import import logging import salt.utils.dictupdate as dictupdate -import salt.ext.six as six +from salt.ext import six log = logging.getLogger(__name__) diff --git a/salt/states/boto_lambda.py b/salt/states/boto_lambda.py index c676ebfe6c5..e190a74d59d 100644 --- a/salt/states/boto_lambda.py +++ b/salt/states/boto_lambda.py @@ -69,9 +69,10 @@ import hashlib import json # Import Salt Libs -import salt.ext.six as six +from salt.ext import six import salt.utils.dictupdate as dictupdate import salt.utils +import salt.utils.files from salt.exceptions import SaltInvocationError log = logging.getLogger(__name__) @@ -407,7 +408,7 @@ def _function_code_present(FunctionName, ZipFile, S3Bucket, S3Key, size = os.path.getsize(ZipFile) if size == func['CodeSize']: sha = hashlib.sha256() - with salt.utils.fopen(ZipFile, 'rb') as f: + with salt.utils.files.fopen(ZipFile, 'rb') as f: sha.update(f.read()) hashed = sha.digest().encode('base64').strip() if hashed != func['CodeSha256']: diff --git a/salt/states/boto_route53.py b/salt/states/boto_route53.py index dddbc2705b9..930790660d1 100644 --- a/salt/states/boto_route53.py +++ b/salt/states/boto_route53.py @@ -76,7 +76,8 @@ from __future__ import absolute_import import json # Import Salt Libs -from salt.utils import SaltInvocationError, exactly_one +from salt.utils import exactly_one +from salt.exceptions import SaltInvocationError import logging log = logging.getLogger(__name__) diff --git a/salt/states/boto_sqs.py b/salt/states/boto_sqs.py index a97b37ba5d2..e9b142f8643 100644 --- a/salt/states/boto_sqs.py +++ b/salt/states/boto_sqs.py @@ -58,9 +58,15 @@ passed in as a dict, or as a string to pull from pillars or minion config: key: askdjghsdfjkghWupUjasdflkdfklgjsdfjajkghs ''' from __future__ import absolute_import -import salt.ext.six as six -import logging + +# Import Python libs +import difflib import json +import logging +import yaml + +# Import 3rd-party libs +from salt.ext import six log = logging.getLogger(__name__) @@ -78,7 +84,8 @@ def present( region=None, key=None, keyid=None, - profile=None): + profile=None, +): ''' Ensure the SQS queue exists. @@ -101,68 +108,143 @@ def present( A dict with region, key and keyid, or a pillar key (string) that contains a dict with region, key and keyid. ''' - ret = {'name': name, 'result': True, 'comment': '', 'changes': {}} + ret = { + 'name': name, + 'result': True, + 'comment': [], + 'changes': {}, + } - is_present = __salt__['boto_sqs.exists'](name, region, key, keyid, profile) + r = __salt__['boto_sqs.exists']( + name, + region=region, + key=key, + keyid=keyid, + profile=profile, + ) + if 'error' in r: + ret['result'] = False + ret['comment'].append(r['error']) + return ret - if not is_present: + if r['result']: + ret['comment'].append('SQS queue {0} present.'.format(name)) + else: if __opts__['test']: - msg = 'AWS SQS queue {0} is set to be created.'.format(name) - ret['comment'] = msg ret['result'] = None + ret['comment'].append( + 'SQS queue {0} is set to be created.'.format(name), + ) + ret['pchanges'] = {'old': None, 'new': name} return ret - created = __salt__['boto_sqs.create'](name, region, key, keyid, - profile) - if created: - ret['changes']['old'] = None - ret['changes']['new'] = {'queue': name} - else: + + r = __salt__['boto_sqs.create']( + name, + attributes=attributes, + region=region, + key=key, + keyid=keyid, + profile=profile, + ) + if 'error' in r: ret['result'] = False - ret['comment'] = 'Failed to create {0} AWS queue'.format(name) + ret['comment'].append( + 'Failed to create SQS queue {0}: {1}'.format(name, r['error']), + ) return ret - else: - ret['comment'] = '{0} present.'.format(name) + + ret['comment'].append('SQS queue {0} created.'.format(name)) + ret['changes']['old'] = None + ret['changes']['new'] = name + # Return immediately, as the create call also set all attributes + return ret + + if not attributes: + return ret + + r = __salt__['boto_sqs.get_attributes']( + name, + region=region, + key=key, + keyid=keyid, + profile=profile, + ) + if 'error' in r: + ret['result'] = False + ret['comment'].append( + 'Failed to get queue attributes: {0}'.format(r['error']), + ) + return ret + current_attributes = r['result'] + attrs_to_set = {} - _attributes = __salt__['boto_sqs.get_attributes'](name, region, key, keyid, - profile) - if attributes: - for attr, val in six.iteritems(attributes): - _val = _attributes.get(attr, None) - if attr == 'Policy': - # Normalize these guys by brute force.. - if isinstance(_val, six.string_types): - _val = json.loads(_val) - if isinstance(val, six.string_types): - val = json.loads(val) - if _val != val: - log.debug('Policies differ:\n{0}\n{1}'.format(_val, val)) - attrs_to_set[attr] = json.dumps(val, sort_keys=True) - elif str(_val) != str(val): - log.debug('Attributes differ:\n{0}\n{1}'.format(_val, val)) - attrs_to_set[attr] = val - attr_names = ','.join(attrs_to_set) - if attrs_to_set: - if __opts__['test']: - ret['comment'] = 'Attribute(s) {0} to be set on {1}.'.format( - attr_names, name) - ret['result'] = None - return ret - msg = (' Setting {0} attribute(s).'.format(attr_names)) - ret['comment'] = ret['comment'] + msg - if 'new' in ret['changes']: - ret['changes']['new']['attributes_set'] = [] - else: - ret['changes']['new'] = {'attributes_set': []} - for attr, val in six.iteritems(attrs_to_set): - set_attr = __salt__['boto_sqs.set_attributes'](name, {attr: val}, - region, key, keyid, - profile) - if not set_attr: - ret['result'] = False - msg = 'Set attribute {0}.'.format(attr) - ret['changes']['new']['attributes_set'].append(attr) - else: - ret['comment'] = ret['comment'] + ' Attributes set.' + for attr, val in six.iteritems(attributes): + _val = current_attributes.get(attr, None) + if attr == 'Policy': + # Normalize by brute force + if isinstance(_val, six.string_types): + _val = json.loads(_val) + if isinstance(val, six.string_types): + val = json.loads(val) + if _val != val: + log.debug('Policies differ:\n{0}\n{1}'.format(_val, val)) + attrs_to_set[attr] = json.dumps(val, sort_keys=True) + elif str(_val) != str(val): + log.debug('Attributes differ:\n{0}\n{1}'.format(_val, val)) + attrs_to_set[attr] = val + attr_names = ', '.join(attrs_to_set) + + if not attrs_to_set: + ret['comment'].append('Queue attributes already set correctly.') + return ret + + final_attributes = current_attributes.copy() + final_attributes.update(attrs_to_set) + + def _yaml_safe_dump(attrs): + '''Safely dump YAML using a readable flow style''' + dumper_name = 'IndentedSafeOrderedDumper' + dumper = __utils__['yamldumper.get_dumper'](dumper_name) + return yaml.dump( + attrs, + default_flow_style=False, + Dumper=dumper, + ) + attributes_diff = ''.join(difflib.unified_diff( + _yaml_safe_dump(current_attributes).splitlines(True), + _yaml_safe_dump(final_attributes).splitlines(True), + )) + + if __opts__['test']: + ret['result'] = None + ret['comment'].append( + 'Attribute(s) {0} set to be updated:\n{1}'.format( + attr_names, + attributes_diff, + ) + ) + ret['pchanges'] = {'attributes': {'diff': attributes_diff}} + return ret + + r = __salt__['boto_sqs.set_attributes']( + name, + attrs_to_set, + region=region, + key=key, + keyid=keyid, + profile=profile, + ) + if 'error' in r: + ret['result'] = False + ret['comment'].append( + 'Failed to set queue attributes: {0}'.format(r['error']), + ) + return ret + + ret['comment'].append( + 'Updated SQS queue attribute(s) {0}.'.format(attr_names), + ) + ret['changes']['attributes'] = {'diff': attributes_diff} return ret @@ -171,7 +253,8 @@ def absent( region=None, key=None, keyid=None, - profile=None): + profile=None, +): ''' Ensure the named sqs queue is deleted. @@ -193,23 +276,44 @@ def absent( ''' ret = {'name': name, 'result': True, 'comment': '', 'changes': {}} - is_present = __salt__['boto_sqs.exists'](name, region, key, keyid, profile) + r = __salt__['boto_sqs.exists']( + name, + region=region, + key=key, + keyid=keyid, + profile=profile, + ) + if 'error' in r: + ret['result'] = False + ret['comment'] = str(r['error']) + return ret - if is_present: - if __opts__['test']: - ret['comment'] = 'AWS SQS queue {0} is set to be removed.'.format( - name) - ret['result'] = None - return ret - deleted = __salt__['boto_sqs.delete'](name, region, key, keyid, - profile) - if deleted: - ret['changes']['old'] = name - ret['changes']['new'] = None - else: - ret['result'] = False - ret['comment'] = 'Failed to delete {0} sqs queue.'.format(name) - else: - ret['comment'] = '{0} does not exist in {1}.'.format(name, region) + if not r['result']: + ret['comment'] = 'SQS queue {0} does not exist in {1}.'.format( + name, + region, + ) + return ret + if __opts__['test']: + ret['result'] = None + ret['comment'] = 'SQS queue {0} is set to be removed.'.format(name) + ret['pchanges'] = {'old': name, 'new': None} + return ret + + r = __salt__['boto_sqs.delete']( + name, + region=region, + key=key, + keyid=keyid, + profile=profile, + ) + if 'error' in r: + ret['result'] = False + ret['comment'] = str(r['error']) + return ret + + ret['comment'] = 'SQS queue {0} was deleted.'.format(name) + ret['changes']['old'] = name + ret['changes']['new'] = None return ret diff --git a/salt/states/boto_vpc.py b/salt/states/boto_vpc.py index b2897896705..238431f2bbc 100644 --- a/salt/states/boto_vpc.py +++ b/salt/states/boto_vpc.py @@ -147,7 +147,7 @@ from __future__ import absolute_import import logging # Import Salt Libs -import salt.ext.six as six +from salt.ext import six import salt.utils.dictupdate as dictupdate __virtualname__ = 'boto_vpc' diff --git a/salt/states/bower.py b/salt/states/bower.py index ffcbb9c8a33..848f97d926e 100644 --- a/salt/states/bower.py +++ b/salt/states/bower.py @@ -36,7 +36,7 @@ from __future__ import absolute_import from salt.exceptions import CommandExecutionError, CommandNotFoundError # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six def __virtual__(): diff --git a/salt/states/cabal.py b/salt/states/cabal.py index 620b0cd160a..8db78609192 100644 --- a/salt/states/cabal.py +++ b/salt/states/cabal.py @@ -26,15 +26,15 @@ pkg.installed state for the package which provides cabal from __future__ import absolute_import from salt.exceptions import CommandExecutionError, CommandNotFoundError -import salt.utils +import salt.utils.path def __virtual__(): ''' Only work when cabal-install is installed. ''' - return (salt.utils.which('cabal') is not None) and \ - (salt.utils.which('ghc-pkg') is not None) + return (salt.utils.path.which('cabal') is not None) and \ + (salt.utils.path.which('ghc-pkg') is not None) def _parse_pkg_string(pkg): diff --git a/salt/states/chocolatey.py b/salt/states/chocolatey.py index 79c69048ef8..d83f9bddd39 100644 --- a/salt/states/chocolatey.py +++ b/salt/states/chocolatey.py @@ -92,7 +92,7 @@ def installed(name, version=None, source=None, force=False, pre_versions=False, # Determine action # Package not installed - if name not in [package.split('|')[0].lower() for package in pre_install.splitlines()]: + if name.lower() not in [package.lower() for package in pre_install.keys()]: if version: ret['changes'] = {name: 'Version {0} will be installed' ''.format(version)} @@ -193,9 +193,13 @@ def uninstalled(name, version=None, uninstall_args=None, override_args=False): pre_uninstall = __salt__['chocolatey.list'](local_only=True) # Determine if package is installed - if name in [package.split('|')[0].lower() for package in pre_uninstall.splitlines()]: - ret['changes'] = {name: '{0} version {1} will be removed' - ''.format(name, pre_uninstall[name][0])} + if name.lower() in [package.lower() for package in pre_uninstall.keys()]: + try: + ret['changes'] = {name: '{0} version {1} will be removed' + ''.format(name, pre_uninstall[name][0])} + except KeyError: + ret['changes'] = {name: '{0} will be removed' + ''.format(name)} else: ret['comment'] = 'The package {0} is not installed'.format(name) return ret diff --git a/salt/states/cimc.py b/salt/states/cimc.py new file mode 100644 index 00000000000..11b84839e7d --- /dev/null +++ b/salt/states/cimc.py @@ -0,0 +1,211 @@ +# -*- coding: utf-8 -*- +''' +A state module to manage Cisco UCS chassis devices. + +:codeauthor: :email:`Spencer Ervin ` +:maturity: new +:depends: none +:platform: unix + + +About +===== +This state module was designed to handle connections to a Cisco Unified Computing System (UCS) chassis. This module +relies on the CIMC proxy module to interface with the device. + +.. seealso:: + :prox:`CIMC Proxy Module ` + +''' + +# Import Python Libs +from __future__ import absolute_import +import logging + +log = logging.getLogger(__name__) + + +def __virtual__(): + return 'cimc.get_system_info' in __salt__ + + +def _default_ret(name): + ''' + Set the default response values. + + ''' + ret = { + 'name': name, + 'changes': {}, + 'result': False, + 'comment': '' + } + return ret + + +def ntp(name, servers): + ''' + Ensures that the NTP servers are configured. Servers are provided as an individual string or list format. Only four + NTP servers will be reviewed. Any entries past four will be ignored. + + name: The name of the module function to execute. + + servers(str, list): The IP address or FQDN of the NTP servers. + + SLS Example: + + .. code-block:: yaml + + ntp_configuration_list: + cimc.ntp: + - servers: + - foo.bar.com + - 10.10.10.10 + + ntp_configuration_str: + cimc.ntp: + - servers: foo.bar.com + + ''' + ret = _default_ret(name) + + ntp_servers = ['', '', '', ''] + + # Parse our server arguments + if isinstance(servers, list): + i = 0 + for x in servers: + ntp_servers[i] = x + i += 1 + else: + ntp_servers[0] = servers + + conf = __salt__['cimc.get_ntp']() + + # Check if our NTP configuration is already set + req_change = False + try: + if conf['outConfigs']['commNtpProvider'][0]['ntpEnable'] != 'yes' \ + or ntp_servers[0] != conf['outConfigs']['commNtpProvider'][0]['ntpServer1'] \ + or ntp_servers[1] != conf['outConfigs']['commNtpProvider'][0]['ntpServer2'] \ + or ntp_servers[2] != conf['outConfigs']['commNtpProvider'][0]['ntpServer3'] \ + or ntp_servers[3] != conf['outConfigs']['commNtpProvider'][0]['ntpServer4']: + req_change = True + except KeyError as err: + ret['result'] = False + ret['comment'] = "Unable to confirm current NTP settings." + log.error(err) + return ret + + if req_change: + + try: + update = __salt__['cimc.set_ntp_server'](ntp_servers[0], + ntp_servers[1], + ntp_servers[2], + ntp_servers[3]) + if update['outConfig']['commNtpProvider'][0]['status'] != 'modified': + ret['result'] = False + ret['comment'] = "Error setting NTP configuration." + return ret + except Exception as err: + ret['result'] = False + ret['comment'] = "Error setting NTP configuration." + log.error(err) + return ret + + ret['changes']['before'] = conf + ret['changes']['after'] = __salt__['cimc.get_ntp']() + ret['comment'] = "NTP settings modified." + else: + ret['comment'] = "NTP already configured. No changes required." + + ret['result'] = True + + return ret + + +def syslog(name, primary=None, secondary=None): + ''' + Ensures that the syslog servers are set to the specified values. A value of None will be ignored. + + name: The name of the module function to execute. + + primary(str): The IP address or FQDN of the primary syslog server. + + secondary(str): The IP address or FQDN of the secondary syslog server. + + SLS Example: + + .. code-block:: yaml + + syslog_configuration: + cimc.syslog: + - primary: 10.10.10.10 + - secondary: foo.bar.com + + ''' + ret = _default_ret(name) + + conf = __salt__['cimc.get_syslog']() + + req_change = False + + if primary: + prim_change = True + if 'outConfigs' in conf and 'commSyslogClient' in conf['outConfigs']: + for entry in conf['outConfigs']['commSyslogClient']: + if entry['name'] != 'primary': + continue + if entry['adminState'] == 'enabled' and entry['hostname'] == primary: + prim_change = False + + if prim_change: + try: + update = __salt__['cimc.set_syslog_server'](primary, "primary") + if update['outConfig']['commSyslogClient'][0]['status'] == 'modified': + req_change = True + else: + ret['result'] = False + ret['comment'] = "Error setting primary SYSLOG server." + return ret + except Exception as err: + ret['result'] = False + ret['comment'] = "Error setting primary SYSLOG server." + log.error(err) + return ret + + if secondary: + sec_change = True + if 'outConfig' in conf and 'commSyslogClient' in conf['outConfig']: + for entry in conf['outConfig']['commSyslogClient']: + if entry['name'] != 'secondary': + continue + if entry['adminState'] == 'enabled' and entry['hostname'] == secondary: + sec_change = False + + if sec_change: + try: + update = __salt__['cimc.set_syslog_server'](secondary, "secondary") + if update['outConfig']['commSyslogClient'][0]['status'] == 'modified': + req_change = True + else: + ret['result'] = False + ret['comment'] = "Error setting secondary SYSLOG server." + return ret + except Exception as err: + ret['result'] = False + ret['comment'] = "Error setting secondary SYSLOG server." + log.error(err) + return ret + + if req_change: + ret['changes']['before'] = conf + ret['changes']['after'] = __salt__['cimc.get_syslog']() + ret['comment'] = "SYSLOG settings modified." + else: + ret['comment'] = "SYSLOG already configured. No changes required." + + ret['result'] = True + + return ret diff --git a/salt/states/cisconso.py b/salt/states/cisconso.py index eb26cfaf332..7622a3102d9 100644 --- a/salt/states/cisconso.py +++ b/salt/states/cisconso.py @@ -8,6 +8,12 @@ For documentation on setting up the cisconso proxy minion look in the documentat for :mod:`salt.proxy.cisconso `. ''' +# Import Python libs +from __future__ import absolute_import + +# Import Salt libs +import salt.utils.compat + def __virtual__(): return 'cisconso.set_data_value' in __salt__ @@ -53,7 +59,7 @@ def value_present(name, datastore, path, config): existing = __salt__['cisconso.get_data'](datastore, path) - if cmp(existing, config): + if salt.utils.compat.cmp(existing, config): ret['result'] = True ret['comment'] = 'Config is already set' diff --git a/salt/states/cloud.py b/salt/states/cloud.py index 299bc2f35cf..773e589e0d9 100644 --- a/salt/states/cloud.py +++ b/salt/states/cloud.py @@ -19,7 +19,7 @@ from __future__ import absolute_import import pprint # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six # Import Salt Libs import salt.utils.cloud as suc diff --git a/salt/states/cmd.py b/salt/states/cmd.py index a07e3f5d785..7b7ef0aecdd 100644 --- a/salt/states/cmd.py +++ b/salt/states/cmd.py @@ -171,8 +171,8 @@ Should I use :mod:`cmd.run ` or :mod:`cmd.wait ` - instead of ``cmd.wait``. + Use :mod:`cmd.run ` together with :ref:`onchanges ` + instead of :mod:`cmd.wait `. These two states are often confused. The important thing to remember about them is that :mod:`cmd.run ` states are run each time the SLS @@ -241,6 +241,7 @@ import logging # Import salt libs import salt.utils +import salt.utils.args from salt.exceptions import CommandExecutionError, SaltRenderError from salt.ext.six import string_types @@ -276,7 +277,7 @@ def _reinterpreted_state(state): out = out[idx + 1:] data = {} try: - for item in salt.utils.shlex_split(out): + for item in salt.utils.args.shlex_split(out): key, val = item.split('=') data[key] = val except ValueError: @@ -415,7 +416,8 @@ def wait(name, .. note:: - Use :mod:`cmd.run ` with :mod:`onchange ` instead. + Use :mod:`cmd.run ` together with :mod:`onchanges ` + instead of :mod:`cmd.wait `. name The command to execute, remember that the command will execute with the @@ -837,7 +839,7 @@ def run(name, ret['changes'] = cmd_all ret['result'] = not bool(cmd_all['retcode']) - ret['comment'] = 'Command "{0}" run'.format(name) + ret['comment'] = u'Command "{0}" run'.format(name) # Ignore timeout errors if asked (for nohups) and treat cmd as a success if ignore_timeout: diff --git a/salt/states/composer.py b/salt/states/composer.py index 744f041117e..193801f898a 100644 --- a/salt/states/composer.py +++ b/salt/states/composer.py @@ -66,17 +66,17 @@ def installed(name, ''' Verify that the correct versions of composer dependencies are present. - dir - Directory location of the composer.json file. + name + Directory location of the ``composer.json`` file. composer - Location of the composer.phar file. If not set composer will - just execute "composer" as if it is installed globally. - (i.e. /path/to/composer.phar) + Location of the ``composer.phar`` file. If not set composer will + just execute ``composer`` as if it is installed globally. + (i.e. ``/path/to/composer.phar``) php Location of the php executable to use with composer. - (i.e. /usr/bin/php) + (i.e. ``/usr/bin/php``) user Which system user to run composer as. @@ -84,32 +84,32 @@ def installed(name, .. versionadded:: 2014.1.4 prefer_source - --prefer-source option of composer. + ``--prefer-source`` option of composer. prefer_dist - --prefer-dist option of composer. + ``--prefer-dist`` option of composer. no_scripts - --no-scripts option of composer. + ``--no-scripts`` option of composer. no_plugins - --no-plugins option of composer. + ``--no-plugins`` option of composer. optimize - --optimize-autoloader option of composer. Recommended for production. + ``--optimize-autoloader`` option of composer. Recommended for production. no_dev - --no-dev option for composer. Recommended for production. + ``--no-dev`` option for composer. Recommended for production. quiet - --quiet option for composer. Whether or not to return output from composer. + ``--quiet`` option for composer. Whether or not to return output from composer. composer_home - $COMPOSER_HOME environment variable + ``$COMPOSER_HOME`` environment variable always_check - If True, _always_ run `composer install` in the directory. This is the - default behavior. If False, only run `composer install` if there is no + If ``True``, *always* run ``composer install`` in the directory. This is the + default behavior. If ``False``, only run ``composer install`` if there is no vendor directory present. ''' ret = {'name': name, 'result': None, 'comment': '', 'changes': {}} @@ -193,17 +193,17 @@ def update(name, Composer update the directory to ensure we have the latest versions of all project dependencies. - dir - Directory location of the composer.json file. + name + Directory location of the ``composer.json`` file. composer - Location of the composer.phar file. If not set composer will - just execute "composer" as if it is installed globally. + Location of the ``composer.phar`` file. If not set composer will + just execute ``composer`` as if it is installed globally. (i.e. /path/to/composer.phar) php Location of the php executable to use with composer. - (i.e. /usr/bin/php) + (i.e. ``/usr/bin/php``) user Which system user to run composer as. @@ -211,28 +211,28 @@ def update(name, .. versionadded:: 2014.1.4 prefer_source - --prefer-source option of composer. + ``--prefer-source`` option of composer. prefer_dist - --prefer-dist option of composer. + ``--prefer-dist`` option of composer. no_scripts - --no-scripts option of composer. + ``--no-scripts`` option of composer. no_plugins - --no-plugins option of composer. + ``--no-plugins`` option of composer. optimize - --optimize-autoloader option of composer. Recommended for production. + ``--optimize-autoloader`` option of composer. Recommended for production. no_dev - --no-dev option for composer. Recommended for production. + ``--no-dev`` option for composer. Recommended for production. quiet - --quiet option for composer. Whether or not to return output from composer. + ``--quiet`` option for composer. Whether or not to return output from composer. composer_home - $COMPOSER_HOME environment variable + ``$COMPOSER_HOME`` environment variable ''' ret = {'name': name, 'result': None, 'comment': '', 'changes': {}} diff --git a/salt/states/cron.py b/salt/states/cron.py index b5d69e81ca1..c00d79ca872 100644 --- a/salt/states/cron.py +++ b/salt/states/cron.py @@ -116,7 +116,7 @@ entry on the minion already contains a numeric value, then using the ``random`` keyword will not modify it. Added the opportunity to set a job with a special keyword like '@reboot' or -'@hourly'. +'@hourly'. Quotes must be used, otherwise PyYAML will strip the '@' sign. .. code-block:: yaml @@ -143,7 +143,6 @@ from __future__ import absolute_import import os # Import salt libs -import salt.utils import salt.utils.files from salt.modules.cron import ( _needs_change, @@ -151,6 +150,13 @@ from salt.modules.cron import ( ) +def __virtual__(): + if 'cron.list_tab' in __salt__: + return True + else: + return (False, 'cron module could not be loaded') + + def _check_cron(user, cmd, minute=None, @@ -296,7 +302,8 @@ def present(name, edits. This defaults to the state id special - A special keyword to specify periodicity (eg. @reboot, @hourly...) + A special keyword to specify periodicity (eg. @reboot, @hourly...). + Quotes must be used, otherwise PyYAML will strip the '@' sign. .. versionadded:: 2016.3.0 ''' @@ -382,7 +389,8 @@ def absent(name, edits. This defaults to the state id special - The special keyword used in the job (eg. @reboot, @hourly...) + The special keyword used in the job (eg. @reboot, @hourly...). + Quotes must be used, otherwise PyYAML will strip the '@' sign. ''' ### NOTE: The keyword arguments in **kwargs are ignored in this state, but ### cannot be removed from the function definition, otherwise the use @@ -529,7 +537,7 @@ def file(name, return ret cron_path = salt.utils.files.mkstemp() - with salt.utils.fopen(cron_path, 'w+') as fp_: + with salt.utils.files.fopen(cron_path, 'w+') as fp_: raw_cron = __salt__['cron.raw_cron'](user) if not raw_cron.endswith('\n'): raw_cron = "{0}\n".format(raw_cron) diff --git a/salt/states/debconfmod.py b/salt/states/debconfmod.py index 67c60a1da9a..8eb5961074e 100644 --- a/salt/states/debconfmod.py +++ b/salt/states/debconfmod.py @@ -38,7 +38,7 @@ set_file the values in the ``data`` dict must be indented four spaces instead of two. ''' from __future__ import absolute_import -import salt.ext.six as six +from salt.ext import six # Define the module's virtual name diff --git a/salt/states/dellchassis.py b/salt/states/dellchassis.py index 3f6d1a86a9d..398cebaddf7 100644 --- a/salt/states/dellchassis.py +++ b/salt/states/dellchassis.py @@ -160,7 +160,7 @@ import logging import os # Import Salt lobs -import salt.ext.six as six +from salt.ext import six from salt.exceptions import CommandExecutionError # Get logging started diff --git a/salt/states/docker.py b/salt/states/docker.py index 962887445c1..cb506178617 100644 --- a/salt/states/docker.py +++ b/salt/states/docker.py @@ -60,7 +60,8 @@ import copy import logging # Import salt libs -import salt.utils +import salt.utils.args +import salt.utils.versions # Enable proper logging log = logging.getLogger(__name__) # pylint: disable=invalid-name @@ -87,14 +88,14 @@ def running(name, **kwargs): ''' ret = __states__['docker_container.running']( name, - **salt.utils.clean_kwargs(**kwargs) + **salt.utils.args.clean_kwargs(**kwargs) ) msg = ( 'The docker.running state has been renamed to ' 'docker_container.running. To get rid of this warning, update your ' 'SLS to use the new name.' ) - salt.utils.warn_until('Fluorine', msg) + salt.utils.versions.warn_until('Fluorine', msg) ret.setdefault('warnings', []).append(msg) return ret @@ -106,14 +107,14 @@ def stopped(**kwargs): `. ''' ret = __states__['docker_container.stopped']( - **salt.utils.clean_kwargs(**kwargs) + **salt.utils.args.clean_kwargs(**kwargs) ) msg = ( 'The docker.stopped state has been renamed to ' 'docker_container.stopped. To get rid of this warning, update your ' 'SLS to use the new name.' ) - salt.utils.warn_until('Fluorine', msg) + salt.utils.versions.warn_until('Fluorine', msg) ret.setdefault('warnings', []).append(msg) return ret @@ -126,14 +127,14 @@ def absent(name, **kwargs): ''' ret = __states__['docker_container.absent']( name, - **salt.utils.clean_kwargs(**kwargs) + **salt.utils.args.clean_kwargs(**kwargs) ) msg = ( 'The docker.absent state has been renamed to ' 'docker_container.absent. To get rid of this warning, update your ' 'SLS to use the new name.' ) - salt.utils.warn_until('Fluorine', msg) + salt.utils.versions.warn_until('Fluorine', msg) ret.setdefault('warnings', []).append(msg) return ret @@ -146,14 +147,14 @@ def network_present(name, **kwargs): ''' ret = __states__['docker_network.present']( name, - **salt.utils.clean_kwargs(**kwargs) + **salt.utils.args.clean_kwargs(**kwargs) ) msg = ( 'The docker.network_present state has been renamed to ' 'docker_network.present. To get rid of this warning, update your SLS ' 'to use the new name.' ) - salt.utils.warn_until('Fluorine', msg) + salt.utils.versions.warn_until('Fluorine', msg) ret.setdefault('warnings', []).append(msg) return ret @@ -166,14 +167,14 @@ def network_absent(name, **kwargs): ''' ret = __states__['docker_network.absent']( name, - **salt.utils.clean_kwargs(**kwargs) + **salt.utils.args.clean_kwargs(**kwargs) ) msg = ( 'The docker.network_absent state has been renamed to ' 'docker_network.absent. To get rid of this warning, update your SLS ' 'to use the new name.' ) - salt.utils.warn_until('Fluorine', msg) + salt.utils.versions.warn_until('Fluorine', msg) ret.setdefault('warnings', []).append(msg) return ret @@ -186,14 +187,14 @@ def image_present(name, **kwargs): ''' ret = __states__['docker_image.present']( name, - **salt.utils.clean_kwargs(**kwargs) + **salt.utils.args.clean_kwargs(**kwargs) ) msg = ( 'The docker.image_present state has been renamed to ' 'docker_image.present. To get rid of this warning, update your SLS ' 'to use the new name.' ) - salt.utils.warn_until('Fluorine', msg) + salt.utils.versions.warn_until('Fluorine', msg) ret.setdefault('warnings', []).append(msg) return ret @@ -205,14 +206,14 @@ def image_absent(**kwargs): `. ''' ret = __states__['docker_image.absent']( - **salt.utils.clean_kwargs(**kwargs) + **salt.utils.args.clean_kwargs(**kwargs) ) msg = ( 'The docker.image_absent state has been renamed to ' 'docker_image.absent. To get rid of this warning, update your SLS to ' 'use the new name.' ) - salt.utils.warn_until('Fluorine', msg) + salt.utils.versions.warn_until('Fluorine', msg) ret.setdefault('warnings', []).append(msg) return ret @@ -232,7 +233,7 @@ def volume_present(name, driver=None, driver_opts=None, force=False): 'docker_volume.present. To get rid of this warning, update your SLS ' 'to use the new name.' ) - salt.utils.warn_until('Fluorine', msg) + salt.utils.versions.warn_until('Fluorine', msg) ret.setdefault('warnings', []).append(msg) return ret @@ -249,7 +250,7 @@ def volume_absent(name, driver=None): 'docker_volume.absent. To get rid of this warning, update your SLS ' 'to use the new name.' ) - salt.utils.warn_until('Fluorine', msg) + salt.utils.versions.warn_until('Fluorine', msg) ret.setdefault('warnings', []).append(msg) return ret diff --git a/salt/states/docker_container.py b/salt/states/docker_container.py index b8eae874fc5..3e5a7646dc5 100644 --- a/salt/states/docker_container.py +++ b/salt/states/docker_container.py @@ -52,8 +52,9 @@ import logging from salt.exceptions import CommandExecutionError import copy import salt.utils +import salt.utils.args import salt.utils.docker -import salt.ext.six as six +from salt.ext import six # Enable proper logging log = logging.getLogger(__name__) # pylint: disable=invalid-name @@ -145,7 +146,7 @@ def running(name, .. _docker-container-running-skip-translate: skip_translate - This function translates Salt CLI input into the format which + This function translates Salt CLI or SLS input into the format which docker-py_ expects. However, in the event that Salt's translation logic fails (due to potential changes in the Docker Remote API, or to bugs in the translation code), this argument can be used to exert granular @@ -283,10 +284,35 @@ def running(name, binds Files/directories to bind mount. Each bind mount should be passed in - the format ``::``, where - ```` is one of ``rw`` (for read-write access) or ``ro`` (for - read-only access). Can be expressed as a comma-separated list or a YAML - list. The below two examples are equivalent: + one of the following formats: + + - ``:`` - ``host_path`` is mounted within + the container as ``container_path`` with read-write access. + - ``::`` - ``host_path`` is + mounted within the container as ``container_path`` with read-write + access. Additionally, the specified selinux context will be set + within the container. + - ``::`` - ``host_path`` is + mounted within the container as ``container_path``, with the + read-only or read-write setting explicitly defined. + - ``::,`` - + ``host_path`` is mounted within the container as ``container_path``, + with the read-only or read-write setting explicitly defined. + Additionally, the specified selinux context will be set within the + container. + + ```` can be either ``ro`` for read-write access, or ``ro`` + for read-only access. When omitted, it is assumed to be read-write. + + ```` can be ``z`` if the volume is shared between + multiple containers, or ``Z`` if the volume should be private. + + .. note:: + When both ```` and ```` are specified, + there must be a comma before ````. + + Binds can be expressed as a comma-separated list or a YAML list. The + below two examples are equivalent: .. code-block:: yaml @@ -304,9 +330,8 @@ def running(name, - /srv/www:/var/www:ro - /home/myuser/conf/foo.conf:/etc/foo.conf:rw - Optionally, the read-only information can be left off the end and the - bind mount will be assumed to be read-write. The example below is - equivalent to the one above: + However, in cases where both ro/rw and an selinux context are combined, + the only option is to use a YAML list, like so: .. code-block:: yaml @@ -314,8 +339,20 @@ def running(name, docker_container.running: - image: bar/baz:latest - binds: - - /srv/www:/var/www:ro - - /home/myuser/conf/foo.conf:/etc/foo.conf + - /srv/www:/var/www:ro,Z + - /home/myuser/conf/foo.conf:/etc/foo.conf:rw,Z + + Since the second bind in the previous example is mounted read-write, + the ``rw`` and comma can be dropped. For example: + + .. code-block:: yaml + + foo: + docker_container.running: + - image: bar/baz:latest + - binds: + - /srv/www:/var/www:ro,Z + - /home/myuser/conf/foo.conf:/etc/foo.conf:Z blkio_weight Block IO weight (relative weight), accepts a weight value between 10 @@ -641,24 +678,14 @@ def running(name, - foo2.domain.tld domainname - Set custom DNS search domains. Can be expressed as a comma-separated - list or a YAML list. The below two examples are equivalent: + The domain name to use for the container .. code-block:: yaml foo: docker_container.running: - image: bar/baz:latest - - dommainname: domain.tld,domain2.tld - - .. code-block:: yaml - - foo: - docker_container.running: - - image: bar/baz:latest - - dommainname: - - domain.tld - - domain2.tld + - dommainname: domain.tld entrypoint Entrypoint for the container @@ -2032,7 +2059,7 @@ def mod_watch(name, sfun=None, **kwargs): return running(name, **watch_kwargs) if sfun == 'stopped': - return stopped(name, **salt.utils.clean_kwargs(**kwargs)) + return stopped(name, **salt.utils.args.clean_kwargs(**kwargs)) return {'name': name, 'changes': {}, diff --git a/salt/states/docker_image.py b/salt/states/docker_image.py index e3c1a377797..670fc891117 100644 --- a/salt/states/docker_image.py +++ b/salt/states/docker_image.py @@ -40,6 +40,8 @@ import logging # Import salt libs import salt.utils.docker +import salt.utils.args +from salt.ext.six.moves import zip # Enable proper logging log = logging.getLogger(__name__) # pylint: disable=invalid-name @@ -135,13 +137,14 @@ def present(name, .. versionadded:: 2016.11.0 sls - Allow for building images with ``dockerng.sls_build`` by specify the - SLS files to build with. This can be a list or comma-seperated string. + Allow for building of image with :py:func:`docker.sls_build + ` by specifying the SLS files with + which to build. This can be a list or comma-seperated string. .. code-block:: yaml myuser/myimage:mytag: - dockerng.image_present: + docker_image.present: - sls: - webapp1 - webapp2 @@ -151,12 +154,14 @@ def present(name, .. versionadded: 2017.7.0 base - Base image with which to start ``dockerng.sls_build`` + Base image with which to start :py:func:`docker.sls_build + ` .. versionadded: 2017.7.0 saltenv - environment from which to pull sls files for ``dockerng.sls_build``. + Environment from which to pull SLS files for :py:func:`docker.sls_build + ` .. versionadded: 2017.7.0 ''' @@ -169,11 +174,14 @@ def present(name, ret['comment'] = 'Only one of \'build\' or \'load\' is permitted.' return ret - # Ensure that we have repo:tag notation image = ':'.join(salt.utils.docker.get_repo_tag(name)) - all_tags = __salt__['docker.list_tags']() + resolved_tag = __salt__['docker.resolve_tag'](image) - if image in all_tags: + if resolved_tag is False: + # Specified image is not present + image_info = None + else: + # Specified image is present if not force: ret['result'] = True ret['comment'] = 'Image \'{0}\' already present'.format(name) @@ -185,8 +193,6 @@ def present(name, ret['comment'] = \ 'Unable to get info for image \'{0}\': {1}'.format(name, exc) return ret - else: - image_info = None if build or sls: action = 'built' @@ -197,15 +203,24 @@ def present(name, if __opts__['test']: ret['result'] = None - if (image in all_tags and force) or image not in all_tags: + if (resolved_tag is not False and force) or resolved_tag is False: ret['comment'] = 'Image \'{0}\' will be {1}'.format(name, action) return ret if build: + # get the functions default value and args + argspec = salt.utils.args.get_function_argspec(__salt__['docker.build']) + # Map any if existing args from kwargs into the build_args dictionary + build_args = dict(list(zip(argspec.args, argspec.defaults))) + for k, v in build_args.items(): + if k in kwargs.get('kwargs', {}): + build_args[k] = kwargs.get('kwargs', {}).get(k) try: - image_update = __salt__['docker.build'](path=build, - image=image, - dockerfile=dockerfile) + # map values passed from the state to the build args + build_args['path'] = build + build_args['image'] = image + build_args['dockerfile'] = dockerfile + image_update = __salt__['docker.build'](**build_args) except Exception as exc: ret['comment'] = ( 'Encountered error building {0} as {1}: {2}' @@ -219,10 +234,10 @@ def present(name, if isinstance(sls, list): sls = ','.join(sls) try: - image_update = __salt__['dockerng.sls_build'](name=image, - base=base, - mods=sls, - saltenv=saltenv) + image_update = __salt__['docker.sls_build'](name=image, + base=base, + mods=sls, + saltenv=saltenv) except Exception as exc: ret['comment'] = ( 'Encountered error using sls {0} for building {1}: {2}' @@ -252,10 +267,8 @@ def present(name, client_timeout=client_timeout ) except Exception as exc: - ret['comment'] = ( - 'Encountered error pulling {0}: {1}' - .format(image, exc) - ) + ret['comment'] = \ + 'Encountered error pulling {0}: {1}'.format(image, exc) return ret if (image_info is not None and image_info['Id'][:12] == image_update .get('Layers', {}) @@ -267,7 +280,7 @@ def present(name, # Only add to the changes dict if layers were pulled ret['changes'] = image_update - ret['result'] = image in __salt__['docker.list_tags']() + ret['result'] = bool(__salt__['docker.resolve_tag'](image)) if not ret['result']: # This shouldn't happen, failure to pull should be caught above @@ -345,23 +358,16 @@ def absent(name=None, images=None, force=False): ret['comment'] = 'One of \'name\' and \'images\' must be provided' return ret elif images is not None: - targets = [] - for target in images: - try: - targets.append(':'.join(salt.utils.docker.get_repo_tag(target))) - except TypeError: - # Don't stomp on images with unicode characters in Python 2, - # only force image to be a str if it wasn't already (which is - # very unlikely). - targets.append(':'.join(salt.utils.docker.get_repo_tag(str(target)))) + targets = images elif name: - try: - targets = [':'.join(salt.utils.docker.get_repo_tag(name))] - except TypeError: - targets = [':'.join(salt.utils.docker.get_repo_tag(str(name)))] + targets = [name] pre_tags = __salt__['docker.list_tags']() - to_delete = [x for x in targets if x in pre_tags] + to_delete = [] + for target in targets: + resolved_tag = __salt__['docker.resolve_tag'](target, tags=pre_tags) + if resolved_tag is not False: + to_delete.append(resolved_tag) log.debug('targets = {0}'.format(targets)) log.debug('to_delete = {0}'.format(to_delete)) diff --git a/salt/states/docker_network.py b/salt/states/docker_network.py index 5faf35d55ed..d5d57afc6b2 100644 --- a/salt/states/docker_network.py +++ b/salt/states/docker_network.py @@ -34,6 +34,7 @@ from __future__ import absolute_import import logging # Import salt libs +from salt.ext import six import salt.utils # Enable proper logging @@ -56,6 +57,9 @@ def __virtual__(): def present(name, driver=None, driver_opts=None, + gateway=None, + ip_range=None, + subnet=None, containers=None): ''' Ensure that a network is present. @@ -69,9 +73,18 @@ def present(name, driver_opts Options for the network driver. + gateway + IPv4 or IPv6 gateway for the master subnet + + ip_range + Allocate container IP from a sub-range within the subnet + containers: List of container names that should be part of this network + subnet: + Subnet in CIDR format that represents a network segment + Usage Examples: .. code-block:: yaml @@ -91,6 +104,18 @@ def present(name, - cont1 - cont2 + + .. code-block:: yaml + + network_baz: + docker_network.present + - name: baz + - driver_opts: + - parent: eth0 + - gateway: "172.20.0.1" + - ip_range: "172.20.0.128/25" + - subnet: "172.20.0.0/24" + ''' ret = {'name': name, 'changes': {}, @@ -100,51 +125,197 @@ def present(name, if salt.utils.is_dictlist(driver_opts): driver_opts = salt.utils.repack_dictlist(driver_opts) - if containers is None: - containers = [] - # map containers to container's Ids. - containers = [__salt__['docker.inspect_container'](c)['Id'] for c in containers] - networks = __salt__['docker.networks'](names=[name]) - if networks: - network = networks[0] # we expect network's name to be unique - if all(c in network['Containers'] for c in containers): - ret['result'] = True - ret['comment'] = 'Network \'{0}\' already exists.'.format(name) - return ret - result = True - for container in containers: - if container not in network['Containers']: - try: - ret['changes']['connected'] = __salt__['docker.connect_container_to_network']( - container, name) - except Exception as exc: - ret['comment'] = ('Failed to connect container \'{0}\' to network \'{1}\' {2}'.format( - container, name, exc)) - result = False - ret['result'] = result + # If any containers are specified, get details of each one, we need the Id and Name fields later + if containers is not None: + containers = [__salt__['docker.inspect_container'](c) for c in containers] + networks = __salt__['docker.networks'](names=[name]) + log.trace( + 'docker_network.present: current networks: {0}'.format(networks) + ) + + # networks will contain all Docker networks which partially match 'name'. + # We need to loop through to find the matching network, if there is one. + network = None + if networks: + for network_iter in networks: + if network_iter['Name'] == name: + network = network_iter + break + + # We might disconnect containers in the process of recreating the network, we'll need to keep track these containers + # so we can reconnect them later. + containers_disconnected = {} + + # If the network already exists + if network is not None: + log.debug('Network \'{0}\' already exists'.format(name)) + + # Set the comment now to say that it already exists, if we need to recreate the network with new config we'll + # update the comment later. + ret['comment'] = 'Network \'{0}\' already exists'.format(name) + + # Update network details with result from network inspect, which will contain details of any containers + # attached to the network. + network = __salt__['docker.inspect_network'](network_id=network['Id']) + + log.trace('Details of \'{0}\' network: {1}'.format(name, network)) + + # For the IPAM and driver config options which can be passed, check that if they are passed, they match the + # current configuration. + original_config = {} + new_config = {} + + if driver and driver != network['Driver']: + new_config['driver'] = driver + original_config['driver'] = network['Driver'] + + if driver_opts and driver_opts != network['Options']: + new_config['driver_opts'] = driver_opts + original_config['driver_opts'] = network['Options'] + + # Multiple IPAM configs is probably not that common so for now we'll only worry about the simple case where + # there's a single IPAM config. If there's more than one (or none at all) then we'll bail out. + if len(network['IPAM']['Config']) != 1: + ret['comment'] = ('docker_network.present does only supports Docker networks with a single IPAM config,' + 'network \'{0}\' has {1}'.format(name, len(network['IPAM']['Config']))) + return ret + + ipam = network['IPAM']['Config'][0] + + if gateway and gateway != ipam['Gateway']: + new_config['gateway'] = gateway + original_config['gateway'] = ipam['Gateway'] + + if subnet and subnet != ipam['Subnet']: + new_config['subnet'] = subnet + original_config['subnet'] = ipam['Subnet'] + + if ip_range: + # IPRange isn't always configured so check it's even set before attempting to compare it. + if 'IPRange' in ipam and ip_range != ipam['IPRange']: + new_config['ip_range'] = ip_range + original_config['ip_range'] = ipam['IPRange'] + elif 'IPRange' not in ipam: + new_config['ip_range'] = ip_range + original_config['ip_range'] = '' + + if new_config != original_config: + log.debug('New config is different to current;\nnew: {0}\ncurrent: {1}'.format(new_config, original_config)) + + if __opts__['test']: + ret['result'] = None + ret['comment'] = 'Network {0} will be recreated with new config'.format(name) + return ret + + remove_result = _remove_network(name, network['Containers']) + if not remove_result['result']: + return remove_result + + # We've removed the network, so there are now no containers attached to it. + if network['Containers']: + containers_disconnected = network['Containers'] + network['Containers'] = [] + + try: + __salt__['docker.create_network']( + name, + driver=driver, + driver_opts=driver_opts, + gateway=gateway, + ip_range=ip_range, + subnet=subnet) + except Exception as exc: + ret['comment'] = ('Failed to replace network \'{0}\': {1}' + .format(name, exc)) + return ret + + ret['changes']['updated'] = {name: {'old': original_config, 'new': new_config}} + ret['comment'] = 'Network \'{0}\' was replaced with updated config'.format(name) + + # If the network does not yet exist, we create it else: + log.debug('The network \'{0}\' will be created'.format(name)) + if __opts__['test']: + ret['result'] = None + ret['comment'] = ('The network \'{0}\' will be created'.format(name)) + return ret try: ret['changes']['created'] = __salt__['docker.create_network']( name, driver=driver, driver_opts=driver_opts, - check_duplicate=True) + gateway=gateway, + ip_range=ip_range, + subnet=subnet) except Exception as exc: ret['comment'] = ('Failed to create network \'{0}\': {1}' .format(name, exc)) - else: - result = True - for container in containers: - try: - ret['changes']['connected'] = __salt__['docker.connect_container_to_network']( - container, name) - except Exception as exc: - ret['comment'] = ('Failed to connect container \'{0}\' to network \'{1}\' {2}'.format( - container, name, exc)) - result = False - ret['result'] = result + return ret + + # Finally, figure out the list of containers which should now be connected. + containers_to_connect = {} + # If no containers were specified in the state but we have disconnected some in the process of recreating the + # network, we should reconnect those containers. + if containers is None and containers_disconnected: + containers_to_connect = containers_disconnected + # If containers were specified in the state, regardless of what we've disconnected, we should now just connect + # the containers specified. + elif containers: + for container in containers: + containers_to_connect[container['Id']] = container + + if network is None: + network = {'Containers': {}} + + # At this point, if all the containers we want connected are already connected to the network, we can set our + # result and finish. + if all(c in network['Containers'] for c in containers_to_connect): + ret['result'] = True + return ret + + # If we've not exited by this point it's because we have containers which we need to connect to the network. + result = True + reconnected_containers = [] + connected_containers = [] + for container_id, container in six.iteritems(containers_to_connect): + if container_id not in network['Containers']: + try: + connect_result = __salt__['docker.connect_container_to_network'](container_id, name) + log.trace( + 'docker.connect_container_to_network({0}, {1}) result: {2}'. + format(container, name, connect_result) + ) + # If this container was one we disconnected earlier, add it to the reconnected list. + if container_id in containers_disconnected: + reconnected_containers.append(container['Name']) + # Otherwise add it to the connected list. + else: + connected_containers.append(container['Name']) + + except Exception as exc: + ret['comment'] = ('Failed to connect container \'{0}\' to network \'{1}\' {2}'.format( + container['Name'], name, exc)) + result = False + + # If we populated any of our container lists then add them to our list of changes. + if connected_containers: + ret['changes']['connected'] = connected_containers + if reconnected_containers: + ret['changes']['reconnected'] = reconnected_containers + + # Figure out if we removed any containers as a result of replacing the network and then not re-connecting the + # containers, because they weren't specified in the state. + disconnected_containers = [] + for container_id, container in six.iteritems(containers_disconnected): + if container_id not in containers_to_connect: + disconnected_containers.append(container['Name']) + + if disconnected_containers: + ret['changes']['disconnected'] = disconnected_containers + + ret['result'] = result return ret @@ -169,21 +340,55 @@ def absent(name, driver=None): 'comment': ''} networks = __salt__['docker.networks'](names=[name]) - if not networks: + log.trace( + 'docker_network.absent: current networks: {0}'.format(networks) + ) + + # networks will contain all Docker networks which partially match 'name'. + # We need to loop through to find the matching network, if there is one. + network = None + if networks: + for network_iter in networks: + if network_iter['Name'] == name: + network = network_iter + break + + if network is None: ret['result'] = True ret['comment'] = 'Network \'{0}\' already absent'.format(name) return ret - for container in networks[0]['Containers']: + if __opts__['test']: + ret['result'] = None + ret['comment'] = ('The network \'{0}\' will be removed'.format(name)) + return ret + + return _remove_network(network=name, containers=networks[0]['Containers']) + + +def _remove_network(network, containers=None): + ''' + Remove network, removing any specified containers from it beforehand + ''' + + ret = {'name': network, + 'changes': {}, + 'result': False, + 'comment': ''} + + if containers is None: + containers = [] + for container in containers: try: - ret['changes']['disconnected'] = __salt__['docker.disconnect_container_from_network'](container, name) + ret['changes']['disconnected'] = __salt__['docker.disconnect_container_from_network'](container, network) except Exception as exc: - ret['comment'] = ('Failed to disconnect container \'{0}\' to network \'{1}\' {2}'.format( - container, name, exc)) + ret['comment'] = ('Failed to disconnect container \'{0}\' from network \'{1}\' {2}'.format( + container, network, exc)) try: - ret['changes']['removed'] = __salt__['docker.remove_network'](name) + ret['changes']['removed'] = __salt__['docker.remove_network'](network) ret['result'] = True except Exception as exc: ret['comment'] = ('Failed to remove network \'{0}\': {1}' - .format(name, exc)) + .format(network, exc)) + return ret diff --git a/salt/states/docker_volume.py b/salt/states/docker_volume.py index 84a4f2154d0..209b3839ac2 100644 --- a/salt/states/docker_volume.py +++ b/salt/states/docker_volume.py @@ -187,7 +187,7 @@ def present(name, driver=None, driver_opts=None, force=False): ret['result'] = result return ret - ret['result'] = None if __opts__['test'] else True + ret['result'] = True ret['comment'] = 'Volume \'{0}\' already exists.'.format(name) return ret diff --git a/salt/states/drac.py b/salt/states/drac.py index 11ee84abbd2..534efacbfb0 100644 --- a/salt/states/drac.py +++ b/salt/states/drac.py @@ -39,13 +39,14 @@ Ensure DRAC network is in a consistent state from __future__ import absolute_import import salt.exceptions +import salt.utils.path def __virtual__(): ''' Ensure the racadm command is installed ''' - if salt.utils.which('racadm'): + if salt.utils.path.which('racadm'): return True return False diff --git a/salt/states/elasticsearch.py b/salt/states/elasticsearch.py index 2c37a304cef..c5f297db59b 100644 --- a/salt/states/elasticsearch.py +++ b/salt/states/elasticsearch.py @@ -230,7 +230,7 @@ def index_template_absent(name): return ret -def index_template_present(name, definition): +def index_template_present(name, definition, check_definition=False): ''' Ensure that the named index templat eis present. @@ -238,6 +238,8 @@ def index_template_present(name, definition): Name of the index to add definition Required dict for creation parameters as per https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-templates.html + check_definition + If the template already exists and the definition is up to date **Example:** @@ -270,7 +272,27 @@ def index_template_present(name, definition): ret['result'] = False ret['comment'] = 'Cannot create index template {0}, {1}'.format(name, output) else: - ret['comment'] = 'Index template {0} is already present'.format(name) + if check_definition: + definition_parsed = json.loads(definition) + current_template = __salt__['elasticsearch.index_template_get'](name=name)[name] + diff = __utils__['dictdiffer.deep_diff'](current_template, definition_parsed) + if len(diff) != 0: + if __opts__['test']: + ret['comment'] = 'Index template {0} exist but need to be updated'.format(name) + ret['changes'] = diff + ret['result'] = None + else: + output = __salt__['elasticsearch.index_template_create'](name=name, body=definition) + if output: + ret['comment'] = 'Successfully updated index template {0}'.format(name) + ret['changes'] = diff + else: + ret['result'] = False + ret['comment'] = 'Cannot update index template {0}, {1}'.format(name, output) + else: + ret['comment'] = 'Index template {0} is already present and up to date'.format(name) + else: + ret['comment'] = 'Index template {0} is already present'.format(name) except Exception as e: ret['result'] = False ret['comment'] = str(e) diff --git a/salt/states/environ.py b/salt/states/environ.py index c2bddfcb8af..8c3ea502261 100644 --- a/salt/states/environ.py +++ b/salt/states/environ.py @@ -4,13 +4,15 @@ Support for getting and setting the environment variables of the current salt process. ''' -# Import python libs +# Import Python libs from __future__ import absolute_import import os -# Import salt libs -import salt.utils as utils -import salt.ext.six as six +# Import Salt libs +import salt.utils.platform + +# Import 3rd-party libs +from salt.ext import six def __virtual__(): @@ -117,7 +119,7 @@ def setenv(name, # false_unsets is True. Otherwise we want to set # the value to '' def key_exists(): - if utils.is_windows(): + if salt.utils.platform.is_windows(): permanent_hive = 'HKCU' permanent_key = 'Environment' if permanent == 'HKLM': diff --git a/salt/states/esxdatacenter.py b/salt/states/esxdatacenter.py new file mode 100644 index 00000000000..462a7633d79 --- /dev/null +++ b/salt/states/esxdatacenter.py @@ -0,0 +1,134 @@ +# -*- coding: utf-8 -*- +''' +Salt states to create and manage VMware vSphere datacenters (datacenters). + +:codeauthor: :email:`Alexandru Bleotu ` + +Dependencies +============ + +- pyVmomi Python Module + +States +====== + +datacenter_configured +--------------------- + +Makes sure a datacenter exists and is correctly configured. + +If the state is run by an ``esxdatacenter`` minion, the name of the datacenter +is retrieved from the proxy details, otherwise the datacenter has the same name +as the state. + +Supported proxies: esxdatacenter + + +Example: + +1. Make sure that a datacenter named ``target_dc`` exists on the vCenter, using a +``esxdatacenter`` proxy: + +Proxy minion configuration (connects passthrough to the vCenter): + +.. code-block:: yaml + proxy: + proxytype: esxdatacenter + datacenter: target_dc + vcenter: vcenter.fake.com + mechanism: sspi + domain: fake.com + principal: host + +State configuration: + +.. code-block:: yaml + + datacenter_state: + esxdatacenter.datacenter_configured +''' + +# Import Python Libs +from __future__ import absolute_import +import logging + +# Import Salt Libs +import salt.exceptions + +# Get Logging Started +log = logging.getLogger(__name__) +LOGIN_DETAILS = {} + + +def __virtual__(): + return 'esxdatacenter' + + +def mod_init(low): + return True + + +def datacenter_configured(name): + ''' + Makes sure a datacenter exists. + + If the state is run by an ``esxdatacenter`` minion, the name of the + datacenter is retrieved from the proxy details, otherwise the datacenter + has the same name as the state. + + Supported proxies: esxdatacenter + + name: + Datacenter name. Ignored if the proxytype is ``esxdatacenter``. + ''' + proxy_type = __salt__['vsphere.get_proxy_type']() + if proxy_type == 'esxdatacenter': + dc_name = __salt__['esxdatacenter.get_details']()['datacenter'] + else: + dc_name = name + log.info('Running datacenter_configured for datacenter \'{0}\'' + ''.format(dc_name)) + ret = {'name': name, 'changes': {}, 'pchanges': {}, + 'result': None, 'comment': 'Default'} + comments = [] + changes = {} + pchanges = {} + si = None + try: + si = __salt__['vsphere.get_service_instance_via_proxy']() + dcs = __salt__['vsphere.list_datacenters_via_proxy']( + datacenter_names=[dc_name], service_instance=si) + if not dcs: + if __opts__['test']: + comments.append('State will create ' + 'datacenter \'{0}\'.'.format(dc_name)) + log.info(comments[-1]) + pchanges.update({'new': {'name': dc_name}}) + else: + log.debug('Creating datacenter \'{0}\'. '.format(dc_name)) + __salt__['vsphere.create_datacenter'](dc_name, si) + comments.append('Created datacenter \'{0}\'.'.format(dc_name)) + log.info(comments[-1]) + changes.update({'new': {'name': dc_name}}) + else: + comments.append('Datacenter \'{0}\' already exists. Nothing to be ' + 'done.'.format(dc_name)) + log.info(comments[-1]) + __salt__['vsphere.disconnect'](si) + if __opts__['test'] and pchanges: + ret_status = None + else: + ret_status = True + ret.update({'result': ret_status, + 'comment': '\n'.join(comments), + 'changes': changes, + 'pchanges': pchanges}) + return ret + except salt.exceptions.CommandExecutionError as exc: + log.error('Error: {}'.format(exc)) + if si: + __salt__['vsphere.disconnect'](si) + ret.update({ + 'result': False if not __opts__['test'] else None, + 'comment': str(exc)}) + return ret diff --git a/salt/states/esxi.py b/salt/states/esxi.py index dd0e8d70eae..12240422e4d 100644 --- a/salt/states/esxi.py +++ b/salt/states/esxi.py @@ -97,8 +97,8 @@ from __future__ import absolute_import import logging # Import Salt Libs -import salt.ext.six as six -import salt.utils +from salt.ext import six +import salt.utils.files from salt.exceptions import CommandExecutionError # Get Logging Started @@ -757,7 +757,7 @@ def ssh_configured(name, if not ssh_key: ssh_key = '' # Open ssh key file and read in contents to create one key string - with salt.utils.fopen(ssh_key_file, 'r') as key_file: + with salt.utils.files.fopen(ssh_key_file, 'r') as key_file: for line in key_file: if line.startswith('#'): # Commented line diff --git a/salt/states/etcd_mod.py b/salt/states/etcd_mod.py index 5d041813aa0..48f5c6f9931 100644 --- a/salt/states/etcd_mod.py +++ b/salt/states/etcd_mod.py @@ -41,6 +41,19 @@ or clusters are available. as this makes all master configuration settings available in all minion's pillars. +Etcd profile configuration can be overriden using following arguments: ``host``, +``port``, ``username``, ``password``, ``ca``, ``client_key`` and ``client_cert``. + +.. code-block:: yaml + my-value: + etcd.set: + - name: /path/to/key + - value: value + - host: 127.0.0.1 + - port: 2379 + - username: user + - password: pass + Available Functions ------------------- @@ -132,7 +145,7 @@ def __virtual__(): return __virtualname__ if HAS_ETCD else False -def set_(name, value, profile=None): +def set_(name, value, profile=None, **kwargs): ''' Set a key in etcd and can be called as ``set``. @@ -143,6 +156,13 @@ def set_(name, value, profile=None): profile Optional, defaults to ``None``. Sets the etcd profile to use which has been defined in the Salt Master config. + + .. code-block:: yaml + + my_etd_config: + etcd.host: 127.0.0.1 + etcd.port: 4001 + ''' created = False @@ -154,11 +174,11 @@ def set_(name, value, profile=None): 'changes': {} } - current = __salt__['etcd.get'](name, profile=profile) + current = __salt__['etcd.get'](name, profile=profile, **kwargs) if not current: created = True - result = __salt__['etcd.set'](name, value, profile=profile) + result = __salt__['etcd.set'](name, value, profile=profile, **kwargs) if result and result != current: if created: @@ -172,7 +192,7 @@ def set_(name, value, profile=None): return rtn -def wait_set(name, value, profile=None): +def wait_set(name, value, profile=None, **kwargs): ''' Set a key in etcd only if the watch statement calls it. This function is also aliased as ``wait_set``. @@ -184,6 +204,13 @@ def wait_set(name, value, profile=None): profile The etcd profile to use that has been configured on the Salt Master, this is optional and defaults to ``None``. + + .. code-block:: yaml + + my_etd_config: + etcd.host: 127.0.0.1 + etcd.port: 4001 + ''' return { @@ -194,7 +221,49 @@ def wait_set(name, value, profile=None): } -def rm_(name, recurse=False, profile=None): +def directory(name, profile=None, **kwargs): + ''' + Create a directory in etcd. + + name + The etcd directory name, for example: ``/foo/bar/baz``. + profile + Optional, defaults to ``None``. Sets the etcd profile to use which has + been defined in the Salt Master config. + + .. code-block:: yaml + + my_etd_config: + etcd.host: 127.0.0.1 + etcd.port: 4001 + ''' + + created = False + + rtn = { + 'name': name, + 'comment': 'Directory exists', + 'result': True, + 'changes': {} + } + + current = __salt__['etcd.get'](name, profile=profile, recurse=True, **kwargs) + if not current: + created = True + + result = __salt__['etcd.set'](name, None, directory=True, profile=profile, **kwargs) + + if result and result != current: + if created: + rtn['comment'] = 'New directory created' + rtn['changes'] = { + name: 'Created' + } + + return rtn + + +def rm_(name, recurse=False, profile=None, **kwargs): ''' Deletes a key from etcd. This function is also aliased as ``rm``. @@ -205,6 +274,12 @@ def rm_(name, recurse=False, profile=None): profile Optional, defaults to ``None``. Sets the etcd profile to use which has been defined in the Salt Master config. + + .. code-block:: yaml + + my_etd_config: + etcd.host: 127.0.0.1 + etcd.port: 4001 ''' rtn = { @@ -213,11 +288,11 @@ def rm_(name, recurse=False, profile=None): 'changes': {} } - if not __salt__['etcd.get'](name, profile=profile): + if not __salt__['etcd.get'](name, profile=profile, **kwargs): rtn['comment'] = 'Key does not exist' return rtn - if __salt__['etcd.rm'](name, recurse=recurse, profile=profile): + if __salt__['etcd.rm'](name, recurse=recurse, profile=profile, **kwargs): rtn['comment'] = 'Key removed' rtn['changes'] = { name: 'Deleted' @@ -228,7 +303,7 @@ def rm_(name, recurse=False, profile=None): return rtn -def wait_rm(name, recurse=False, profile=None): +def wait_rm(name, recurse=False, profile=None, **kwargs): ''' Deletes a key from etcd only if the watch statement calls it. This function is also aliased as ``wait_rm``. @@ -241,6 +316,12 @@ def wait_rm(name, recurse=False, profile=None): profile Optional, defaults to ``None``. Sets the etcd profile to use which has been defined in the Salt Master config. + + .. code-block:: yaml + + my_etd_config: + etcd.host: 127.0.0.1 + etcd.port: 4001 ''' return { diff --git a/salt/states/file.py b/salt/states/file.py index 84a6f12d5b3..05801ff5442 100644 --- a/salt/states/file.py +++ b/salt/states/file.py @@ -285,19 +285,21 @@ import salt.payload import salt.utils import salt.utils.dictupdate import salt.utils.files +import salt.utils.platform import salt.utils.templates import salt.utils.url +import salt.utils.versions from salt.utils.locales import sdecode from salt.exceptions import CommandExecutionError, SaltInvocationError -if salt.utils.is_windows(): +if salt.utils.platform.is_windows(): import salt.utils.win_dacl import salt.utils.win_functions # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import zip_longest -if salt.utils.is_windows(): +if salt.utils.platform.is_windows(): import pywintypes import win32com.client @@ -320,7 +322,7 @@ def _load_accumulators(): serial = salt.payload.Serial(__opts__) ret = {'accumulators': {}, 'accumulators_deps': {}} try: - with salt.utils.fopen(path, 'rb') as f: + with salt.utils.files.fopen(path, 'rb') as f: loaded = serial.load(f) return loaded if loaded else ret except (IOError, NameError): @@ -338,7 +340,7 @@ def _persist_accummulators(accumulators, accumulators_deps): serial = salt.payload.Serial(__opts__) try: - with salt.utils.fopen(_get_accumulator_filepath(), 'w+b') as f: + with salt.utils.files.fopen(_get_accumulator_filepath(), 'w+b') as f: serial.dump(accumm_data, f) except NameError: # msgpack error from salt-ssh @@ -625,7 +627,11 @@ def _clean_dir(root, keep, exclude_pat): while True: fn_ = os.path.dirname(fn_) real_keep.add(fn_) - if fn_ in ['/', ''.join([os.path.splitdrive(fn_)[0], '\\\\'])]: + if fn_ in [ + os.sep, + ''.join([os.path.splitdrive(fn_)[0], os.sep]), + ''.join([os.path.splitdrive(fn_)[0], os.sep, os.sep]) + ]: break def _delete_not_kept(nfn): @@ -1065,7 +1071,7 @@ def _get_template_texts(source_list=None, log.debug(msg.format(rndrd_templ_fn, source)) if rndrd_templ_fn: tmplines = None - with salt.utils.fopen(rndrd_templ_fn, 'rb') as fp_: + with salt.utils.files.fopen(rndrd_templ_fn, 'rb') as fp_: tmplines = fp_.read() if six.PY3: tmplines = tmplines.decode(__salt_system_encoding__) @@ -1262,7 +1268,7 @@ def symlink( if user is None: user = __opts__['user'] - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): # Make sure the user exists in Windows # Salt default is 'root' @@ -1405,7 +1411,8 @@ def symlink( return ret -def absent(name): +def absent(name, + **kwargs): ''' Make sure that the named file or directory is absent. If it exists, it will be deleted. This will work to reverse any of the functions in the file @@ -1436,7 +1443,10 @@ def absent(name): ret['comment'] = 'File {0} is set for removal'.format(name) return ret try: - __salt__['file.remove'](name) + if salt.utils.platform.is_windows(): + __salt__['file.remove'](name, force=True) + else: + __salt__['file.remove'](name) ret['comment'] = 'Removed file {0}'.format(name) ret['changes']['removed'] = name return ret @@ -1450,7 +1460,10 @@ def absent(name): ret['comment'] = 'Directory {0} is set for removal'.format(name) return ret try: - __salt__['file.remove'](name) + if salt.utils.platform.is_windows(): + __salt__['file.remove'](name, force=True) + else: + __salt__['file.remove'](name) ret['comment'] = 'Removed directory {0}'.format(name) ret['changes']['removed'] = name return ret @@ -1461,7 +1474,8 @@ def absent(name): return ret -def exists(name): +def exists(name, + **kwargs): ''' Verify that the named file or directory is present or exists. Ensures pre-requisites outside of Salt's purview @@ -1479,15 +1493,16 @@ def exists(name): 'result': True, 'comment': ''} if not name: - return _error(ret, 'Must provide name to file.exists') + return _error(ret, u'Must provide name to file.exists') if not os.path.exists(name): - return _error(ret, 'Specified path {0} does not exist'.format(name)) + return _error(ret, u'Specified path {0} does not exist'.format(name)) - ret['comment'] = 'Path {0} exists'.format(name) + ret['comment'] = u'Path {0} exists'.format(name) return ret -def missing(name): +def missing(name, + **kwargs): ''' Verify that the named file or directory is missing, this returns True only if the named file is missing but does not remove the file if it is present. @@ -1503,11 +1518,11 @@ def missing(name): 'result': True, 'comment': ''} if not name: - return _error(ret, 'Must provide name to file.missing') + return _error(ret, u'Must provide name to file.missing') if os.path.exists(name): - return _error(ret, 'Specified path {0} exists'.format(name)) + return _error(ret, u'Specified path {0} exists'.format(name)) - ret['comment'] = 'Path {0} is missing'.format(name) + ret['comment'] = u'Path {0} is missing'.format(name) return ret @@ -1550,7 +1565,7 @@ def managed(name, the salt master and potentially run through a templating system. name - The location of the file to manage + The location of the file to manage, as an absolute path. source The source file to download to the minion, this source file can be @@ -1720,13 +1735,15 @@ def managed(name, group The group ownership set for the file, this defaults to the group salt - is running as on the minion On Windows, this is ignored + is running as on the minion. On Windows, this is ignored mode - The permissions to set on this file, e.g. ``644``, ``0775``, or ``4664``. + The permissions to set on this file, e.g. ``644``, ``0775``, or + ``4664``. - The default mode for new files and directories corresponds umask of salt - process. For existing files and directories it's not enforced. + The default mode for new files and directories corresponds to the + umask of the salt process. The mode of existing files and directories + will only be changed if ``mode`` is specified. .. note:: This option is **not** supported on Windows. @@ -2067,12 +2084,7 @@ def managed(name, - win_inheritance: False ''' if 'env' in kwargs: - salt.utils.warn_until( - 'Oxygen', - 'Parameter \'env\' has been detected in the argument list. This ' - 'parameter is no longer used and has been replaced by \'saltenv\' ' - 'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.' - ) + # "env" is not supported; Use "saltenv". kwargs.pop('env') name = os.path.expanduser(name) @@ -2083,10 +2095,10 @@ def managed(name, 'name': name, 'result': True} - if mode is not None and salt.utils.is_windows(): + if mode is not None and salt.utils.platform.is_windows(): return _error(ret, 'The \'mode\' option is not supported on Windows') - if attrs is not None and salt.utils.is_windows(): + if attrs is not None and salt.utils.platform.is_windows(): return _error(ret, 'The \'attrs\' option is not supported on Windows') try: @@ -2239,7 +2251,7 @@ def managed(name, if not name: return _error(ret, 'Must provide name to file.managed') user = _test_owner(kwargs, user=user) - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): # If win_owner not passed, use user if win_owner is None: @@ -2257,8 +2269,8 @@ def managed(name, if not create: if not os.path.isfile(name): # Don't create a file that is not already present - ret['comment'] = ('File {0} is not present and is not set for ' - 'creation').format(name) + ret['comment'] = (u'File {0} is not present and is not set for ' + u'creation').format(name) return ret u_check = _check_user(user, group) if u_check: @@ -2266,10 +2278,10 @@ def managed(name, return _error(ret, u_check) if not os.path.isabs(name): return _error( - ret, 'Specified file {0} is not an absolute path'.format(name)) + ret, u'Specified file {0} is not an absolute path'.format(name)) if os.path.isdir(name): - ret['comment'] = 'Specified target {0} is a directory'.format(name) + ret['comment'] = u'Specified target {0} is a directory'.format(name) ret['result'] = False return ret @@ -2284,7 +2296,7 @@ def managed(name, if not replace and os.path.exists(name): # Check and set the permissions if necessary - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): ret = __salt__['file.check_perms']( name, ret, win_owner, win_perms, win_deny_perms, None, win_inheritance) @@ -2292,10 +2304,10 @@ def managed(name, ret, _ = __salt__['file.check_perms']( name, ret, user, group, mode, attrs, follow_symlinks) if __opts__['test']: - ret['comment'] = 'File {0} not updated'.format(name) + ret['comment'] = u'File {0} not updated'.format(name) elif not ret['changes'] and ret['result']: - ret['comment'] = ('File {0} exists with proper permissions. ' - 'No changes made.'.format(name)) + ret['comment'] = (u'File {0} exists with proper permissions. ' + u'No changes made.'.format(name)) return ret accum_data, _ = _load_accumulators() @@ -2326,7 +2338,7 @@ def managed(name, **kwargs ) - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): ret = __salt__['file.check_perms']( name, ret, win_owner, win_perms, win_deny_perms, None, win_inheritance) @@ -2335,14 +2347,14 @@ def managed(name, ret['result'], ret['comment'] = ret['pchanges'] elif ret['pchanges']: ret['result'] = None - ret['comment'] = 'The file {0} is set to be changed'.format(name) + ret['comment'] = u'The file {0} is set to be changed'.format(name) if show_changes and 'diff' in ret['pchanges']: ret['changes']['diff'] = ret['pchanges']['diff'] if not show_changes: ret['changes']['diff'] = '' else: ret['result'] = True - ret['comment'] = 'The file {0} is in the correct state'.format(name) + ret['comment'] = u'The file {0} is in the correct state'.format(name) return ret @@ -2452,12 +2464,16 @@ def managed(name, if sfn and os.path.isfile(sfn): os.remove(sfn) return ret + + if sfn and os.path.isfile(sfn): + os.remove(sfn) + # Since we generated a new tempfile and we are not returning here # lets change the original sfn to the new tempfile or else we will # get file not found - if sfn and os.path.isfile(sfn): - os.remove(sfn) - sfn = tmp_filename + + sfn = tmp_filename + else: ret = {'changes': {}, 'comment': '', @@ -2570,7 +2586,7 @@ def directory(name, Ensure that a named directory is present and has the right perms name - The location to create or manage a directory + The location to create or manage a directory, as an absolute path user The user to own the directory; this defaults to the user salt is @@ -2770,7 +2786,7 @@ def directory(name, return _error(ret, 'Cannot specify both max_depth and clean') user = _test_owner(kwargs, user=user) - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): # If win_owner not passed, use user if win_owner is None: @@ -2795,7 +2811,7 @@ def directory(name, dir_mode = salt.utils.normalize_mode(dir_mode) file_mode = salt.utils.normalize_mode(file_mode) - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): # Verify win_owner is valid on the target system try: salt.utils.win_dacl.get_sid(win_owner) @@ -2848,7 +2864,7 @@ def directory(name, 'Specified location {0} exists and is a symlink'.format(name)) # Check directory? - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): presult, pcomment, ret['pchanges'] = _check_directory_win( name, win_owner, win_perms, win_deny_perms, win_inheritance) else: @@ -2867,7 +2883,7 @@ def directory(name, # The parent directory does not exist, create them if makedirs: # Everything's good, create the parent Dirs - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): # Make sure the drive is mapped before trying to create the # path in windows drive, path = os.path.splitdrive(name) @@ -2883,7 +2899,7 @@ def directory(name, return _error( ret, 'No directory to create {0} in'.format(name)) - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): __salt__['file.mkdir'](name, win_owner, win_perms, win_deny_perms, win_inheritance) else: @@ -2897,7 +2913,7 @@ def directory(name, # issue 32707: skip this __salt__['file.check_perms'] call if children_only == True # Check permissions if not children_only: - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): ret = __salt__['file.check_perms']( name, ret, win_owner, win_perms, win_deny_perms, None, win_inheritance) else: @@ -2968,7 +2984,7 @@ def directory(name, for fn_ in files: full = os.path.join(root, fn_) try: - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): ret = __salt__['file.check_perms']( full, ret, win_owner, win_perms, win_deny_perms, None, win_inheritance) @@ -2983,7 +2999,7 @@ def directory(name, for dir_ in dirs: full = os.path.join(root, dir_) try: - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): ret = __salt__['file.check_perms']( full, ret, win_owner, win_perms, win_deny_perms, None, win_inheritance) @@ -3001,14 +3017,14 @@ def directory(name, removed = _clean_dir(name, list(keep), exclude_pat) if removed: ret['changes']['removed'] = removed - ret['comment'] = 'Files cleaned from directory {0}'.format(name) + ret['comment'] = u'Files cleaned from directory {0}'.format(name) # issue 32707: reflect children_only selection in comments if not ret['comment']: if children_only: - ret['comment'] = 'Directory {0}/* updated'.format(name) + ret['comment'] = u'Directory {0}/* updated'.format(name) else: - ret['comment'] = 'Directory {0} updated'.format(name) + ret['comment'] = u'Directory {0} updated'.format(name) if __opts__['test']: ret['comment'] = 'Directory {0} not updated'.format(name) @@ -3017,9 +3033,9 @@ def directory(name, if ret['comment']: orig_comment = ret['comment'] - ret['comment'] = 'Directory {0} is in the correct state'.format(name) + ret['comment'] = u'Directory {0} is in the correct state'.format(name) if orig_comment: - ret['comment'] = '\n'.join([ret['comment'], orig_comment]) + ret['comment'] = u'\n'.join([ret['comment'], orig_comment]) if errors: ret['result'] = False @@ -3041,6 +3057,7 @@ def recurse(name, sym_mode=None, template=None, context=None, + replace=True, defaults=None, include_empty=False, backup='', @@ -3129,6 +3146,11 @@ def recurse(name, The template option is required when recursively applying templates. + replace : True + If set to ``False`` and the file already exists, the file will not be + modified even if changes would otherwise be made. Permissions and + ownership will still be enforced, however. + context Overrides default context variables passed to the template. @@ -3196,18 +3218,13 @@ def recurse(name, option is usually not needed except in special circumstances. ''' if 'env' in kwargs: - salt.utils.warn_until( - 'Oxygen', - 'Parameter \'env\' has been detected in the argument list. This ' - 'parameter is no longer used and has been replaced by \'saltenv\' ' - 'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.' - ) + # "env" is not supported; Use "saltenv". kwargs.pop('env') name = os.path.expanduser(sdecode(name)) user = _test_owner(kwargs, user=user) - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): if group is not None: log.warning( 'The group argument for {0} has been ignored as this ' @@ -3231,7 +3248,7 @@ def recurse(name, return ret if any([x is not None for x in (dir_mode, file_mode, sym_mode)]) \ - and salt.utils.is_windows(): + and salt.utils.platform.is_windows(): return _error(ret, 'mode management is not supported on Windows') # Make sure that leading zeros stripped by YAML loader are added back @@ -3285,8 +3302,8 @@ def recurse(name, if x.startswith(srcpath + '/'))): ret['result'] = False ret['comment'] = ( - 'The directory \'{0}\' does not exist on the salt fileserver ' - 'in saltenv \'{1}\''.format(srcpath, senv) + u'The directory \'{0}\' does not exist on the salt fileserver ' + u'in saltenv \'{1}\''.format(srcpath, senv) ) return ret @@ -3318,12 +3335,12 @@ def recurse(name, if _ret['changes']: ret['changes'][path] = _ret['changes'] - def manage_file(path, source): - if clean and os.path.exists(path) and os.path.isdir(path): + def manage_file(path, source, replace): + if clean and os.path.exists(path) and os.path.isdir(path) and replace: _ret = {'name': name, 'changes': {}, 'result': True, 'comment': ''} if __opts__['test']: - _ret['comment'] = 'Replacing directory {0} with a ' \ - 'file'.format(path) + _ret['comment'] = u'Replacing directory {0} with a ' \ + u'file'.format(path) _ret['result'] = None merge_ret(path, _ret) return @@ -3349,6 +3366,7 @@ def recurse(name, attrs=None, template=template, makedirs=True, + replace=replace, context=context, defaults=defaults, backup=backup, @@ -3361,7 +3379,7 @@ def recurse(name, if clean and os.path.exists(path) and not os.path.isdir(path): _ret = {'name': name, 'changes': {}, 'result': True, 'comment': ''} if __opts__['test']: - _ret['comment'] = 'Replacing {0} with a directory'.format(path) + _ret['comment'] = u'Replacing {0} with a directory'.format(path) _ret['result'] = None merge_ret(path, _ret) return @@ -3405,7 +3423,7 @@ def recurse(name, for dirname in mng_dirs: manage_directory(dirname) for dest, src in mng_files: - manage_file(dest, src) + manage_file(dest, src, replace) if clean: # TODO: Use directory(clean=True) instead @@ -3426,10 +3444,10 @@ def recurse(name, ) for (k, v) in six.iteritems(ret['comment'])).strip() if not ret['comment']: - ret['comment'] = 'Recursively updated {0}'.format(name) + ret['comment'] = u'Recursively updated {0}'.format(name) if not ret['changes'] and ret['result']: - ret['comment'] = 'The directory {0} is in the correct state'.format( + ret['comment'] = u'The directory {0} is in the correct state'.format( name ) @@ -3613,13 +3631,13 @@ def retention_schedule(name, retain, strptime_format=None, timezone=None): # TODO: track and report how much space was / would be reclaimed if __opts__['test']: - ret['comment'] = '{0} backups would have been removed from {1}.\n'.format(len(deletable_files), name) + ret['comment'] = u'{0} backups would have been removed from {1}.\n'.format(len(deletable_files), name) if deletable_files: ret['result'] = None else: for f in deletable_files: __salt__['file.remove'](os.path.join(name, f)) - ret['comment'] = '{0} backups were removed from {1}.\n'.format(len(deletable_files), name) + ret['comment'] = u'{0} backups were removed from {1}.\n'.format(len(deletable_files), name) ret['changes'] = changes return ret @@ -3756,7 +3774,13 @@ def line(name, content=None, match=None, mode=None, location=None, if not name: return _error(ret, 'Must provide name to file.line') - managed(name, create=create, user=user, group=group, mode=file_mode) + managed( + name, + create=create, + user=user, + group=group, + mode=file_mode, + replace=False) check_res, check_msg = _check_file(name) if not check_res: @@ -3804,7 +3828,8 @@ def replace(name, not_found_content=None, backup='.bak', show_changes=True, - ignore_if_missing=False): + ignore_if_missing=False, + backslash_literal=False): r''' Maintain an edit in a file. @@ -3818,12 +3843,13 @@ def replace(name, A regular expression, to be matched using Python's :py:func:`~re.search`. - ..note:: + .. note:: + If you need to match a literal string that contains regex special characters, you may want to use salt's custom Jinja filter, ``escape_regex``. - ..code-block:: jinja + .. code-block:: jinja {{ 'http://example.com?foo=bar%20baz' | escape_regex }} @@ -3903,6 +3929,14 @@ def replace(name, state will display an error raised by the execution module. If set to ``True``, the state will simply report no changes. + backslash_literal : False + .. versionadded:: 2016.11.7 + + Interpret backslashes as literal backslashes for the repl and not + escape characters. This will help when using append/prepend so that + the backslashes are not interpreted for the repl on the second run of + the state. + For complex regex patterns, it can be useful to avoid the need for complex quoting and escape sequences by making use of YAML's multiline string syntax. @@ -3951,7 +3985,8 @@ def replace(name, backup=backup, dry_run=__opts__['test'], show_changes=show_changes, - ignore_if_missing=ignore_if_missing) + ignore_if_missing=ignore_if_missing, + backslash_literal=backslash_literal) if changes: ret['pchanges']['diff'] = changes @@ -4311,10 +4346,10 @@ def comment(name, regex, char='#', backup='.bak'): ret['pchanges'][name] = 'updated' if __opts__['test']: - ret['comment'] = 'File {0} is set to be updated'.format(name) + ret['comment'] = u'File {0} is set to be updated'.format(name) ret['result'] = None return ret - with salt.utils.fopen(name, 'rb') as fp_: + with salt.utils.files.fopen(name, 'rb') as fp_: slines = fp_.read() if six.PY3: slines = slines.decode(__salt_system_encoding__) @@ -4323,7 +4358,7 @@ def comment(name, regex, char='#', backup='.bak'): # Perform the edit __salt__['file.comment_line'](name, regex, char, True, backup) - with salt.utils.fopen(name, 'rb') as fp_: + with salt.utils.files.fopen(name, 'rb') as fp_: nlines = fp_.read() if six.PY3: nlines = nlines.decode(__salt_system_encoding__) @@ -4333,7 +4368,7 @@ def comment(name, regex, char='#', backup='.bak'): ret['result'] = __salt__['file.search'](name, unanchor_regex, multiline=True) if slines != nlines: - if not salt.utils.istextfile(name): + if not __utils__['files.is_text_file'](name): ret['changes']['diff'] = 'Replace binary file' else: # Changes happened, add them @@ -4418,11 +4453,11 @@ def uncomment(name, regex, char='#', backup='.bak'): ret['pchanges'][name] = 'updated' if __opts__['test']: - ret['comment'] = 'File {0} is set to be updated'.format(name) + ret['comment'] = u'File {0} is set to be updated'.format(name) ret['result'] = None return ret - with salt.utils.fopen(name, 'rb') as fp_: + with salt.utils.files.fopen(name, 'rb') as fp_: slines = fp_.read() if six.PY3: slines = slines.decode(__salt_system_encoding__) @@ -4431,7 +4466,7 @@ def uncomment(name, regex, char='#', backup='.bak'): # Perform the edit __salt__['file.comment_line'](name, regex, char, False, backup) - with salt.utils.fopen(name, 'rb') as fp_: + with salt.utils.files.fopen(name, 'rb') as fp_: nlines = fp_.read() if six.PY3: nlines = nlines.decode(__salt_system_encoding__) @@ -4445,7 +4480,7 @@ def uncomment(name, regex, char='#', backup='.bak'): ) if slines != nlines: - if not salt.utils.istextfile(name): + if not __utils__['files.is_text_file'](name): ret['changes']['diff'] = 'Replace binary file' else: # Changes happened, add them @@ -4655,7 +4690,7 @@ def append(name, text = _validate_str_list(text) - with salt.utils.fopen(name, 'rb') as fp_: + with salt.utils.files.fopen(name, 'rb') as fp_: slines = fp_.read() if six.PY3: slines = slines.decode(__salt_system_encoding__) @@ -4683,12 +4718,12 @@ def append(name, return _error(ret, 'No text found to append. Nothing appended') if __opts__['test']: - ret['comment'] = 'File {0} is set to be updated'.format(name) + ret['comment'] = u'File {0} is set to be updated'.format(name) ret['result'] = None nlines = list(slines) nlines.extend(append_lines) if slines != nlines: - if not salt.utils.istextfile(name): + if not __utils__['files.is_text_file'](name): ret['changes']['diff'] = 'Replace binary file' else: # Changes happened, add them @@ -4696,7 +4731,7 @@ def append(name, '\n'.join(difflib.unified_diff(slines, nlines)) ) else: - ret['comment'] = 'File {0} is in correct state'.format(name) + ret['comment'] = u'File {0} is in correct state'.format(name) ret['result'] = True return ret @@ -4704,16 +4739,16 @@ def append(name, __salt__['file.append'](name, args=append_lines) ret['comment'] = 'Appended {0} lines'.format(len(append_lines)) else: - ret['comment'] = 'File {0} is in correct state'.format(name) + ret['comment'] = u'File {0} is in correct state'.format(name) - with salt.utils.fopen(name, 'rb') as fp_: + with salt.utils.files.fopen(name, 'rb') as fp_: nlines = fp_.read() if six.PY3: nlines = nlines.decode(__salt_system_encoding__) nlines = nlines.splitlines() if slines != nlines: - if not salt.utils.istextfile(name): + if not __utils__['files.is_text_file'](name): ret['changes']['diff'] = 'Replace binary file' else: # Changes happened, add them @@ -4847,7 +4882,7 @@ def prepend(name, text = _validate_str_list(text) - with salt.utils.fopen(name, 'rb') as fp_: + with salt.utils.files.fopen(name, 'rb') as fp_: slines = fp_.read() if six.PY3: slines = slines.decode(__salt_system_encoding__) @@ -4871,7 +4906,7 @@ def prepend(name, for line in lines: if __opts__['test']: - ret['comment'] = 'File {0} is set to be updated'.format(name) + ret['comment'] = u'File {0} is set to be updated'.format(name) ret['result'] = None test_lines.append('{0}\n'.format(line)) else: @@ -4881,7 +4916,7 @@ def prepend(name, if __opts__['test']: nlines = test_lines + slines if slines != nlines: - if not salt.utils.istextfile(name): + if not __utils__['files.is_text_file'](name): ret['changes']['diff'] = 'Replace binary file' else: # Changes happened, add them @@ -4890,13 +4925,13 @@ def prepend(name, ) ret['result'] = None else: - ret['comment'] = 'File {0} is in correct state'.format(name) + ret['comment'] = u'File {0} is in correct state'.format(name) ret['result'] = True return ret # if header kwarg is True, use verbatim compare if header: - with salt.utils.fopen(name, 'rb') as fp_: + with salt.utils.files.fopen(name, 'rb') as fp_: # read as many lines of target file as length of user input contents = fp_.read() if six.PY3: @@ -4917,14 +4952,14 @@ def prepend(name, else: __salt__['file.prepend'](name, *preface) - with salt.utils.fopen(name, 'rb') as fp_: + with salt.utils.files.fopen(name, 'rb') as fp_: nlines = fp_.read() if six.PY3: nlines = nlines.decode(__salt_system_encoding__) nlines = nlines.splitlines(True) if slines != nlines: - if not salt.utils.istextfile(name): + if not __utils__['files.is_text_file'](name): ret['changes']['diff'] = 'Replace binary file' else: # Changes happened, add them @@ -4935,7 +4970,7 @@ def prepend(name, if count: ret['comment'] = 'Prepended {0} lines'.format(count) else: - ret['comment'] = 'File {0} is in correct state'.format(name) + ret['comment'] = u'File {0} is in correct state'.format(name) ret['result'] = True return ret @@ -5000,12 +5035,7 @@ def patch(name, hash_ = kwargs.pop('hash', None) if 'env' in kwargs: - salt.utils.warn_until( - 'Oxygen', - 'Parameter \'env\' has been detected in the argument list. This ' - 'parameter is no longer used and has been replaced by \'saltenv\' ' - 'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.' - ) + # "env" is not supported; Use "saltenv". kwargs.pop('env') name = os.path.expanduser(name) @@ -5033,7 +5063,7 @@ def patch(name, # get cached file or copy it to cache cached_source_path = __salt__['cp.cache_file'](source, __env__) if not cached_source_path: - ret['comment'] = ('Unable to cache {0} from saltenv \'{1}\'' + ret['comment'] = (u'Unable to cache {0} from saltenv \'{1}\'' .format(source, __env__)) return ret @@ -5047,7 +5077,7 @@ def patch(name, name, cached_source_path, options=options, dry_run=True ) if __opts__['test']: - ret['comment'] = 'File {0} will be patched'.format(name) + ret['comment'] = u'File {0} will be patched'.format(name) ret['result'] = None return ret if ret['changes']['retcode'] != 0: @@ -5126,10 +5156,10 @@ def touch(name, atime=None, mtime=None, makedirs=False): ret['result'] = __salt__['file.touch'](name, atime, mtime) if not extant and ret['result']: - ret['comment'] = 'Created empty file {0}'.format(name) + ret['comment'] = u'Created empty file {0}'.format(name) ret['changes']['new'] = name elif extant and ret['result']: - ret['comment'] = 'Updated times on {0} {1}'.format( + ret['comment'] = u'Updated times on {0} {1}'.format( 'directory' if os.path.isdir(name) else 'file', name ) ret['changes']['touched'] = name @@ -5217,7 +5247,7 @@ def copy( ret = { 'name': name, 'changes': {}, - 'comment': 'Copied "{0}" to "{1}"'.format(source, name), + 'comment': u'Copied "{0}" to "{1}"'.format(source, name), 'result': True} if not name: return _error(ret, 'Must provide name to file.copy') @@ -5239,7 +5269,7 @@ def copy( if user is None: user = __opts__['user'] - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): if group is not None: log.warning( 'The group argument for {0} has been ignored as this is ' @@ -5287,20 +5317,20 @@ def copy( if __opts__['test']: if changed: - ret['comment'] = 'File "{0}" is set to be copied to "{1}"'.format( + ret['comment'] = u'File "{0}" is set to be copied to "{1}"'.format( source, name ) ret['result'] = None else: - ret['comment'] = ('The target file "{0}" exists and will not be ' - 'overwritten'.format(name)) + ret['comment'] = (u'The target file "{0}" exists and will not be ' + u'overwritten'.format(name)) ret['result'] = True return ret if not changed: - ret['comment'] = ('The target file "{0}" exists and will not be ' - 'overwritten'.format(name)) + ret['comment'] = (u'The target file "{0}" exists and will not be ' + u'overwritten'.format(name)) ret['result'] = True return ret @@ -5369,17 +5399,17 @@ def rename(name, source, force=False, makedirs=False): if not os.path.isabs(name): return _error( - ret, 'Specified file {0} is not an absolute path'.format(name)) + ret, u'Specified file {0} is not an absolute path'.format(name)) if not os.path.lexists(source): - ret['comment'] = ('Source file "{0}" has already been moved out of ' - 'place').format(source) + ret['comment'] = (u'Source file "{0}" has already been moved out of ' + u'place').format(source) return ret if os.path.lexists(source) and os.path.lexists(name): if not force: - ret['comment'] = ('The target file "{0}" exists and will not be ' - 'overwritten'.format(name)) + ret['comment'] = (u'The target file "{0}" exists and will not be ' + u'overwritten'.format(name)) ret['result'] = False return ret elif not __opts__['test']: @@ -5389,12 +5419,12 @@ def rename(name, source, force=False, makedirs=False): except (IOError, OSError): return _error( ret, - 'Failed to delete "{0}" in preparation for ' - 'forced move'.format(name) + u'Failed to delete "{0}" in preparation for ' + u'forced move'.format(name) ) if __opts__['test']: - ret['comment'] = 'File "{0}" is set to be moved to "{1}"'.format( + ret['comment'] = u'File "{0}" is set to be moved to "{1}"'.format( source, name ) @@ -5409,7 +5439,7 @@ def rename(name, source, force=False, makedirs=False): else: return _error( ret, - 'The target directory {0} is not present'.format(dname)) + u'The target directory {0} is not present'.format(dname)) # All tests pass, move the file into place try: if os.path.islink(source): @@ -5420,9 +5450,9 @@ def rename(name, source, force=False, makedirs=False): shutil.move(source, name) except (IOError, OSError): return _error( - ret, 'Failed to move "{0}" to "{1}"'.format(source, name)) + ret, u'Failed to move "{0}" to "{1}"'.format(source, name)) - ret['comment'] = 'Moved "{0}" to "{1}"'.format(source, name) + ret['comment'] = u'Moved "{0}" to "{1}"'.format(source, name) ret['changes'] = {name: source} return ret @@ -5501,7 +5531,7 @@ def accumulated(name, filename, text, **kwargs): deps = require_in + watch_in if not [x for x in deps if 'file' in x]: ret['result'] = False - ret['comment'] = 'Orphaned accumulator {0} in {1}:{2}'.format( + ret['comment'] = u'Orphaned accumulator {0} in {1}:{2}'.format( name, __low__['__sls__'], __low__['__id__'] @@ -5525,8 +5555,8 @@ def accumulated(name, filename, text, **kwargs): for chunk in text: if chunk not in accum_data[filename][name]: accum_data[filename][name].append(chunk) - ret['comment'] = ('Accumulator {0} for file {1} ' - 'was charged by text'.format(name, filename)) + ret['comment'] = (u'Accumulator {0} for file {1} ' + u'was charged by text'.format(name, filename)) _persist_accummulators(accum_data, accum_deps) return ret @@ -5658,12 +5688,7 @@ def serialize(name, } ''' if 'env' in kwargs: - salt.utils.warn_until( - 'Oxygen', - 'Parameter \'env\' has been detected in the argument list. This ' - 'parameter is no longer used and has been replaced by \'saltenv\' ' - 'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.' - ) + # "env" is not supported; Use "saltenv". kwargs.pop('env') name = os.path.expanduser(name) @@ -5687,8 +5712,8 @@ def serialize(name, if not create: if not os.path.isfile(name): # Don't create a file that is not already present - ret['comment'] = ('File {0} is not present and is not set for ' - 'creation').format(name) + ret['comment'] = (u'File {0} is not present and is not set for ' + u'creation').format(name) return ret formatter = kwargs.pop('formatter', 'yaml').lower() @@ -5704,11 +5729,11 @@ def serialize(name, return _error( ret, 'Neither \'dataset\' nor \'dataset_pillar\' was defined') - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): if group is not None: log.warning( - 'The group argument for {0} has been ignored as this ' - 'is a Windows system.'.format(name) + u'The group argument for %s has been ignored as this ' + u'is a Windows system.', name ) group = user @@ -5717,7 +5742,7 @@ def serialize(name, if serializer_name not in __serializers__: return {'changes': {}, - 'comment': '{0} format is not supported'.format( + 'comment': u'{0} format is not supported'.format( formatter.capitalize()), 'name': name, 'result': False @@ -5727,19 +5752,19 @@ def serialize(name, if os.path.isfile(name): if '{0}.deserialize'.format(formatter) not in __serializers__: return {'changes': {}, - 'comment': ('{0} format is not supported for merging' + 'comment': (u'{0} format is not supported for merging' .format(formatter.capitalize())), 'name': name, 'result': False} - with salt.utils.fopen(name, 'r') as fhr: + with salt.utils.files.fopen(name, 'r') as fhr: existing_data = __serializers__[deserializer_name](fhr) if existing_data is not None: merged_data = salt.utils.dictupdate.merge_recurse(existing_data, dataset) if existing_data == merged_data: ret['result'] = True - ret['comment'] = 'The file {0} is in the correct state'.format(name) + ret['comment'] = u'The file {0} is in the correct state'.format(name) return ret dataset = merged_data contents = __serializers__[serializer_name](dataset, **default_serializer_opts.get(serializer_name, {})) @@ -5770,14 +5795,14 @@ def serialize(name, if ret['changes']: ret['result'] = None - ret['comment'] = 'Dataset will be serialized and stored into {0}'.format( + ret['comment'] = u'Dataset will be serialized and stored into {0}'.format( name) if not show_changes: ret['changes']['diff'] = '' else: ret['result'] = True - ret['comment'] = 'The file {0} is in the correct state'.format(name) + ret['comment'] = u'The file {0} is in the correct state'.format(name) return ret return __salt__['file.manage_file'](name=name, @@ -5882,16 +5907,15 @@ def mknod(name, ntype, major=0, minor=0, user=None, group=None, mode='0600'): # Check for file existence if __salt__['file.file_exists'](name): ret['comment'] = ( - 'File exists and is not a character device {0}. Cowardly ' - 'refusing to continue'.format(name) + u'File {0} exists and is not a character device. Refusing ' + u'to continue'.format(name) ) # Check if it is a character device elif not __salt__['file.is_chrdev'](name): if __opts__['test']: - ret['comment'] = ( - 'Character device {0} is set to be created' - ).format(name) + ret['comment'] = \ + u'Character device {0} is set to be created'.format(name) ret['result'] = None else: ret = __salt__['file.mknod'](name, @@ -5907,8 +5931,8 @@ def mknod(name, ntype, major=0, minor=0, user=None, group=None, mode='0600'): devmaj, devmin = __salt__['file.get_devmm'](name) if (major, minor) != (devmaj, devmin): ret['comment'] = ( - 'Character device {0} exists and has a different ' - 'major/minor {1}/{2}. Cowardly refusing to continue' + u'Character device {0} exists and has a different ' + u'major/minor {1}/{2}. Refusing to continue' .format(name, devmaj, devmin) ) # Check the perms @@ -5920,7 +5944,7 @@ def mknod(name, ntype, major=0, minor=0, user=None, group=None, mode='0600'): mode)[0] if not ret['changes']: ret['comment'] = ( - 'Character device {0} is in the correct state'.format( + u'Character device {0} is in the correct state'.format( name ) ) @@ -5929,16 +5953,14 @@ def mknod(name, ntype, major=0, minor=0, user=None, group=None, mode='0600'): # Check for file existence if __salt__['file.file_exists'](name): ret['comment'] = ( - 'File exists and is not a block device {0}. Cowardly ' - 'refusing to continue'.format(name) + u'File {0} exists and is not a block device. Refusing to ' + u'continue'.format(name) ) # Check if it is a block device elif not __salt__['file.is_blkdev'](name): if __opts__['test']: - ret['comment'] = ( - 'Block device {0} is set to be created' - ).format(name) + ret['comment'] = u'Block device {0} is set to be created'.format(name) ret['result'] = None else: ret = __salt__['file.mknod'](name, @@ -5954,8 +5976,8 @@ def mknod(name, ntype, major=0, minor=0, user=None, group=None, mode='0600'): devmaj, devmin = __salt__['file.get_devmm'](name) if (major, minor) != (devmaj, devmin): ret['comment'] = ( - 'Block device {0} exists and has a different major/minor ' - '{1}/{2}. Cowardly refusing to continue'.format( + u'Block device {0} exists and has a different major/minor ' + u'{1}/{2}. Refusing to continue'.format( name, devmaj, devmin ) ) @@ -5968,21 +5990,21 @@ def mknod(name, ntype, major=0, minor=0, user=None, group=None, mode='0600'): mode)[0] if not ret['changes']: ret['comment'] = ( - 'Block device {0} is in the correct state'.format(name) + u'Block device {0} is in the correct state'.format(name) ) elif ntype == 'p': # Check for file existence if __salt__['file.file_exists'](name): ret['comment'] = ( - 'File exists and is not a fifo pipe {0}. Cowardly refusing ' - 'to continue'.format(name) + u'File {0} exists and is not a fifo pipe. Refusing to ' + u'continue'.format(name) ) # Check if it is a fifo elif not __salt__['file.is_fifo'](name): if __opts__['test']: - ret['comment'] = 'Fifo pipe {0} is set to be created'.format( + ret['comment'] = u'Fifo pipe {0} is set to be created'.format( name ) ret['result'] = None @@ -6004,7 +6026,7 @@ def mknod(name, ntype, major=0, minor=0, user=None, group=None, mode='0600'): mode)[0] if not ret['changes']: ret['comment'] = ( - 'Fifo pipe {0} is in the correct state'.format(name) + u'Fifo pipe {0} is in the correct state'.format(name) ) else: @@ -6227,7 +6249,7 @@ def shortcut( 'changes': {}, 'result': True, 'comment': ''} - if not salt.utils.is_windows(): + if not salt.utils.platform.is_windows(): return _error(ret, 'Shortcuts are only supported on Windows') if not name: return _error(ret, 'Must provide name to file.shortcut') @@ -6360,18 +6382,18 @@ def shortcut( else: if _check_shortcut_ownership(name, user): # The shortcut looks good! - ret['comment'] = ('Shortcut {0} is present and owned by ' - '{1}'.format(name, user)) + ret['comment'] = (u'Shortcut {0} is present and owned by ' + u'{1}'.format(name, user)) else: if _set_shortcut_ownership(name, user): - ret['comment'] = ('Set ownership of shortcut {0} to ' - '{1}'.format(name, user)) + ret['comment'] = (u'Set ownership of shortcut {0} to ' + u'{1}'.format(name, user)) ret['changes']['ownership'] = '{0}'.format(user) else: ret['result'] = False ret['comment'] += ( - 'Failed to set ownership of shortcut {0} to ' - '{1}'.format(name, user) + u'Failed to set ownership of shortcut {0} to ' + u'{1}'.format(name, user) ) return ret @@ -6390,12 +6412,12 @@ def shortcut( scut.Save() except (AttributeError, pywintypes.com_error) as exc: ret['result'] = False - ret['comment'] = ('Unable to create new shortcut {0} -> ' - '{1}: {2}'.format(name, target, exc)) + ret['comment'] = (u'Unable to create new shortcut {0} -> ' + u'{1}: {2}'.format(name, target, exc)) return ret else: - ret['comment'] = ('Created new shortcut {0} -> ' - '{1}'.format(name, target)) + ret['comment'] = (u'Created new shortcut {0} -> ' + u'{1}'.format(name, target)) ret['changes']['new'] = name if not _check_shortcut_ownership(name, user): diff --git a/salt/states/firewalld.py b/salt/states/firewalld.py index 5ef4f3cd468..7b751547fb4 100644 --- a/salt/states/firewalld.py +++ b/salt/states/firewalld.py @@ -82,7 +82,7 @@ import logging # Import Salt Libs from salt.exceptions import CommandExecutionError -import salt.utils +import salt.utils.path log = logging.getLogger(__name__) @@ -145,7 +145,7 @@ def __virtual__(): ''' Ensure the firewall-cmd is available ''' - if salt.utils.which('firewall-cmd'): + if salt.utils.path.which('firewall-cmd'): return True return (False, 'firewall-cmd is not available, firewalld is probably not installed.') diff --git a/salt/states/git.py b/salt/states/git.py index 7a710fee109..dc94570d75c 100644 --- a/salt/states/git.py +++ b/salt/states/git.py @@ -21,13 +21,15 @@ import re import string # Import salt libs -import salt.utils +import salt.utils.args +import salt.utils.files import salt.utils.url +import salt.utils.versions from salt.exceptions import CommandExecutionError from salt.utils.versions import LooseVersion as _LooseVersion # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six log = logging.getLogger(__name__) @@ -572,11 +574,11 @@ def latest(name, ''' ret = {'name': name, 'result': True, 'comment': '', 'changes': {}} - kwargs = salt.utils.clean_kwargs(**kwargs) + kwargs = salt.utils.args.clean_kwargs(**kwargs) if kwargs: return _fail( ret, - salt.utils.invalid_kwargs(kwargs, raise_exc=False) + salt.utils.args.invalid_kwargs(kwargs, raise_exc=False) ) if not remote: @@ -1138,13 +1140,22 @@ def latest(name, password=password, https_user=https_user, https_pass=https_pass) - comments.append( - 'Remote \'{0}\' changed from {1} to {2}'.format( - remote, - salt.utils.url.redact_http_basic_auth(fetch_url), - redacted_fetch_url + if fetch_url is None: + comments.append( + 'Remote \'{0}\' set to {1}'.format( + remote, + redacted_fetch_url + ) + ) + ret['changes']['new'] = name + ' => ' + remote + else: + comments.append( + 'Remote \'{0}\' changed from {1} to {2}'.format( + remote, + salt.utils.url.redact_http_basic_auth(fetch_url), + redacted_fetch_url + ) ) - ) if remote_rev is not None: if __opts__['test']: @@ -1292,6 +1303,23 @@ def latest(name, 'if it does not already exist).', comments ) + remote_tags = set([ + x.replace('refs/tags/', '') for x in __salt__['git.ls_remote']( + cwd=target, + remote=remote, + opts="--tags", + user=user, + password=password, + identity=identity, + saltenv=__env__, + ignore_retcode=True, + ).keys() if '^{}' not in x + ]) + if set(all_local_tags) != remote_tags: + has_remote_rev = False + ret['changes']['new_tags'] = list(remote_tags.symmetric_difference( + all_local_tags + )) if not has_remote_rev: try: @@ -1447,8 +1475,6 @@ def latest(name, user=user, password=password, ignore_retcode=True): - merge_rev = remote_rev if rev == 'HEAD' \ - else desired_upstream if git_ver >= _LooseVersion('1.8.1.6'): # --ff-only added in version 1.8.1.6. It's not @@ -1465,7 +1491,7 @@ def latest(name, __salt__['git.merge']( target, - rev=merge_rev, + rev=remote_rev, opts=merge_opts, user=user, password=password) @@ -1597,7 +1623,7 @@ def latest(name, for target_object in target_contents: target_path = os.path.join(target, target_object) try: - salt.utils.rm_rf(target_path) + salt.utils.files.rm_rf(target_path) except OSError as exc: if exc.errno != errno.ENOENT: removal_errors[target_path] = exc @@ -1950,7 +1976,7 @@ def present(name, if os.path.islink(name): os.unlink(name) else: - salt.utils.rm_rf(name) + salt.utils.files.rm_rf(name) except OSError as exc: return _fail( ret, @@ -2107,11 +2133,11 @@ def detached(name, ret = {'name': name, 'result': True, 'comment': '', 'changes': {}} ref = kwargs.pop('ref', None) - kwargs = salt.utils.clean_kwargs(**kwargs) + kwargs = salt.utils.args.clean_kwargs(**kwargs) if kwargs: return _fail( ret, - salt.utils.invalid_kwargs(kwargs, raise_exc=False) + salt.utils.args.invalid_kwargs(kwargs, raise_exc=False) ) if ref is not None: @@ -2121,7 +2147,7 @@ def detached(name, 'consistency. Please update your SLS to reflect this.' ) ret.setdefault('warnings', []).append(deprecation_msg) - salt.utils.warn_until('Fluorine', deprecation_msg) + salt.utils.versions.warn_until('Fluorine', deprecation_msg) if not rev: return _fail( @@ -2213,7 +2239,7 @@ def detached(name, # Determine if supplied ref is a hash remote_rev_type = 'ref' - if len(ref) <= 40 \ + if len(rev) <= 40 \ and all(x in string.hexdigits for x in rev): rev = rev.lower() remote_rev_type = 'hash' @@ -2229,13 +2255,18 @@ def detached(name, local_commit_id = _get_local_rev_and_branch(target, user, password)[0] - if remote_rev_type is 'hash' \ - and __salt__['git.describe'](target, - rev, - user=user, - password=password): - # The rev is a hash and it exists locally so skip to checkout - hash_exists_locally = True + if remote_rev_type is 'hash': + try: + __salt__['git.describe'](target, + rev, + user=user, + password=password, + ignore_retcode=True) + except CommandExecutionError: + hash_exists_locally = False + else: + # The rev is a hash and it exists locally so skip to checkout + hash_exists_locally = True else: # Check that remote is present and set to correct url remotes = __salt__['git.remotes'](target, @@ -2299,7 +2330,7 @@ def detached(name, if os.path.islink(target): os.unlink(target) else: - salt.utils.rm_rf(target) + salt.utils.files.rm_rf(target) except OSError as exc: return _fail( ret, @@ -2419,7 +2450,7 @@ def detached(name, https_pass=https_pass, ignore_retcode=False) - if 'refs/remotes/'+remote+'/'+ref in all_remote_refs: + if 'refs/remotes/'+remote+'/'+rev in all_remote_refs: checkout_commit_id = all_remote_refs['refs/remotes/' + remote + '/' + rev] elif 'refs/tags/' + rev in all_remote_refs: checkout_commit_id = all_remote_refs['refs/tags/' + rev] @@ -2596,13 +2627,13 @@ def config_unset(name, # allows us to accept 'global' as an argument to this function without # shadowing global(), while also not allowing unwanted arguments to be # passed. - kwargs = salt.utils.clean_kwargs(**kwargs) + kwargs = salt.utils.args.clean_kwargs(**kwargs) global_ = kwargs.pop('global', False) all_ = kwargs.pop('all', False) if kwargs: return _fail( ret, - salt.utils.invalid_kwargs(kwargs, raise_exc=False) + salt.utils.args.invalid_kwargs(kwargs, raise_exc=False) ) if not global_ and not repo: @@ -2847,12 +2878,12 @@ def config_set(name, # allows us to accept 'global' as an argument to this function without # shadowing global(), while also not allowing unwanted arguments to be # passed. - kwargs = salt.utils.clean_kwargs(**kwargs) + kwargs = salt.utils.args.clean_kwargs(**kwargs) global_ = kwargs.pop('global', False) if kwargs: return _fail( ret, - salt.utils.invalid_kwargs(kwargs, raise_exc=False) + salt.utils.args.invalid_kwargs(kwargs, raise_exc=False) ) if not global_ and not repo: diff --git a/salt/states/github.py b/salt/states/github.py index 4b84ca77a55..05266433053 100644 --- a/salt/states/github.py +++ b/salt/states/github.py @@ -22,7 +22,7 @@ import datetime import logging # Import Salt Libs -import salt.ext.six as six +from salt.ext import six from salt.exceptions import CommandExecutionError from salt.ext.six.moves import range @@ -123,29 +123,30 @@ def absent(name, profile="github", **kwargs): target = __salt__['github.get_user'](name, profile=profile, **kwargs) - if not target: + if target: + if isinstance(target, bool) or target.get('in_org', False): + if __opts__['test']: + ret['comment'] = "User {0} will be deleted".format(name) + ret['result'] = None + return ret + + result = __salt__['github.remove_user'](name, profile=profile, **kwargs) + + if result: + ret['comment'] = 'Deleted user {0}'.format(name) + ret['changes'].setdefault('old', 'User {0} exists'.format(name)) + ret['changes'].setdefault('new', 'User {0} deleted'.format(name)) + ret['result'] = True + else: + ret['comment'] = 'Failed to delete {0}'.format(name) + ret['result'] = False + else: + ret['comment'] = "User {0} has already been deleted!".format(name) + ret['result'] = True + else: ret['comment'] = 'User {0} does not exist'.format(name) ret['result'] = True return ret - elif isinstance(target, bool) and target: - if __opts__['test']: - ret['comment'] = "User {0} will be deleted".format(name) - ret['result'] = None - return ret - - result = __salt__['github.remove_user'](name, profile=profile, **kwargs) - - if result: - ret['comment'] = 'Deleted user {0}'.format(name) - ret['changes'].setdefault('old', 'User {0} exists'.format(name)) - ret['changes'].setdefault('new', 'User {0} deleted'.format(name)) - ret['result'] = True - else: - ret['comment'] = 'Failed to delete {0}'.format(name) - ret['result'] = False - else: - ret['comment'] = "User {0} has already been deleted!".format(name) - ret['result'] = True return ret diff --git a/salt/states/glusterfs.py b/salt/states/glusterfs.py index d2fbe176c6e..cdc3684d944 100644 --- a/salt/states/glusterfs.py +++ b/salt/states/glusterfs.py @@ -120,13 +120,13 @@ def volume_present(name, bricks, stripe=False, replica=False, device_vg=False, .. code-block:: yaml myvolume: - glusterfs.created: + glusterfs.volume_present: - bricks: - host1:/srv/gluster/drive1 - host2:/srv/gluster/drive2 Replicated Volume: - glusterfs.created: + glusterfs.volume_present: - name: volume2 - bricks: - host1:/srv/gluster/drive2 diff --git a/salt/states/gnomedesktop.py b/salt/states/gnomedesktop.py index d3713e70734..e02f38166d6 100644 --- a/salt/states/gnomedesktop.py +++ b/salt/states/gnomedesktop.py @@ -29,6 +29,9 @@ from __future__ import absolute_import import logging import re +# Import 3rd-party libs +from salt.ext import six + log = logging.getLogger(__name__) @@ -65,7 +68,7 @@ def _do(name, gnome_kwargs, preferences): elif isinstance(value, int): ftype = 'int' - elif isinstance(value, str): + elif isinstance(value, six.string_types): ftype = 'string' else: ftype = 'string' diff --git a/salt/states/grafana4_dashboard.py b/salt/states/grafana4_dashboard.py index 0f3318c3b88..984b96273e6 100644 --- a/salt/states/grafana4_dashboard.py +++ b/salt/states/grafana4_dashboard.py @@ -44,7 +44,7 @@ import copy import json # Import Salt libs -import salt.ext.six as six +from salt.ext import six from salt.utils.dictdiffer import DictDiffer diff --git a/salt/states/grafana4_org.py b/salt/states/grafana4_org.py index 08376ba5cd9..791e6f61e87 100644 --- a/salt/states/grafana4_org.py +++ b/salt/states/grafana4_org.py @@ -42,11 +42,13 @@ Basic auth setup ''' from __future__ import absolute_import -from salt.ext.six import string_types -from salt.utils import dictupdate +import salt.utils.dictupdate as dictupdate from salt.utils.dictdiffer import deep_diff from requests.exceptions import HTTPError +# Import 3rd-party libs +from salt.ext.six import string_types + def __virtual__(): '''Only load if grafana4 module is available''' diff --git a/salt/states/grafana4_user.py b/salt/states/grafana4_user.py index ce1a46c50bb..a8211bac63b 100644 --- a/salt/states/grafana4_user.py +++ b/salt/states/grafana4_user.py @@ -37,10 +37,12 @@ Basic auth setup ''' from __future__ import absolute_import -from salt.ext.six import string_types -from salt.utils import dictupdate +import salt.utils.dictupdate as dictupdate from salt.utils.dictdiffer import deep_diff +# Import 3rd-party libs +from salt.ext.six import string_types + def __virtual__(): '''Only load if grafana4 module is available''' diff --git a/salt/states/grafana_dashboard.py b/salt/states/grafana_dashboard.py index bc088ef5b01..6e1ef15aab9 100644 --- a/salt/states/grafana_dashboard.py +++ b/salt/states/grafana_dashboard.py @@ -45,7 +45,7 @@ import json import requests # Import Salt libs -import salt.ext.six as six +from salt.ext import six from salt.utils.dictdiffer import DictDiffer diff --git a/salt/states/group.py b/salt/states/group.py index 8218e415d6e..8153e2da7fd 100644 --- a/salt/states/group.py +++ b/salt/states/group.py @@ -1,10 +1,15 @@ # -*- coding: utf-8 -*- -''' +r''' Management of user groups ========================= -The group module is used to create and manage unix group settings, groups -can be either present or absent: +The group module is used to create and manage group settings, groups can be +either present or absent. User/Group names can be passed to the ``adduser``, +``deluser``, and ``members`` parameters. ``adduser`` and ``deluser`` can be used +together but not with ``members``. + +In Windows, if no domain is specified in the user or group name (ie: +`DOMAIN\username``) the module will assume a local user or group. .. code-block:: yaml @@ -34,7 +39,11 @@ from __future__ import absolute_import import sys # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six + +# Import Salt libs +import salt.utils.platform +import salt.utils.win_functions def _changes(name, @@ -50,6 +59,18 @@ def _changes(name, if not lgrp: return False + # User and Domain names are not case sensitive in Windows. Let's make them + # all lower case so we can compare properly + if salt.utils.platform.is_windows(): + if lgrp['members']: + lgrp['members'] = [user.lower() for user in lgrp['members']] + if members: + members = [salt.utils.win_functions.get_sam_name(user) for user in members] + if addusers: + addusers = [salt.utils.win_functions.get_sam_name(user) for user in addusers] + if delusers: + delusers = [salt.utils.win_functions.get_sam_name(user) for user in delusers] + change = {} if gid: if lgrp['gid'] != gid: @@ -57,7 +78,7 @@ def _changes(name, if members: # -- if new member list if different than the current - if set(lgrp['members']) ^ set(members): + if set(lgrp['members']).symmetric_difference(members): change['members'] = members if addusers: @@ -79,31 +100,58 @@ def present(name, addusers=None, delusers=None, members=None): - ''' + r''' Ensure that a group is present - name - The name of the group to manage + Args: - gid - The group id to assign to the named group; if left empty, then the next - available group id will be assigned + name (str): + The name of the group to manage - system - Whether or not the named group is a system group. This is essentially - the '-r' option of 'groupadd'. + gid (str): + The group id to assign to the named group; if left empty, then the + next available group id will be assigned. Ignored on Windows - addusers - List of additional users to be added as a group members. + system (bool): + Whether or not the named group is a system group. This is essentially + the '-r' option of 'groupadd'. Ignored on Windows - delusers - Ensure these user are removed from the group membership. + addusers (list): + List of additional users to be added as a group members. Cannot + conflict with names in delusers. Cannot be used in conjunction with + members. - members - Replace existing group members with a list of new members. + delusers (list): + Ensure these user are removed from the group membership. Cannot + conflict with names in addusers. Cannot be used in conjunction with + members. - Note: Options 'members' and 'addusers/delusers' are mutually exclusive and - can not be used together. + members (list): + Replace existing group members with a list of new members. Cannot be + used in conjunction with addusers or delusers. + + Example: + + .. code-block:: yaml + + # Adds DOMAIN\db_admins and Administrators to the local db_admin group + # Removes Users + db_admin: + group.present: + - addusers: + - DOMAIN\db_admins + - Administrators + - delusers: + - Users + + # Ensures only DOMAIN\domain_admins and the local Administrator are + # members of the local Administrators group. All other users are + # removed + Administrators: + group.present: + - members: + - DOMAIN\domain_admins + - Administrator ''' ret = {'name': name, 'changes': {}, @@ -233,8 +281,17 @@ def absent(name): ''' Ensure that the named group is absent - name - The name of the group to remove + Args: + name (str): + The name of the group to remove + + Example: + + .. code-block:: yaml + + # Removes the local group `db_admin` + db_admin: + group.absent ''' ret = {'name': name, 'changes': {}, diff --git a/salt/states/heat.py b/salt/states/heat.py index c5f40f16878..caf549f4333 100644 --- a/salt/states/heat.py +++ b/salt/states/heat.py @@ -39,8 +39,7 @@ import json import logging # Import third party libs -import salt.ext.six as six -import salt.utils +from salt.ext import six import salt.utils.files import salt.exceptions import yaml @@ -212,9 +211,9 @@ def deployed(name, template=None, enviroment=None, params=None, poll=5, if (template_manage_result['result']) or \ ((__opts__['test']) and (template_manage_result['result'] is not False)): - with salt.utils.fopen(template_tmp_file, 'r') as tfp_: + with salt.utils.files.fopen(template_tmp_file, 'r') as tfp_: tpl = tfp_.read() - salt.utils.safe_rm(template_tmp_file) + salt.utils.files.safe_rm(template_tmp_file) try: if isinstance(tpl, six.binary_type): tpl = tpl.decode('utf-8') @@ -223,7 +222,7 @@ def deployed(name, template=None, enviroment=None, params=None, poll=5, template_new = yaml.dump(template_parse, Dumper=YamlDumper) else: template_new = jsonutils.dumps(template_parse, indent=2, ensure_ascii=False) - salt.utils.safe_rm(template_tmp_file) + salt.utils.files.safe_rm(template_tmp_file) except ValueError as ex: ret['result'] = False ret['comment'] = 'Error parsing template {0}'.format(ex) diff --git a/salt/states/hg.py b/salt/states/hg.py index 0b6425d8630..9e5ec190d28 100644 --- a/salt/states/hg.py +++ b/salt/states/hg.py @@ -14,23 +14,20 @@ in ~/.ssh/known_hosts, and the remote host has this host's public key. - target: /tmp/example_repo ''' -# Import python libs +# Import Python libs from __future__ import absolute_import import logging import os import shutil -# Import salt libs -import salt.utils +# Import Salt libs +import salt.utils.platform from salt.exceptions import CommandExecutionError from salt.states.git import _fail, _neutral_test log = logging.getLogger(__name__) -if salt.utils.is_windows(): - HG_BINARY = "hg.exe" -else: - HG_BINARY = "hg" +HG_BINARY = 'hg.exe' if salt.utils.platform.is_windows() else 'hg' def __virtual__(): diff --git a/salt/states/highstate_doc.py b/salt/states/highstate_doc.py new file mode 100644 index 00000000000..eded8e8db33 --- /dev/null +++ b/salt/states/highstate_doc.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- + +''' +To be used with proccessors in module `highstate_doc`. +''' + +__virtualname__ = 'highstate_doc' + + +def note(name, source=None, contents=None, **kwargs): + ''' + Add content to a document generated using `highstate_doc.render`. + + This state does not preform any tasks on the host. It only is used in highstate_doc lowstate proccessers + to include extra documents. + + .. code-block:: yaml + + {{sls}} example note: + highstate_doc.note: + - name: example note + - require_in: + - pkg: somepackage + - contents: | + example `highstate_doc.note` + ------------------ + This state does not do anything to the system! It is only used by a `proccesser` + you can use `requisites` and `order` to move your docs around the rendered file. + .. this message appare aboce the `pkg: somepackage` state. + - source: salt://{{tpldir}}/also_include_a_file.md + + {{sls}} extra help: + highstate_doc.note: + - name: example + - order: 0 + - source: salt://{{tpldir}}/HELP.md + ''' + comment = '' + if source: + comment += 'include file: {0}\n'.format(source) + if contents and len(contents) < 200: + comment += contents + return {'name': name, 'result': True, 'comment': comment, 'changes': {}} diff --git a/salt/states/host.py b/salt/states/host.py index 1c038d36fff..8345168cd54 100644 --- a/salt/states/host.py +++ b/salt/states/host.py @@ -61,7 +61,7 @@ Or delete all existing names for an address: from __future__ import absolute_import import salt.utils.validate.net -import salt.ext.six as six +from salt.ext import six def present(name, ip): # pylint: disable=C0103 diff --git a/salt/states/htpasswd.py b/salt/states/htpasswd.py index 160dcffc770..4bd0cdf8e66 100644 --- a/salt/states/htpasswd.py +++ b/salt/states/htpasswd.py @@ -15,7 +15,7 @@ Support for htpasswd module. Requires the apache2-utils package for Debian-based ''' from __future__ import absolute_import -import salt.utils +import salt.utils.path __virtualname__ = 'webutil' @@ -26,7 +26,7 @@ def __virtual__(): depends on webutil module ''' - return __virtualname__ if salt.utils.which('htpasswd') else False + return __virtualname__ if salt.utils.path.which('htpasswd') else False def user_exists(name, password=None, htpasswd_file=None, options='', diff --git a/salt/states/icinga2.py b/salt/states/icinga2.py index fe2b75b5bb5..7c7d73a8dc6 100644 --- a/salt/states/icinga2.py +++ b/salt/states/icinga2.py @@ -24,7 +24,7 @@ from __future__ import absolute_import import os.path # Import Salt libs -import salt.utils +import salt.utils.files def __virtual__(): @@ -119,7 +119,7 @@ def generate_ticket(name, output=None, grain=None, key=None, overwrite=True): ret['changes']['ticket'] = "Executed. Output into grain: {0}:{1}".format(grain, key) elif output: ret['changes']['ticket'] = "Executed. Output into {0}".format(output) - with salt.utils.fopen(output, 'w') as output_file: + with salt.utils.files.fopen(output, 'w') as output_file: output_file.write(str(ticket)) else: ret['changes']['ticket'] = "Executed" diff --git a/salt/states/infoblox.py b/salt/states/infoblox.py deleted file mode 100644 index 35d65fdb20b..00000000000 --- a/salt/states/infoblox.py +++ /dev/null @@ -1,239 +0,0 @@ -# -*- coding: utf-8 -*- -''' -states for infoblox stuff - -ensures a record is either present or absent in an Infoblox DNS system - -.. versionadded:: 2016.3.0 -''' -from __future__ import absolute_import - -# Import Python libs -import logging -from salt.ext import six - -log = logging.getLogger(__name__) - - -def __virtual__(): - ''' - make sure the infoblox module is available - ''' - return True if 'infoblox.get_record' in __salt__ else False - - -def present(name, - value, - record_type, - dns_view, - infoblox_server=None, - infoblox_user=None, - infoblox_password=None, - infoblox_api_version='v1.4.2', - sslVerify=True): - ''' - Ensure a record exists - - name - Name of the record - - value - Value of the record - - record_type - record type (host, a, cname, etc) - - dns_view - DNS View - - infoblox_server - infoblox server to connect to (will try pillar if not specified) - - infoblox_user - username to use to connect to infoblox (will try pillar if not specified) - - infoblox_password - password to use to connect to infoblox (will try pillar if not specified) - - verify_ssl - verify SSL certificates - - Example: - - .. code-block:: yaml - - some-state: - infoblox.present: - - name: some.dns.record - - value: 10.1.1.3 - - record_type: host - - sslVerify: False - ''' - record_type = record_type.lower() - value_utf8 = six.text_type(value, "utf-8") - ret = {'name': name, 'result': True, 'comment': '', 'changes': {}} - records = __salt__['infoblox.get_record'](name, - record_type, - infoblox_server=infoblox_server, - infoblox_user=infoblox_user, - infoblox_password=infoblox_password, - dns_view=dns_view, - infoblox_api_version=infoblox_api_version, - sslVerify=sslVerify) - if records: - # check records for updates - for record in records: - update_record = False - if record_type == 'cname': - if record['Canonical Name'] != value_utf8: - update_record = True - elif record_type == 'a': - if record['IP Address'] != value_utf8: - update_record = True - elif record_type == 'host': - if record['IP Addresses'] != [value_utf8]: - update_record = True - if update_record: - if __opts__['test']: - ret['result'] = None - ret['comment'] = ' '.join([ret['comment'], - 'DNS {0} record {1} in view {2} will be update'.format(record_type, - name, - dns_view)]) - else: - retval = __salt__['infoblox.update_record'](name, - value, - dns_view, - record_type, - infoblox_server=infoblox_server, - infoblox_user=infoblox_user, - infoblox_password=infoblox_password, - infoblox_api_version=infoblox_api_version, - sslVerify=sslVerify) - if retval: - if 'old' not in ret['changes']: - ret['changes']['old'] = [] - if 'new' not in ret['changes']: - ret['changes']['new'] = [] - ret['changes']['old'].append(record) - ret['changes']['new'].append(__salt__['infoblox.get_record'](name, - record_type, - infoblox_server=infoblox_server, - infoblox_user=infoblox_user, - infoblox_password=infoblox_password, - dns_view=dns_view, - infoblox_api_version=infoblox_api_version, - sslVerify=sslVerify)) - else: - ret['result'] = False - return ret - else: - # no records - if __opts__['test']: - ret['result'] = None - ret['comment'] = ' '.join([ret['comment'], - 'DNS {0} record {1} set to be added to view {2}'.format(record_type, - name, - dns_view)]) - return ret - retval = __salt__['infoblox.add_record'](name, - value, - record_type, - dns_view, - infoblox_server=infoblox_server, - infoblox_user=infoblox_user, - infoblox_password=infoblox_password, - infoblox_api_version='v1.4.2', - sslVerify=sslVerify) - if retval: - ret['result'] = True - ret['changes']['old'] = None - ret['changes']['new'] = __salt__['infoblox.get_record'](name, - record_type, - infoblox_server=infoblox_server, - infoblox_user=infoblox_user, - infoblox_password=infoblox_password, - dns_view=dns_view, - infoblox_api_version=infoblox_api_version, - sslVerify=sslVerify) - return ret - - -def absent(name, - record_type, - dns_view, - infoblox_server=None, - infoblox_user=None, - infoblox_password=None, - infoblox_api_version='v1.4.2', - sslVerify=True): - ''' - Ensure a record does not exists - - name - Name of the record - record_type - record type (host, a, cname, etc) - dns_view - DNS View - infoblox_server - infoblox server to connect to (will try pillar if not specified) - infoblox_user - username to use to connect to infoblox (will try pillar if not specified) - infoblox_password - password to use to connect to infoblox (will try pillar if not specified) - verify_ssl - verify SSL certificates - - Example: - - .. code-block:: yaml - - some-state: - infoblox.absent: - - name: some.dns.record - - record_type: host - - dns_view: MyView - - sslVerify: False - ''' - ret = {'name': name, 'result': True, 'comment': '', 'changes': {}} - record = __salt__['infoblox.get_record'](name, - record_type, - infoblox_server=infoblox_server, - infoblox_user=infoblox_user, - infoblox_password=infoblox_password, - dns_view=dns_view, - infoblox_api_version=infoblox_api_version, - sslVerify=sslVerify) - if record: - if __opts__['test']: - ret['result'] = None - ret['comment'] = ' '.join([ret['comment'], - 'DNS {0} record {1} in view {2} will be removed'.format(record_type, - name, - dns_view)]) - else: - retval = __salt__['infoblox.delete_record'](name, - dns_view, - record_type, - infoblox_server=infoblox_server, - infoblox_user=infoblox_user, - infoblox_password=infoblox_password, - infoblox_api_version=infoblox_api_version, - sslVerify=sslVerify) - if retval: - if 'old' not in ret['changes']: - ret['changes']['old'] = [] - ret['changes']['new'] = None - ret['changes']['old'].append(record) - else: - ret['result'] = False - return ret - else: - # record not found - ret['result'] = True - ret['changes']['old'] = None - ret['changes']['new'] = None - ret['comment'] = 'DNS record does not exist' - - return ret diff --git a/salt/states/infoblox_a.py b/salt/states/infoblox_a.py new file mode 100644 index 00000000000..1af4510cedf --- /dev/null +++ b/salt/states/infoblox_a.py @@ -0,0 +1,121 @@ +# -*- coding: utf-8 -*- +''' +Infoblox A record managment. + +functions accept api_opts: + + api_verifyssl: verify SSL [default to True or pillar value] + api_url: server to connect to [default to pillar value] + api_username: [default to pillar value] + api_password: [default to pillar value] +''' + + +def present(name=None, ipv4addr=None, data=None, ensure_data=True, **api_opts): + ''' + Ensure infoblox A record. + + When you wish to update a hostname ensure `name` is set to the hostname + of the current record. You can give a new name in the `data.name`. + + State example: + + .. code-block:: yaml + + infoblox_a.present: + - name: example-ha-0.domain.com + - data: + name: example-ha-0.domain.com + ipv4addr: 123.0.31.2 + view: Internal + ''' + ret = {'name': name, 'result': False, 'comment': '', 'changes': {}} + + if not data: + data = {} + if 'name' not in data: + data.update({'name': name}) + if 'ipv4addr' not in data: + data.update({'ipv4addr': ipv4addr}) + + obj = __salt__['infoblox.get_a'](name=name, ipv4addr=ipv4addr, allow_array=False, **api_opts) + if obj is None: + # perhaps the user updated the name + obj = __salt__['infoblox.get_a'](name=data['name'], ipv4addr=data['ipv4addr'], allow_array=False, **api_opts) + if obj: + # warn user that the data was updated and does not match + ret['result'] = False + ret['comment'] = '** please update the name: {0} to equal the updated data name {1}'.format(name, data['name']) + return ret + + if obj: + obj = obj[0] + if not ensure_data: + ret['result'] = True + ret['comment'] = 'infoblox record already created (supplied fields not ensured to match)' + return ret + + diff = __salt__['infoblox.diff_objects'](data, obj) + if not diff: + ret['result'] = True + ret['comment'] = 'supplied fields already updated (note: removing fields might not update)' + return ret + + if diff: + ret['changes'] = {'diff': diff} + if __opts__['test']: + ret['result'] = None + ret['comment'] = 'would attempt to update infoblox record' + return ret + ## TODO: perhaps need to review the output of new_obj + new_obj = __salt__['infoblox.update_object'](obj['_ref'], data=data, **api_opts) + ret['result'] = True + ret['comment'] = 'infoblox record fields updated (note: removing fields might not update)' + return ret + + if __opts__['test']: + ret['result'] = None + ret['comment'] = 'would attempt to create infoblox record {0}'.format(data['name']) + return ret + + new_obj_ref = __salt__['infoblox.create_a'](data=data, **api_opts) + new_obj = __salt__['infoblox.get_a'](name=name, ipv4addr=ipv4addr, allow_array=False, **api_opts) + + ret['result'] = True + ret['comment'] = 'infoblox record created' + ret['changes'] = {'old': 'None', 'new': {'_ref': new_obj_ref, 'data': new_obj}} + return ret + + +def absent(name=None, ipv4addr=None, **api_opts): + ''' + Ensure infoblox A record is removed. + + State example: + + .. code-block:: yaml + + infoblox_a.absent: + - name: example-ha-0.domain.com + + infoblox_a.absent: + - name: + - ipv4addr: 127.0.23.23 + ''' + ret = {'name': name, 'result': False, 'comment': '', 'changes': {}} + obj = __salt__['infoblox.get_a'](name=name, ipv4addr=ipv4addr, allow_array=False, **api_opts) + + if not obj: + ret['result'] = True + ret['comment'] = 'infoblox already removed' + return ret + + if __opts__['test']: + ret['result'] = None + ret['changes'] = {'old': obj, 'new': 'absent'} + return ret + + if __salt__['infoblox.delete_a'](name=name, ipv4addr=ipv4addr, **api_opts): + ret['result'] = True + ret['changes'] = {'old': obj, 'new': 'absent'} + return ret diff --git a/salt/states/infoblox_cname.py b/salt/states/infoblox_cname.py new file mode 100644 index 00000000000..35509e042bd --- /dev/null +++ b/salt/states/infoblox_cname.py @@ -0,0 +1,121 @@ +# -*- coding: utf-8 -*- +''' +Infoblox CNAME managment. + +functions accept api_opts: + + api_verifyssl: verify SSL [default to True or pillar value] + api_url: server to connect to [default to pillar value] + api_username: [default to pillar value] + api_password: [default to pillar value] +''' + + +def present(name=None, data=None, ensure_data=True, **api_opts): + ''' + Ensure the CNAME with the given data is present. + + name + CNAME of record + data + raw CNAME api data see: https://INFOBLOX/wapidoc + + State example: + + .. code-block:: yaml + + infoblox_cname.present: + - name: example-ha-0.domain.com + - data: + name: example-ha-0.domain.com + canonical: example.domain.com + zone: example.com + view: Internal + comment: Example comment + + infoblox_cname.present: + - name: example-ha-0.domain.com + - data: + name: example-ha-0.domain.com + canonical: example.domain.com + zone: example.com + view: Internal + comment: Example comment + - api_url: https://INFOBLOX/wapi/v1.2.1 + - api_username: username + - api_password: passwd + ''' + ret = {'name': name, 'result': False, 'comment': '', 'changes': {}} + if not data: + data = {} + if 'name' not in data: + data.update({'name': name}) + + obj = __salt__['infoblox.get_cname'](name=name, **api_opts) + if obj is None: + # perhaps the user updated the name + obj = __salt__['infoblox.get_cname'](name=data['name'], **api_opts) + if obj: + # warn user that the data was updated and does not match + ret['result'] = False + ret['comment'] = '** please update the name: {0} to equal the updated data name {1}'.format(name, data['name']) + return ret + + if obj: + if not ensure_data: + ret['result'] = True + ret['comment'] = 'infoblox record already created (supplied fields not ensured to match)' + return ret + + diff = __salt__['infoblox.diff_objects'](data, obj) + if not diff: + ret['result'] = True + ret['comment'] = 'supplied fields already updated (note: removing fields might not update)' + return ret + + if diff: + ret['changes'] = {'diff': diff} + if __opts__['test']: + ret['result'] = None + ret['comment'] = 'would attempt to update infoblox record' + return ret + new_obj = __salt__['infoblox.update_object'](obj['_ref'], data=data, **api_opts) + ret['result'] = True + ret['comment'] = 'infoblox record fields updated (note: removing fields might not update)' + return ret + + if __opts__['test']: + ret['result'] = None + ret['comment'] = 'would attempt to create infoblox record {0}'.format(data['name']) + return ret + + new_obj_ref = __salt__['infoblox.create_cname'](data=data, **api_opts) + new_obj = __salt__['infoblox.get_cname'](name=name, **api_opts) + + ret['result'] = True + ret['comment'] = 'infoblox record created' + ret['changes'] = {'old': 'None', 'new': {'_ref': new_obj_ref, 'data': new_obj}} + return ret + + +def absent(name=None, canonical=None, **api_opts): + ''' + Ensure the CNAME with the given name or canonical name is removed + ''' + ret = {'name': name, 'result': False, 'comment': '', 'changes': {}} + obj = __salt__['infoblox.get_cname'](name=name, canonical=canonical, **api_opts) + + if not obj: + ret['result'] = True + ret['comment'] = 'infoblox already removed' + return ret + + if __opts__['test']: + ret['result'] = None + ret['changes'] = {'old': obj, 'new': 'absent'} + return ret + + if __salt__['infoblox.delete_cname'](name=name, canonical=canonical, **api_opts): + ret['result'] = True + ret['changes'] = {'old': obj, 'new': 'absent'} + return ret diff --git a/salt/states/infoblox_host_record.py b/salt/states/infoblox_host_record.py new file mode 100644 index 00000000000..fbf8ab3061e --- /dev/null +++ b/salt/states/infoblox_host_record.py @@ -0,0 +1,163 @@ +# -*- coding: utf-8 -*- +''' +Infoblox host record managment. + +functions accept api_opts: + + api_verifyssl: verify SSL [default to True or pillar value] + api_url: server to connect to [default to pillar value] + api_username: [default to pillar value] + api_password: [default to pillar value] +''' + + +def present(name=None, data=None, ensure_data=True, **api_opts): + ''' + This will ensure that a host with the provided name exists. + This will try to ensure that the state of the host matches the given data + If the host is not found then one will be created. + + When trying to update a hostname ensure `name` is set to the hostname + of the current record. You can give a new name in the `data.name`. + + Avoid race conditions, use func:nextavailableip: + - func:nextavailableip:network/ZG54dfgsrDFEFfsfsLzA:10.0.0.0/8/default + - func:nextavailableip:10.0.0.0/8 + - func:nextavailableip:10.0.0.0/8,externalconfigure_for_dns + - func:nextavailableip:10.0.0.3-10.0.0.10 + + State Example: + + .. code-block:: yaml + + # this would update `original_hostname.example.ca` to changed `data`. + infoblox_host_record.present: + - name: original_hostname.example.ca + - data: {'namhostname.example.cae': 'hostname.example.ca', + 'aliases': ['hostname.math.example.ca'], + 'extattrs': [{'Business Contact': {'value': 'EXAMPLE@example.ca'}}], + 'ipv4addrs': [{'configure_for_dhcp': True, + 'ipv4addr': 'func:nextavailableip:129.97.139.0/24', + 'mac': '00:50:56:84:6e:ae'}], + 'ipv6addrs': [], } + ''' + ret = {'name': name, 'result': False, 'comment': '', 'changes': {}} + if data is None: + data = {} + if 'name' not in data: + data.update({'name': name}) + + obj = __salt__['infoblox.get_host'](name=name, **api_opts) + if obj is None: + # perhaps the user updated the name + obj = __salt__['infoblox.get_host'](name=data['name'], **api_opts) + if obj: + # warn user that the host name was updated and does not match + ret['result'] = False + ret['comment'] = 'please update the name: {0} to equal the updated data name {1}'.format(name, data['name']) + return ret + + if obj: + if not ensure_data: + ret['result'] = True + ret['comment'] = 'infoblox record already created (supplied fields not ensured to match)' + return ret + + obj = __salt__['infoblox.get_host_advanced'](name=name, **api_opts) + diff = __salt__['infoblox.diff_objects'](data, obj) + if not diff: + ret['result'] = True + ret['comment'] = 'supplied fields already updated (note: removing fields might not update)' + return ret + + if diff: + ret['changes'] = {'diff': diff} + if __opts__['test']: + ret['result'] = None + ret['comment'] = 'would attempt to update infoblox record' + return ret + + # replace func:nextavailableip with current ip address if in range + # get list of ipaddresses that are defined. + obj_addrs = [] + if 'ipv4addrs' in obj: + for addr in obj['ipv4addrs']: + if 'ipv4addr' in addr: + obj_addrs.append(addr['ipv4addr']) + if 'ipv6addrs' in obj: + for addr in obj['ipv6addrs']: + if 'ipv6addr' in addr: + obj_addrs.append(addr['ipv6addr']) + + # replace func:nextavailableip: if an ip address is already found in that range. + if 'ipv4addrs' in data: + for addr in data['ipv4addrs']: + if 'ipv4addr' in addr: + addrobj = addr['ipv4addr'] + if addrobj.startswith('func:nextavailableip:'): + found_matches = 0 + for ip in obj_addrs: + if __salt__['infoblox.is_ipaddr_in_ipfunc_range'](ip, addrobj): + addr['ipv4addr'] = ip + found_matches += 1 + if found_matches > 1: + ret['comment'] = 'infoblox record cant updated because ipaddress {0} matches mutiple func:nextavailableip'.format(ip) + ret['result'] = False + return ret + + new_obj = __salt__['infoblox.update_object'](obj['_ref'], data=data, **api_opts) + ret['result'] = True + ret['comment'] = 'infoblox record fields updated (note: removing fields might not update)' + #ret['changes'] = {'diff': diff } + return ret + + if __opts__['test']: + ret['result'] = None + ret['comment'] = 'would attempt to create infoblox record {0}'.format(name) + return ret + + new_obj_ref = __salt__['infoblox.create_host'](data=data, **api_opts) + new_obj = __salt__['infoblox.get_host'](name=name, **api_opts) + + ret['result'] = True + ret['comment'] = 'infoblox record created' + ret['changes'] = {'old': 'None', 'new': {'_ref': new_obj_ref, 'data': new_obj}} + return ret + + +def absent(name=None, ipv4addr=None, mac=None, **api_opts): + ''' + Ensure the host with the given Name ipv4addr or mac is removed. + + State example: + + .. code-block:: yaml + + infoblox_host_record.absent: + - name: hostname.of.record.to.remove + + infoblox_host_record.absent: + - name: + - ipv4addr: 192.168.0.1 + + infoblox_host_record.absent: + - name: + - mac: 12:02:12:31:23:43 + ''' + ret = {'name': name, 'result': False, 'comment': '', 'changes': {}} + obj = __salt__['infoblox.get_host'](name=name, ipv4addr=ipv4addr, mac=mac, **api_opts) + + if not obj: + ret['result'] = True + ret['comment'] = 'infoblox already removed' + return ret + + if __opts__['test']: + ret['result'] = None + ret['changes'] = {'old': obj, 'new': 'absent'} + return ret + + if __salt__['infoblox.delete_host'](name=name, mac=mac, **api_opts): + ret['result'] = True + ret['changes'] = {'old': obj, 'new': 'absent'} + return ret diff --git a/salt/states/infoblox_range.py b/salt/states/infoblox_range.py new file mode 100644 index 00000000000..aa8769b338d --- /dev/null +++ b/salt/states/infoblox_range.py @@ -0,0 +1,188 @@ +# -*- coding: utf-8 -*- +''' +Infoblox host record managment. + +functions accept api_opts: + + api_verifyssl: verify SSL [default to True or pillar value] + api_url: server to connect to [default to pillar value] + api_username: [default to pillar value] + api_password: [default to pillar value] +''' + + +def present(name=None, start_addr=None, end_addr=None, data=None, **api_opts): + ''' + Ensure range record is present. + + infoblox_range.present: + start_addr: '129.97.150.160', + end_addr: '129.97.150.170', + + Verbose state example: + + .. code-block:: yaml + + infoblox_range.present: + data: { + 'always_update_dns': False, + 'authority': False, + 'comment': 'range of IP addresses used for salt.. was used for ghost images deployment', + 'ddns_generate_hostname': True, + 'deny_all_clients': False, + 'deny_bootp': False, + 'disable': False, + 'email_list': [], + 'enable_ddns': False, + 'enable_dhcp_thresholds': False, + 'enable_email_warnings': False, + 'enable_ifmap_publishing': False, + 'enable_snmp_warnings': False, + 'end_addr': '129.97.150.169', + 'exclude': [], + 'extattrs': {}, + 'fingerprint_filter_rules': [], + 'high_water_mark': 95, + 'high_water_mark_reset': 85, + 'ignore_dhcp_option_list_request': False, + 'lease_scavenge_time': -1, + 'logic_filter_rules': [], + 'low_water_mark': 0, + 'low_water_mark_reset': 10, + 'mac_filter_rules': [], + 'member': {'_struct': 'dhcpmember', + 'ipv4addr': '129.97.128.9', + 'name': 'cn-dhcp-mc.example.ca'}, + 'ms_options': [], + 'nac_filter_rules': [], + 'name': 'ghost-range', + 'network': '129.97.150.0/24', + 'network_view': 'default', + 'option_filter_rules': [], + 'options': [{'name': 'dhcp-lease-time', + 'num': 51, + 'use_option': False, + 'value': '43200', + 'vendor_class': 'DHCP'}], + 'recycle_leases': True, + 'relay_agent_filter_rules': [], + 'server_association_type': 'MEMBER', + 'start_addr': '129.97.150.160', + 'update_dns_on_lease_renewal': False, + 'use_authority': False, + 'use_bootfile': False, + 'use_bootserver': False, + 'use_ddns_domainname': False, + 'use_ddns_generate_hostname': True, + 'use_deny_bootp': False, + 'use_email_list': False, + 'use_enable_ddns': False, + 'use_enable_dhcp_thresholds': False, + 'use_enable_ifmap_publishing': False, + 'use_ignore_dhcp_option_list_request': False, + 'use_known_clients': False, + 'use_lease_scavenge_time': False, + 'use_nextserver': False, + 'use_options': False, + 'use_recycle_leases': False, + 'use_unknown_clients': False, + 'use_update_dns_on_lease_renewal': False + } + ''' + ret = {'name': name, 'result': False, 'comment': '', 'changes': {}} + + if not data: + data = {} + if 'name' not in data: + data.update({'name': name}) + if 'start_addr' not in data: + data.update({'start_addr': start_addr}) + if 'end_addr' not in data: + data.update({'end_addr': end_addr}) + + obj = __salt__['infoblox.get_ipv4_range'](data['start_addr'], data['end_addr'], **api_opts) + if obj is None: + obj = __salt__['infoblox.get_ipv4_range'](start_addr=data['start_addr'], end_addr=None, **api_opts) + if obj is None: + obj = __salt__['infoblox.get_ipv4_range'](start_addr=None, end_addr=data['end_addr'], **api_opts) + + if obj: + diff = __salt__['infoblox.diff_objects'](data, obj) + if not diff: + ret['result'] = True + ret['comment'] = 'supplied fields in correct state' + return ret + if diff: + if __opts__['test']: + ret['result'] = None + ret['comment'] = 'would attempt to update record' + return ret + new_obj = __salt__['infoblox.update_object'](obj['_ref'], data=data, **api_opts) + ret['result'] = True + ret['comment'] = 'record fields updated' + ret['changes'] = {'diff': diff} + return ret + + if __opts__['test']: + ret['result'] = None + ret['comment'] = 'would attempt to create record {0}'.format(name) + return ret + + new_obj_ref = __salt__['infoblox.create_ipv4_range'](data, **api_opts) + new_obj = __salt__['infoblox.get_ipv4_range'](data['start_addr'], data['end_addr'], **api_opts) + + ret['result'] = True + ret['comment'] = 'record created' + ret['changes'] = {'old': 'None', 'new': {'_ref': new_obj_ref, 'data': new_obj}} + return ret + + +def absent(name=None, start_addr=None, end_addr=None, data=None, **api_opts): + ''' + Ensure the range is removed + + Supplying the end of the range is optional. + + State example: + + .. code-block:: yaml + + infoblox_range.absent: + - name: 'vlan10' + + infoblox_range.absent: + - name: + - start_addr: 127.0.1.20 + ''' + ret = {'name': name, 'result': False, 'comment': '', 'changes': {}} + + if not data: + data = {} + if 'name' not in data: + data.update({'name': name}) + if 'start_addr' not in data: + data.update({'start_addr': start_addr}) + if 'end_addr' not in data: + data.update({'end_addr': end_addr}) + + obj = __salt__['infoblox.get_ipv4_range'](data['start_addr'], data['end_addr'], **api_opts) + if obj is None: + obj = __salt__['infoblox.get_ipv4_range'](start_addr=data['start_addr'], end_addr=None, **api_opts) + if obj is None: + obj = __salt__['infoblox.get_ipv4_range'](start_addr=None, end_addr=data['end_addr'], **api_opts) + + if not obj: + ret['result'] = True + ret['comment'] = 'already deleted' + return ret + + if __opts__['test']: + ret['result'] = None + ret['comment'] = 'would attempt to delete range' + return ret + + if __salt__['infoblox.delete_object'](objref=obj['_ref']): + ret['result'] = True + ret['changes'] = {'old': 'Found {0} - {1}'.format(start_addr, end_addr), + 'new': 'Removed'} + return ret diff --git a/salt/states/ini_manage.py b/salt/states/ini_manage.py index e22202e0f10..d0ed67e8d37 100644 --- a/salt/states/ini_manage.py +++ b/salt/states/ini_manage.py @@ -14,7 +14,7 @@ Manage ini files from __future__ import absolute_import # Import Salt libs -import salt.ext.six as six +from salt.ext import six __virtualname__ = 'ini' @@ -206,7 +206,7 @@ def sections_present(name, sections=None, separator='='): ret['result'] = False ret['comment'] = "{0}".format(err) return ret - if cmp(dict(sections[section]), cur_section) == 0: + if dict(sections[section]) == cur_section: ret['comment'] += 'Section unchanged {0}.\n'.format(section) continue elif cur_section: diff --git a/salt/states/iptables.py b/salt/states/iptables.py index ec4eec4e960..6163e799508 100644 --- a/salt/states/iptables.py +++ b/salt/states/iptables.py @@ -194,7 +194,6 @@ at some point be deprecated in favor of a more generic ``firewall`` state. from __future__ import absolute_import # Import salt libs -import salt.utils from salt.state import STATE_INTERNAL_KEYWORDS as _STATE_INTERNAL_KEYWORDS @@ -810,7 +809,7 @@ def mod_aggregate(low, chunks, running): if low.get('fun') not in agg_enabled: return low for chunk in chunks: - tag = salt.utils.gen_state_tag(chunk) + tag = __utils__['state.gen_tag'](chunk) if tag in running: # Already ran the iptables state, skip aggregation continue diff --git a/salt/states/jboss7.py b/salt/states/jboss7.py index 9c5d2950da0..18aa1e449a6 100644 --- a/salt/states/jboss7.py +++ b/salt/states/jboss7.py @@ -44,11 +44,11 @@ import re import traceback # Import Salt libs -from salt.utils import dictdiffer +import salt.utils.dictdiffer as dictdiffer from salt.exceptions import CommandExecutionError # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six log = logging.getLogger(__name__) diff --git a/salt/states/jenkins.py b/salt/states/jenkins.py index 01548e53816..95e5c545850 100644 --- a/salt/states/jenkins.py +++ b/salt/states/jenkins.py @@ -13,9 +13,10 @@ import difflib import logging # Import Salt libs -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import zip -import salt.utils +from salt.exceptions import CommandExecutionError +import salt.utils.files # Import XML parser import xml.etree.ElementTree as ET @@ -36,17 +37,23 @@ def _elements_equal(e1, e2): return all(_elements_equal(c1, c2) for c1, c2 in zip(e1, e2)) +def _fail(ret, msg): + ret['comment'] = msg + ret['result'] = False + return ret + + def present(name, config=None, **kwargs): ''' - Ensure the job is present in the Jenkins - configured jobs + Ensure the job is present in the Jenkins configured jobs + name The unique name for the Jenkins job + config - The Salt URL for the file to use for - configuring the job. + The Salt URL for the file to use for configuring the job ''' ret = {'name': name, @@ -54,35 +61,40 @@ def present(name, 'changes': {}, 'comment': ['Job {0} is up to date.'.format(name)]} - _job_exists = __salt__['jenkins.job_exists'](name) - - if _job_exists: + if __salt__['jenkins.job_exists'](name): _current_job_config = __salt__['jenkins.get_job_config'](name) buf = six.moves.StringIO(_current_job_config) oldXML = ET.fromstring(buf.read()) cached_source_path = __salt__['cp.cache_file'](config, __env__) - with salt.utils.fopen(cached_source_path) as _fp: + with salt.utils.files.fopen(cached_source_path) as _fp: newXML = ET.fromstring(_fp.read()) if not _elements_equal(oldXML, newXML): diff = difflib.unified_diff( ET.tostringlist(oldXML, encoding='utf8', method='xml'), ET.tostringlist(newXML, encoding='utf8', method='xml'), lineterm='') - __salt__['jenkins.update_job'](name, config, __env__) - ret['changes'][name] = ''.join(diff) - ret['comment'].append('Job {0} updated.'.format(name)) + try: + __salt__['jenkins.update_job'](name, config, __env__) + except CommandExecutionError as exc: + return _fail(ret, exc.strerror) + else: + ret['changes'] = ''.join(diff) + ret['comment'].append('Job \'{0}\' updated.'.format(name)) else: cached_source_path = __salt__['cp.cache_file'](config, __env__) - with salt.utils.fopen(cached_source_path) as _fp: + with salt.utils.files.fopen(cached_source_path) as _fp: new_config_xml = _fp.read() - __salt__['jenkins.create_job'](name, config, __env__) + try: + __salt__['jenkins.create_job'](name, config, __env__) + except CommandExecutionError as exc: + return _fail(ret, exc.strerror) buf = six.moves.StringIO(new_config_xml) diff = difflib.unified_diff('', buf.readlines(), lineterm='') ret['changes'][name] = ''.join(diff) - ret['comment'].append('Job {0} added.'.format(name)) + ret['comment'].append('Job \'{0}\' added.'.format(name)) ret['comment'] = '\n'.join(ret['comment']) return ret @@ -91,24 +103,23 @@ def present(name, def absent(name, **kwargs): ''' - Ensure the job is present in the Jenkins - configured jobs + Ensure the job is absent from the Jenkins configured jobs name - The name of the Jenkins job to remove. - + The name of the Jenkins job to remove ''' - ret = {'name': name, 'result': True, 'changes': {}, 'comment': []} - _job_exists = __salt__['jenkins.job_exists'](name) - - if _job_exists: - __salt__['jenkins.delete_job'](name) - ret['comment'] = 'Job {0} deleted.'.format(name) + if __salt__['jenkins.job_exists'](name): + try: + __salt__['jenkins.delete_job'](name) + except CommandExecutionError as exc: + return _fail(ret, exc.strerror) + else: + ret['comment'] = 'Job \'{0}\' deleted.'.format(name) else: - ret['comment'] = 'Job {0} already absent.'.format(name) + ret['comment'] = 'Job \'{0}\' already absent.'.format(name) return ret diff --git a/salt/states/k8s.py b/salt/states/k8s.py index 879843be174..da52e36f761 100644 --- a/salt/states/k8s.py +++ b/salt/states/k8s.py @@ -25,6 +25,11 @@ Manage Kubernetes - node: myothernodename - apiserver: http://mykubeapiserer:8080 ''' +from __future__ import absolute_import + +# Import salt libs +import salt.utils.versions + __virtualname__ = 'k8s' @@ -42,6 +47,10 @@ def label_present( node=None, apiserver=None): ''' + .. deprecated:: 2017.7.0 + This state has been moved to :py:func:`kubernetes.node_label_present + ` for details on - how to detect this condition, :mod:`kernelpkg.latest_active ` + See :py:func:`kernelpkg.needs_reboot ` for details on + how to detect this condition, and :py:func:`~salt.states.kernelpkg.latest_active` to initiale a reboot when needed. name @@ -113,8 +113,9 @@ def latest_active(name, at_time=None, **kwargs): # pylint: disable=unused-argum system. If the running version is not the latest one installed, this state will reboot the system. - See :mod:`kernelpkg.upgrade ` and - :mod:`kernelpkg.latest_installed ` for ways to install new kernel packages. + See :py:func:`kernelpkg.upgrade ` and + :py:func:`~salt.states.kernelpkg.latest_installed` + for ways to install new kernel packages. This module does not attempt to understand or manage boot loader configurations it is possible to have a new kernel installed, but a boot loader configuration @@ -122,7 +123,8 @@ def latest_active(name, at_time=None, **kwargs): # pylint: disable=unused-argum schedule this state to run automatically. Because this state function may cause the system to reboot, it may be preferable - to move it to the very end of the state run. See :mod:`kernelpkg.latest_wait ` + to move it to the very end of the state run. + See :py:func:`~salt.states.kernelpkg.latest_wait` for a waitable state that can be called with the `listen` requesite. name @@ -168,7 +170,7 @@ def latest_active(name, at_time=None, **kwargs): # pylint: disable=unused-argum def latest_wait(name, at_time=None, **kwargs): # pylint: disable=unused-argument ''' Initiate a reboot if the running kernel is not the latest one installed. This is the - waitable version of :mod:`kernelpkg.latest_active ` and + waitable version of :py:func:`~salt.states.kernelpkg.latest_active` and will not take any action unless triggered by a watch or listen requesite. .. note:: diff --git a/salt/states/kubernetes.py b/salt/states/kubernetes.py new file mode 100644 index 00000000000..952b78cbb62 --- /dev/null +++ b/salt/states/kubernetes.py @@ -0,0 +1,995 @@ +# -*- coding: utf-8 -*- +''' +Manage kubernetes resources as salt states +========================================== + +NOTE: This module requires the proper pillar values set. See +salt.modules.kubernetes for more information. + +The kubernetes module is used to manage different kubernetes resources. + + +.. code-block:: yaml + + my-nginx: + kubernetes.deployment_present: + - namespace: default + metadata: + app: frontend + spec: + replicas: 1 + template: + metadata: + labels: + run: my-nginx + spec: + containers: + - name: my-nginx + image: nginx + ports: + - containerPort: 80 + + my-mariadb: + kubernetes.deployment_absent: + - namespace: default + + # kubernetes deployment as specified inside of + # a file containing the definition of the the + # deployment using the official kubernetes format + redis-master-deployment: + kubernetes.deployment_present: + - name: redis-master + - source: salt://k8s/redis-master-deployment.yml + require: + - pip: kubernetes-python-module + + # kubernetes service as specified inside of + # a file containing the definition of the the + # service using the official kubernetes format + redis-master-service: + kubernetes.service_present: + - name: redis-master + - source: salt://k8s/redis-master-service.yml + require: + - kubernetes.deployment_present: redis-master + + # kubernetes deployment as specified inside of + # a file containing the definition of the the + # deployment using the official kubernetes format + # plus some jinja directives + nginx-source-template: + kubernetes.deployment_present: + - source: salt://k8s/nginx.yml.jinja + - template: jinja + require: + - pip: kubernetes-python-module + + + # Kubernetes secret + k8s-secret: + kubernetes.secret_present: + - name: top-secret + data: + key1: value1 + key2: value2 + key3: value3 + +.. versionadded: 2017.7.0 +''' +from __future__ import absolute_import + +import copy +import logging + +# Import 3rd-party libs +from salt.ext import six + +log = logging.getLogger(__name__) + + +def __virtual__(): + ''' + Only load if the kubernetes module is available in __salt__ + ''' + return 'kubernetes.ping' in __salt__ + + +def _error(ret, err_msg): + ''' + Helper function to propagate errors to + the end user. + ''' + ret['result'] = False + ret['comment'] = err_msg + return ret + + +def deployment_absent(name, namespace='default', **kwargs): + ''' + Ensures that the named deployment is absent from the given namespace. + + name + The name of the deployment + + namespace + The name of the namespace + ''' + + ret = {'name': name, + 'changes': {}, + 'result': False, + 'comment': ''} + + deployment = __salt__['kubernetes.show_deployment'](name, namespace, **kwargs) + + if deployment is None: + ret['result'] = True if not __opts__['test'] else None + ret['comment'] = 'The deployment does not exist' + return ret + + if __opts__['test']: + ret['comment'] = 'The deployment is going to be deleted' + ret['result'] = None + return ret + + res = __salt__['kubernetes.delete_deployment'](name, namespace, **kwargs) + if res['code'] == 200: + ret['result'] = True + ret['changes'] = { + 'kubernetes.deployment': { + 'new': 'absent', 'old': 'present'}} + ret['comment'] = res['message'] + else: + ret['comment'] = 'Something went wrong, response: {0}'.format(res) + + return ret + + +def deployment_present( + name, + namespace='default', + metadata=None, + spec=None, + source='', + template='', + **kwargs): + ''' + Ensures that the named deployment is present inside of the specified + namespace with the given metadata and spec. + If the deployment exists it will be replaced. + + name + The name of the deployment. + + namespace + The namespace holding the deployment. The 'default' one is going to be + used unless a different one is specified. + + metadata + The metadata of the deployment object. + + spec + The spec of the deployment object. + + source + A file containing the definition of the deployment (metadata and + spec) in the official kubernetes format. + + template + Template engine to be used to render the source file. + ''' + ret = {'name': name, + 'changes': {}, + 'result': False, + 'comment': ''} + + if (metadata or spec) and source: + return _error( + ret, + '\'source\' cannot be used in combination with \'metadata\' or ' + '\'spec\'' + ) + + if metadata is None: + metadata = {} + + if spec is None: + spec = {} + + deployment = __salt__['kubernetes.show_deployment'](name, namespace, **kwargs) + + if deployment is None: + if __opts__['test']: + ret['result'] = None + ret['comment'] = 'The deployment is going to be created' + return ret + res = __salt__['kubernetes.create_deployment'](name=name, + namespace=namespace, + metadata=metadata, + spec=spec, + source=source, + template=template, + saltenv=__env__, + **kwargs) + ret['changes']['{0}.{1}'.format(namespace, name)] = { + 'old': {}, + 'new': res} + else: + if __opts__['test']: + ret['result'] = None + return ret + + # TODO: improve checks # pylint: disable=fixme + log.info('Forcing the recreation of the deployment') + ret['comment'] = 'The deployment is already present. Forcing recreation' + res = __salt__['kubernetes.replace_deployment']( + name=name, + namespace=namespace, + metadata=metadata, + spec=spec, + source=source, + template=template, + saltenv=__env__, + **kwargs) + + ret['changes'] = { + 'metadata': metadata, + 'spec': spec + } + ret['result'] = True + return ret + + +def service_present( + name, + namespace='default', + metadata=None, + spec=None, + source='', + template='', + **kwargs): + ''' + Ensures that the named service is present inside of the specified namespace + with the given metadata and spec. + If the deployment exists it will be replaced. + + name + The name of the service. + + namespace + The namespace holding the service. The 'default' one is going to be + used unless a different one is specified. + + metadata + The metadata of the service object. + + spec + The spec of the service object. + + source + A file containing the definition of the service (metadata and + spec) in the official kubernetes format. + + template + Template engine to be used to render the source file. + ''' + ret = {'name': name, + 'changes': {}, + 'result': False, + 'comment': ''} + + if (metadata or spec) and source: + return _error( + ret, + '\'source\' cannot be used in combination with \'metadata\' or ' + '\'spec\'' + ) + + if metadata is None: + metadata = {} + + if spec is None: + spec = {} + + service = __salt__['kubernetes.show_service'](name, namespace, **kwargs) + + if service is None: + if __opts__['test']: + ret['result'] = None + ret['comment'] = 'The service is going to be created' + return ret + res = __salt__['kubernetes.create_service'](name=name, + namespace=namespace, + metadata=metadata, + spec=spec, + source=source, + template=template, + saltenv=__env__, + **kwargs) + ret['changes']['{0}.{1}'.format(namespace, name)] = { + 'old': {}, + 'new': res} + else: + if __opts__['test']: + ret['result'] = None + return ret + + # TODO: improve checks # pylint: disable=fixme + log.info('Forcing the recreation of the service') + ret['comment'] = 'The service is already present. Forcing recreation' + res = __salt__['kubernetes.replace_service']( + name=name, + namespace=namespace, + metadata=metadata, + spec=spec, + source=source, + template=template, + old_service=service, + saltenv=__env__, + **kwargs) + + ret['changes'] = { + 'metadata': metadata, + 'spec': spec + } + ret['result'] = True + return ret + + +def service_absent(name, namespace='default', **kwargs): + ''' + Ensures that the named service is absent from the given namespace. + + name + The name of the service + + namespace + The name of the namespace + ''' + + ret = {'name': name, + 'changes': {}, + 'result': False, + 'comment': ''} + + service = __salt__['kubernetes.show_service'](name, namespace, **kwargs) + + if service is None: + ret['result'] = True if not __opts__['test'] else None + ret['comment'] = 'The service does not exist' + return ret + + if __opts__['test']: + ret['comment'] = 'The service is going to be deleted' + ret['result'] = None + return ret + + res = __salt__['kubernetes.delete_service'](name, namespace, **kwargs) + if res['code'] == 200: + ret['result'] = True + ret['changes'] = { + 'kubernetes.service': { + 'new': 'absent', 'old': 'present'}} + ret['comment'] = res['message'] + else: + ret['comment'] = 'Something went wrong, response: {0}'.format(res) + + return ret + + +def namespace_absent(name, **kwargs): + ''' + Ensures that the named namespace is absent. + + name + The name of the namespace + ''' + + ret = {'name': name, + 'changes': {}, + 'result': False, + 'comment': ''} + + namespace = __salt__['kubernetes.show_namespace'](name, **kwargs) + + if namespace is None: + ret['result'] = True if not __opts__['test'] else None + ret['comment'] = 'The namespace does not exist' + return ret + + if __opts__['test']: + ret['comment'] = 'The namespace is going to be deleted' + ret['result'] = None + return ret + + res = __salt__['kubernetes.delete_namespace'](name, **kwargs) + if ( + res['code'] == 200 or + ( + isinstance(res['status'], six.string_types) and + 'Terminating' in res['status'] + ) or + ( + isinstance(res['status'], dict) and + res['status']['phase'] == 'Terminating' + )): + ret['result'] = True + ret['changes'] = { + 'kubernetes.namespace': { + 'new': 'absent', 'old': 'present'}} + if res['message']: + ret['comment'] = res['message'] + else: + ret['comment'] = 'Terminating' + else: + ret['comment'] = 'Something went wrong, response: {0}'.format(res) + + return ret + + +def namespace_present(name, **kwargs): + ''' + Ensures that the named namespace is present. + + name + The name of the deployment. + + ''' + ret = {'name': name, + 'changes': {}, + 'result': False, + 'comment': ''} + + namespace = __salt__['kubernetes.show_namespace'](name, **kwargs) + + if namespace is None: + if __opts__['test']: + ret['result'] = None + ret['comment'] = 'The namespace is going to be created' + return ret + + res = __salt__['kubernetes.create_namespace'](name, **kwargs) + ret['changes']['namespace'] = { + 'old': {}, + 'new': res} + else: + ret['result'] = True if not __opts__['test'] else None + ret['comment'] = 'The namespace already exists' + + return ret + + +def secret_absent(name, namespace='default', **kwargs): + ''' + Ensures that the named secret is absent from the given namespace. + + name + The name of the secret + + namespace + The name of the namespace + ''' + + ret = {'name': name, + 'changes': {}, + 'result': False, + 'comment': ''} + + secret = __salt__['kubernetes.show_secret'](name, namespace, **kwargs) + + if secret is None: + ret['result'] = True if not __opts__['test'] else None + ret['comment'] = 'The secret does not exist' + return ret + + if __opts__['test']: + ret['comment'] = 'The secret is going to be deleted' + ret['result'] = None + return ret + + __salt__['kubernetes.delete_secret'](name, namespace, **kwargs) + + # As for kubernetes 1.6.4 doesn't set a code when deleting a secret + # The kubernetes module will raise an exception if the kubernetes + # server will return an error + ret['result'] = True + ret['changes'] = { + 'kubernetes.secret': { + 'new': 'absent', 'old': 'present'}} + ret['comment'] = 'Secret deleted' + return ret + + +def secret_present( + name, + namespace='default', + data=None, + source='', + template='', + **kwargs): + ''' + Ensures that the named secret is present inside of the specified namespace + with the given data. + If the secret exists it will be replaced. + + name + The name of the secret. + + namespace + The namespace holding the secret. The 'default' one is going to be + used unless a different one is specified. + + data + The dictionary holding the secrets. + + source + A file containing the data of the secret in plain format. + + template + Template engine to be used to render the source file. + ''' + ret = {'name': name, + 'changes': {}, + 'result': False, + 'comment': ''} + + if data and source: + return _error( + ret, + '\'source\' cannot be used in combination with \'data\'' + ) + + secret = __salt__['kubernetes.show_secret'](name, namespace, **kwargs) + + if secret is None: + if data is None: + data = {} + + if __opts__['test']: + ret['result'] = None + ret['comment'] = 'The secret is going to be created' + return ret + res = __salt__['kubernetes.create_secret'](name=name, + namespace=namespace, + data=data, + source=source, + template=template, + saltenv=__env__, + **kwargs) + ret['changes']['{0}.{1}'.format(namespace, name)] = { + 'old': {}, + 'new': res} + else: + if __opts__['test']: + ret['result'] = None + return ret + + # TODO: improve checks # pylint: disable=fixme + log.info('Forcing the recreation of the service') + ret['comment'] = 'The secret is already present. Forcing recreation' + res = __salt__['kubernetes.replace_secret']( + name=name, + namespace=namespace, + data=data, + source=source, + template=template, + saltenv=__env__, + **kwargs) + + ret['changes'] = { + # Omit values from the return. They are unencrypted + # and can contain sensitive data. + 'data': res['data'].keys() + } + ret['result'] = True + + return ret + + +def configmap_absent(name, namespace='default', **kwargs): + ''' + Ensures that the named configmap is absent from the given namespace. + + name + The name of the configmap + + namespace + The name of the namespace + ''' + + ret = {'name': name, + 'changes': {}, + 'result': False, + 'comment': ''} + + configmap = __salt__['kubernetes.show_configmap'](name, namespace, **kwargs) + + if configmap is None: + ret['result'] = True if not __opts__['test'] else None + ret['comment'] = 'The configmap does not exist' + return ret + + if __opts__['test']: + ret['comment'] = 'The configmap is going to be deleted' + ret['result'] = None + return ret + + __salt__['kubernetes.delete_configmap'](name, namespace, **kwargs) + # As for kubernetes 1.6.4 doesn't set a code when deleting a configmap + # The kubernetes module will raise an exception if the kubernetes + # server will return an error + ret['result'] = True + ret['changes'] = { + 'kubernetes.configmap': { + 'new': 'absent', 'old': 'present'}} + ret['comment'] = 'ConfigMap deleted' + + return ret + + +def configmap_present( + name, + namespace='default', + data=None, + source='', + template='', + **kwargs): + ''' + Ensures that the named configmap is present inside of the specified namespace + with the given data. + If the configmap exists it will be replaced. + + name + The name of the configmap. + + namespace + The namespace holding the configmap. The 'default' one is going to be + used unless a different one is specified. + + data + The dictionary holding the configmaps. + + source + A file containing the data of the configmap in plain format. + + template + Template engine to be used to render the source file. + ''' + ret = {'name': name, + 'changes': {}, + 'result': False, + 'comment': ''} + + if data and source: + return _error( + ret, + '\'source\' cannot be used in combination with \'data\'' + ) + + configmap = __salt__['kubernetes.show_configmap'](name, namespace, **kwargs) + + if configmap is None: + if __opts__['test']: + ret['result'] = None + ret['comment'] = 'The configmap is going to be created' + return ret + res = __salt__['kubernetes.create_configmap'](name=name, + namespace=namespace, + data=data, + source=source, + template=template, + saltenv=__env__, + **kwargs) + ret['changes']['{0}.{1}'.format(namespace, name)] = { + 'old': {}, + 'new': res} + else: + if __opts__['test']: + ret['result'] = None + return ret + + # TODO: improve checks # pylint: disable=fixme + log.info('Forcing the recreation of the service') + ret['comment'] = 'The configmap is already present. Forcing recreation' + res = __salt__['kubernetes.replace_configmap']( + name=name, + namespace=namespace, + data=data, + source=source, + template=template, + saltenv=__env__, + **kwargs) + + ret['changes'] = { + 'data': res['data'] + } + ret['result'] = True + return ret + + +def pod_absent(name, namespace='default', **kwargs): + ''' + Ensures that the named pod is absent from the given namespace. + + name + The name of the pod + + namespace + The name of the namespace + ''' + + ret = {'name': name, + 'changes': {}, + 'result': False, + 'comment': ''} + + pod = __salt__['kubernetes.show_pod'](name, namespace, **kwargs) + + if pod is None: + ret['result'] = True if not __opts__['test'] else None + ret['comment'] = 'The pod does not exist' + return ret + + if __opts__['test']: + ret['comment'] = 'The pod is going to be deleted' + ret['result'] = None + return ret + + res = __salt__['kubernetes.delete_pod'](name, namespace, **kwargs) + if res['code'] == 200 or res['code'] is None: + ret['result'] = True + ret['changes'] = { + 'kubernetes.pod': { + 'new': 'absent', 'old': 'present'}} + if res['code'] is None: + ret['comment'] = 'In progress' + else: + ret['comment'] = res['message'] + else: + ret['comment'] = 'Something went wrong, response: {0}'.format(res) + + return ret + + +def pod_present( + name, + namespace='default', + metadata=None, + spec=None, + source='', + template='', + **kwargs): + ''' + Ensures that the named pod is present inside of the specified + namespace with the given metadata and spec. + If the pod exists it will be replaced. + + name + The name of the pod. + + namespace + The namespace holding the pod. The 'default' one is going to be + used unless a different one is specified. + + metadata + The metadata of the pod object. + + spec + The spec of the pod object. + + source + A file containing the definition of the pod (metadata and + spec) in the official kubernetes format. + + template + Template engine to be used to render the source file. + ''' + ret = {'name': name, + 'changes': {}, + 'result': False, + 'comment': ''} + + if (metadata or spec) and source: + return _error( + ret, + '\'source\' cannot be used in combination with \'metadata\' or ' + '\'spec\'' + ) + + if metadata is None: + metadata = {} + + if spec is None: + spec = {} + + pod = __salt__['kubernetes.show_pod'](name, namespace, **kwargs) + + if pod is None: + if __opts__['test']: + ret['result'] = None + ret['comment'] = 'The pod is going to be created' + return ret + res = __salt__['kubernetes.create_pod'](name=name, + namespace=namespace, + metadata=metadata, + spec=spec, + source=source, + template=template, + saltenv=__env__, + **kwargs) + ret['changes']['{0}.{1}'.format(namespace, name)] = { + 'old': {}, + 'new': res} + else: + if __opts__['test']: + ret['result'] = None + return ret + + # TODO: fix replace_namespaced_pod validation issues + ret['comment'] = 'salt is currently unable to replace a pod without ' \ + 'deleting it. Please perform the removal of the pod requiring ' \ + 'the \'pod_absent\' state if this is the desired behaviour.' + ret['result'] = False + return ret + + ret['changes'] = { + 'metadata': metadata, + 'spec': spec + } + ret['result'] = True + return ret + + +def node_label_absent(name, node, **kwargs): + ''' + Ensures that the named label is absent from the node. + + name + The name of the label + + node + The name of the node + ''' + + ret = {'name': name, + 'changes': {}, + 'result': False, + 'comment': ''} + + labels = __salt__['kubernetes.node_labels'](node, **kwargs) + + if name not in labels: + ret['result'] = True if not __opts__['test'] else None + ret['comment'] = 'The label does not exist' + return ret + + if __opts__['test']: + ret['comment'] = 'The label is going to be deleted' + ret['result'] = None + return ret + + __salt__['kubernetes.node_remove_label']( + node_name=node, + label_name=name, + **kwargs) + + ret['result'] = True + ret['changes'] = { + 'kubernetes.node_label': { + 'new': 'absent', 'old': 'present'}} + ret['comment'] = 'Label removed from node' + + return ret + + +def node_label_folder_absent(name, node, **kwargs): + ''' + Ensures the label folder doesn't exist on the specified node. + + name + The name of label folder + + node + The name of the node + ''' + + ret = {'name': name, + 'changes': {}, + 'result': False, + 'comment': ''} + labels = __salt__['kubernetes.node_labels'](node, **kwargs) + + folder = name.strip("/") + "/" + labels_to_drop = [] + new_labels = [] + for label in labels: + if label.startswith(folder): + labels_to_drop.append(label) + else: + new_labels.append(label) + + if not labels_to_drop: + ret['result'] = True if not __opts__['test'] else None + ret['comment'] = 'The label folder does not exist' + return ret + + if __opts__['test']: + ret['comment'] = 'The label folder is going to be deleted' + ret['result'] = None + return ret + + for label in labels_to_drop: + __salt__['kubernetes.node_remove_label']( + node_name=node, + label_name=label, + **kwargs) + + ret['result'] = True + ret['changes'] = { + 'kubernetes.node_label_folder_absent': { + 'new': new_labels, 'old': labels.keys()}} + ret['comment'] = 'Label folder removed from node' + + return ret + + +def node_label_present( + name, + node, + value, + **kwargs): + ''' + Ensures that the named label is set on the named node + with the given value. + If the label exists it will be replaced. + + name + The name of the label. + + value + Value of the label. + + node + Node to change. + ''' + ret = {'name': name, + 'changes': {}, + 'result': False, + 'comment': ''} + + labels = __salt__['kubernetes.node_labels'](node, **kwargs) + + if name not in labels: + if __opts__['test']: + ret['result'] = None + ret['comment'] = 'The label is going to be set' + return ret + __salt__['kubernetes.node_add_label'](label_name=name, + label_value=value, + node_name=node, + **kwargs) + elif labels[name] == value: + ret['result'] = True + ret['comment'] = 'The label is already set and has the specified value' + return ret + else: + if __opts__['test']: + ret['result'] = None + return ret + + ret['comment'] = 'The label is already set, changing the value' + __salt__['kubernetes.node_add_label']( + node_name=node, + label_name=name, + label_value=value, + **kwargs) + + old_labels = copy.copy(labels) + labels[name] = value + + ret['changes']['{0}.{1}'.format(node, name)] = { + 'old': old_labels, + 'new': labels} + ret['result'] = True + + return ret diff --git a/salt/states/ldap.py b/salt/states/ldap.py index e12ee23a0a4..5ea2b96c3b2 100644 --- a/salt/states/ldap.py +++ b/salt/states/ldap.py @@ -14,8 +14,9 @@ from __future__ import absolute_import import copy import inspect import logging -import salt.ext.six as six +from salt.ext import six from salt.utils.odict import OrderedDict +from salt.utils.oset import OrderedSet log = logging.getLogger(__name__) @@ -335,16 +336,16 @@ def managed(name, entries, connect_spec=None): changed_old[dn] = o changed_new[dn] = n success_dn_set[dn] = True - except ldap3.LDAPError: - log.exception('failed to %s entry %s', op, dn) - errs.append((op, dn)) + except ldap3.LDAPError as err: + log.exception('failed to %s entry %s (%s)', op, dn, err) + errs.append((op, dn, err)) continue if len(errs): ret['result'] = False ret['comment'] = 'failed to ' \ - + ', '.join((op + ' entry ' + dn - for op, dn in errs)) + + ', '.join((op + ' entry ' + dn + '(' + str(err) + ')' + for op, dn, err in errs)) # set ret['changes']. filter out any unchanged attributes, and # convert the value sets to lists before returning them to the @@ -421,7 +422,7 @@ def _process_entries(l, entries): results = __salt__['ldap3.search'](l, dn, 'base') if len(results) == 1: attrs = results[dn] - olde = dict(((attr, set(attrs[attr])) + olde = dict(((attr, OrderedSet(attrs[attr])) for attr in attrs if len(attrs[attr]))) else: @@ -478,7 +479,7 @@ def _update_entry(entry, status, directives): if len(vals): entry[attr] = vals elif directive == 'delete': - existing_vals = entry.pop(attr, set()) + existing_vals = entry.pop(attr, OrderedSet()) if len(vals): existing_vals -= vals if len(existing_vals): @@ -496,9 +497,16 @@ def _toset(thing): This enables flexibility in what users provide as the list of LDAP entry attribute values. Note that the LDAP spec prohibits - duplicate values in an attribute and that the order is - unspecified, so a set is good for automatically removing - duplicates. + duplicate values in an attribute. + + RFC 2251 states that: + "The order of attribute values within the vals set is undefined and + implementation-dependent, and MUST NOT be relied upon." + However, OpenLDAP have an X-ORDERED that is used in the config schema. + Using sets would mean we can't pass ordered values and therefore can't + manage parts of the OpenLDAP configuration, hence the use of OrderedSet. + + Sets are also good for automatically removing duplicates. None becomes an empty set. Iterables except for strings have their elements added to a new set. Non-None scalars (strings, @@ -507,12 +515,12 @@ def _toset(thing): ''' if thing is None: - return set() + return OrderedSet() if isinstance(thing, six.string_types): - return set((thing,)) + return OrderedSet((thing,)) # convert numbers to strings so that equality checks work # (LDAP stores numbers as strings) try: - return set((str(x) for x in thing)) + return OrderedSet((str(x) for x in thing)) except TypeError: - return set((str(thing),)) + return OrderedSet((str(thing),)) diff --git a/salt/states/linux_acl.py b/salt/states/linux_acl.py index a6a54a7fcdc..0f733f31c27 100644 --- a/salt/states/linux_acl.py +++ b/salt/states/linux_acl.py @@ -32,10 +32,13 @@ from __future__ import absolute_import import os # Import salt libs -import salt.utils +import salt.utils.path + +# Impot salt exceptions +from salt.exceptions import CommandExecutionError # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six __virtualname__ = 'acl' @@ -44,7 +47,7 @@ def __virtual__(): ''' Ensure getfacl & setfacl exist ''' - if salt.utils.which('getfacl') and salt.utils.which('setfacl'): + if salt.utils.path.which('getfacl') and salt.utils.path.which('setfacl'): return __virtualname__ return False @@ -57,6 +60,7 @@ def present(name, acl_type, acl_name='', perms='', recurse=False): ret = {'name': name, 'result': True, 'changes': {}, + 'pchanges': {}, 'comment': ''} _octal = {'r': 4, 'w': 2, 'x': 1, '-': 0} @@ -99,21 +103,54 @@ def present(name, acl_type, acl_name='', perms='', recurse=False): if user[_search_name]['octal'] == sum([_octal.get(i, i) for i in perms]): ret['comment'] = 'Permissions are in the desired state' else: - ret['comment'] = 'Permissions have been updated' + changes = {'new': {'acl_name': acl_name, + 'acl_type': acl_type, + 'perms': perms}, + 'old': {'acl_name': acl_name, + 'acl_type': acl_type, + 'perms': str(user[_search_name]['octal'])}} if __opts__['test']: - ret['result'] = None + ret.update({'comment': 'Updated permissions will be applied for ' + '{0}: {1} -> {2}'.format( + acl_name, + str(user[_search_name]['octal']), + perms), + 'result': None, 'pchanges': changes}) return ret - - __salt__['acl.modfacl'](acl_type, acl_name, perms, name, recursive=recurse) + try: + __salt__['acl.modfacl'](acl_type, acl_name, perms, name, + recursive=recurse, raise_err=True) + ret.update({'comment': 'Updated permissions for ' + '{0}'.format(acl_name), + 'result': True, 'changes': changes}) + except CommandExecutionError as exc: + ret.update({'comment': 'Error updating permissions for ' + '{0}: {1}'.format(acl_name, exc.strerror), + 'result': False}) else: - ret['comment'] = 'Permissions will be applied' + changes = {'new': {'acl_name': acl_name, + 'acl_type': acl_type, + 'perms': perms}} if __opts__['test']: + ret.update({'comment': 'New permissions will be applied for ' + '{0}: {1}'.format(acl_name, perms), + 'result': None, 'pchanges': changes}) ret['result'] = None return ret - __salt__['acl.modfacl'](acl_type, acl_name, perms, name, recursive=recurse) + try: + __salt__['acl.modfacl'](acl_type, acl_name, perms, name, + recursive=recurse, raise_err=True) + ret.update({'comment': 'Applied new permissions for ' + '{0}'.format(acl_name), + 'result': True, 'changes': changes}) + except CommandExecutionError as exc: + ret.update({'comment': 'Error updating permissions for {0}: ' + '{1}'.format(acl_name, exc.strerror), + 'result': False}) + else: ret['comment'] = 'ACL Type does not exist' ret['result'] = False diff --git a/salt/states/logadm.py b/salt/states/logadm.py index 8618509db7d..0a1af8f1bdc 100644 --- a/salt/states/logadm.py +++ b/salt/states/logadm.py @@ -22,6 +22,7 @@ import logging # Import salt libs import salt.utils +import salt.utils.args log = logging.getLogger(__name__) @@ -60,7 +61,7 @@ def rotate(name, **kwargs): 'comment': ''} ## cleanup kwargs - kwargs = salt.utils.clean_kwargs(**kwargs) + kwargs = salt.utils.args.clean_kwargs(**kwargs) ## inject name as entryname if 'entryname' not in kwargs: diff --git a/salt/states/lvm.py b/salt/states/lvm.py index d7f0770007b..9a2378fe46d 100644 --- a/salt/states/lvm.py +++ b/salt/states/lvm.py @@ -27,15 +27,15 @@ from __future__ import absolute_import import os # Import salt libs -import salt.utils -import salt.ext.six as six +import salt.utils.path +from salt.ext import six def __virtual__(): ''' Only load the module if lvm is installed ''' - if salt.utils.which('lvm'): + if salt.utils.path.which('lvm'): return 'lvm' return False @@ -268,7 +268,7 @@ def lv_present(name, else: lvpath = '/dev/{0}/{1}'.format(vgname, name) - if __salt__['lvm.lvdisplay'](lvpath): + if __salt__['lvm.lvdisplay'](lvpath, quiet=True): ret['comment'] = 'Logical Volume {0} already present'.format(name) elif __opts__['test']: ret['comment'] = 'Logical Volume {0} is set to be created'.format(name) @@ -290,7 +290,7 @@ def lv_present(name, ret['comment'] = 'Created Logical Volume {0}'.format(name) ret['changes']['created'] = changes else: - ret['comment'] = 'Failed to create Logical Volume {0}'.format(name) + ret['comment'] = 'Failed to create Logical Volume {0}. Error: {1}'.format(name, changes) ret['result'] = False return ret diff --git a/salt/states/lxc.py b/salt/states/lxc.py index ccb7907093d..e05061a6ad2 100644 --- a/salt/states/lxc.py +++ b/salt/states/lxc.py @@ -8,7 +8,7 @@ from __future__ import absolute_import __docformat__ = 'restructuredtext en' # Import salt libs -import salt.utils +import salt.utils.versions from salt.exceptions import CommandExecutionError, SaltInvocationError @@ -708,7 +708,7 @@ def edited_conf(name, lxc_conf=None, lxc_conf_unset=None): # Until a reasonable alternative for this state function is created, we need # to keep this function around and cannot officially remove it. Progress of # the new function will be tracked in https://github.com/saltstack/salt/issues/35523 - salt.utils.warn_until( + salt.utils.versions.warn_until( 'Oxygen', 'This state is unsuitable for setting parameters that appear more ' 'than once in an LXC config file, or parameters which must appear in ' diff --git a/salt/states/mac_assistive.py b/salt/states/mac_assistive.py index 77ca78093d1..8cc5dd4c903 100644 --- a/salt/states/mac_assistive.py +++ b/salt/states/mac_assistive.py @@ -12,12 +12,12 @@ Install, enable and disable assistive access on macOS minions - enabled: True ''' -# Import python libs +# Import Python libs from __future__ import absolute_import import logging -# Import salt libs -import salt.utils +# Import Salt libs +import salt.utils.platform from salt.utils.versions import LooseVersion as _LooseVersion log = logging.getLogger(__name__) @@ -29,7 +29,8 @@ def __virtual__(): ''' Only work on Mac OS ''' - if salt.utils.is_darwin() and _LooseVersion(__grains__['osrelease']) >= _LooseVersion('10.9'): + if salt.utils.platform.is_darwin() \ + and _LooseVersion(__grains__['osrelease']) >= _LooseVersion('10.9'): return True return False diff --git a/salt/states/mac_defaults.py b/salt/states/mac_defaults.py index 0eee644753f..be4076e2516 100644 --- a/salt/states/mac_defaults.py +++ b/salt/states/mac_defaults.py @@ -5,12 +5,12 @@ Writing/reading defaults from a macOS minion ''' -# Import python libs +# Import Python libs from __future__ import absolute_import import logging -# Import salt libs -import salt.utils +# Import Salt libs +import salt.utils.platform log = logging.getLogger(__name__) __virtualname__ = 'macdefaults' @@ -20,7 +20,7 @@ def __virtual__(): ''' Only work on Mac OS ''' - if salt.utils.is_darwin(): + if salt.utils.platform.is_darwin(): return __virtualname__ return False diff --git a/salt/states/mac_keychain.py b/salt/states/mac_keychain.py index e67a99a4035..e54c2d60ef2 100644 --- a/salt/states/mac_keychain.py +++ b/salt/states/mac_keychain.py @@ -13,12 +13,12 @@ Install certificats to the macOS keychain ''' from __future__ import absolute_import -# Import python libs +# Import Python libs import logging import os -# Import salt libs -import salt.utils +# Import Salt libs +import salt.utils.platform log = logging.getLogger(__name__) __virtualname__ = 'keychain' @@ -28,7 +28,7 @@ def __virtual__(): ''' Only work on Mac OS ''' - if salt.utils.is_darwin(): + if salt.utils.platform.is_darwin(): return __virtualname__ return False diff --git a/salt/states/mac_package.py b/salt/states/mac_package.py index ed7dae04037..4d100e65c67 100644 --- a/salt/states/mac_package.py +++ b/salt/states/mac_package.py @@ -22,15 +22,14 @@ Install any kind of pkg, dmg or app file on macOS: - target: /Applications/Xcode.app - version_check: xcodebuild -version=Xcode 7.1\n.*7B91b ''' - -# Import python libs +# Import Python libs from __future__ import absolute_import import logging import os import re -# Import salt libs -import salt.utils +# Import Salt libs +import salt.utils.platform from salt.exceptions import CommandExecutionError log = logging.getLogger(__name__) @@ -41,7 +40,7 @@ def __virtual__(): ''' Only work on Mac OS ''' - if salt.utils.is_darwin(): + if salt.utils.platform.is_darwin(): return __virtualname__ return False diff --git a/salt/states/makeconf.py b/salt/states/makeconf.py index 0458c4aaa70..aa442ed65a5 100644 --- a/salt/states/makeconf.py +++ b/salt/states/makeconf.py @@ -11,6 +11,10 @@ A state module to manage Gentoo's ``make.conf`` file makeconf.present: - value: '-j3' ''' +from __future__ import absolute_import + +# Import 3rd-party libs +from salt.ext import six def __virtual__(): @@ -27,7 +31,7 @@ def _make_set(var): if var is None: return set() if not isinstance(var, list): - if isinstance(var, str): + if isinstance(var, six.string_types): var = var.split() else: var = list(var) diff --git a/salt/states/mdadm.py b/salt/states/mdadm.py index b2e25f07ae3..2b1c8340877 100644 --- a/salt/states/mdadm.py +++ b/salt/states/mdadm.py @@ -23,10 +23,10 @@ from __future__ import absolute_import import logging # Import salt libs -import salt.utils +import salt.utils.path # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six # Set up logger log = logging.getLogger(__name__) @@ -41,7 +41,7 @@ def __virtual__(): ''' if __grains__['kernel'] != 'Linux': return False - if not salt.utils.which('mdadm'): + if not salt.utils.path.which('mdadm'): return False return __virtualname__ diff --git a/salt/states/modjk.py b/salt/states/modjk.py index 90cf6cb3657..8d138d580b6 100644 --- a/salt/states/modjk.py +++ b/salt/states/modjk.py @@ -8,7 +8,7 @@ from __future__ import absolute_import import logging # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six log = logging.getLogger(__name__) diff --git a/salt/states/modjk_worker.py b/salt/states/modjk_worker.py index f902d99c0aa..baa8d3f93d5 100644 --- a/salt/states/modjk_worker.py +++ b/salt/states/modjk_worker.py @@ -19,7 +19,7 @@ Mandatory Settings: execution module :mod:`documentation ` ''' from __future__ import absolute_import -import salt.utils +import salt.utils.versions def __virtual__(): @@ -195,7 +195,7 @@ def stop(name, lbn, target, profile='default', tgt_type='glob', expr_form=None): # remember to remove the expr_form argument from this function when # performing the cleanup on this deprecation. if expr_form is not None: - salt.utils.warn_until( + salt.utils.versions.warn_until( 'Fluorine', 'the target type should be passed using the \'tgt_type\' ' 'argument instead of \'expr_form\'. Support for using ' @@ -229,7 +229,7 @@ def activate(name, lbn, target, profile='default', tgt_type='glob', expr_form=No # remember to remove the expr_form argument from this function when # performing the cleanup on this deprecation. if expr_form is not None: - salt.utils.warn_until( + salt.utils.versions.warn_until( 'Fluorine', 'the target type should be passed using the \'tgt_type\' ' 'argument instead of \'expr_form\'. Support for using ' @@ -264,7 +264,7 @@ def disable(name, lbn, target, profile='default', tgt_type='glob', expr_form=Non # remember to remove the expr_form argument from this function when # performing the cleanup on this deprecation. if expr_form is not None: - salt.utils.warn_until( + salt.utils.versions.warn_until( 'Fluorine', 'the target type should be passed using the \'tgt_type\' ' 'argument instead of \'expr_form\'. Support for using ' diff --git a/salt/states/module.py b/salt/states/module.py index 6fd9f541cff..a253db9ae96 100644 --- a/salt/states/module.py +++ b/salt/states/module.py @@ -163,10 +163,11 @@ functions at once the following way: - user: myuser - opts: '--all' -By default this behaviour is not turned on. In ordder to do so, please add the following +By default this behaviour is not turned on. In order to do so, please add the following configuration to the minion: .. code-block:: yaml + use_superseded: - module.run @@ -214,11 +215,12 @@ def wait(name, **kwargs): watch = salt.utils.alias_function(wait, 'watch') -@with_deprecated(globals(), "Fluorine", policy=with_deprecated.OPT_IN) +@with_deprecated(globals(), "Sodium", policy=with_deprecated.OPT_IN) def run(**kwargs): ''' Run a single module function or a range of module functions in a batch. - Supersedes `module.run` function, which requires `m_` prefix to function-specific parameters. + Supersedes ``module.run`` function, which requires ``m_`` prefix to + function-specific parameters. :param returner: Specify a common returner for the whole batch to send the return data @@ -227,8 +229,9 @@ def run(**kwargs): Pass any arguments needed to execute the function(s) .. code-block:: yaml + some_id_of_state: - module.xrun: + module.run: - network.ip_addrs: - interface: eth0 - cloud.create: @@ -259,6 +262,7 @@ def run(**kwargs): missing = [] tests = [] for func in functions: + func = func.split(':')[0] if func not in __salt__: missing.append(func) elif __opts__['test']: @@ -281,8 +285,9 @@ def run(**kwargs): failures = [] success = [] for func in functions: + _func = func.split(':')[0] try: - func_ret = _call_function(func, returner=kwargs.get('returner'), + func_ret = _call_function(_func, returner=kwargs.get('returner'), func_args=kwargs.get(func)) if not _get_result(func_ret, ret['changes'].get('ret', {})): if isinstance(func_ret, dict): @@ -310,27 +315,40 @@ def _call_function(name, returner=None, **kwargs): ''' argspec = salt.utils.args.get_function_argspec(__salt__[name]) func_kw = dict(zip(argspec.args[-len(argspec.defaults or []):], # pylint: disable=incompatible-py3-code - argspec.defaults or [])) - func_args = [] - for funcset in kwargs.get('func_args') or {}: - if isinstance(funcset, dict): - func_kw.update(funcset) + argspec.defaults or [])) + arg_type, na_type, kw_type = [], {}, False + for funcset in reversed(kwargs.get('func_args') or []): + if not isinstance(funcset, dict): + kw_type = True + if kw_type: + if isinstance(funcset, dict): + arg_type += funcset.values() + na_type.update(funcset) + else: + arg_type.append(funcset) else: - func_args.append(funcset) + func_kw.update(funcset) + arg_type.reverse() + _exp_prm = len(argspec.args or []) - len(argspec.defaults or []) + _passed_prm = len(arg_type) missing = [] - for arg in argspec.args: - if arg not in func_kw: - missing.append(arg) + if na_type and _exp_prm > _passed_prm: + for arg in argspec.args: + if arg not in func_kw: + missing.append(arg) if missing: raise SaltInvocationError('Missing arguments: {0}'.format(', '.join(missing))) + elif _exp_prm > _passed_prm: + raise SaltInvocationError('Function expects {0} parameters, got only {1}'.format( + _exp_prm, _passed_prm)) - mret = __salt__[name](*func_args, **func_kw) + mret = __salt__[name](*arg_type, **func_kw) if returner is not None: returners = salt.loader.returners(__opts__, __salt__) if returner in returners: returners[returner]({'id': __opts__['id'], 'ret': mret, - 'fun': name, 'jid': salt.utils.jid.gen_jid()}) + 'fun': name, 'jid': salt.utils.jid.gen_jid(__opts__)}) return mret @@ -424,16 +442,30 @@ def _run(name, **kwargs): ret['result'] = False return ret - if aspec.varargs and aspec.varargs in kwargs: - varargs = kwargs.pop(aspec.varargs) + if aspec.varargs: + if aspec.varargs == 'name': + rarg = 'm_name' + elif aspec.varargs == 'fun': + rarg = 'm_fun' + elif aspec.varargs == 'names': + rarg = 'm_names' + elif aspec.varargs == 'state': + rarg = 'm_state' + elif aspec.varargs == 'saltenv': + rarg = 'm_saltenv' + else: + rarg = aspec.varargs - if not isinstance(varargs, list): - msg = "'{0}' must be a list." - ret['comment'] = msg.format(aspec.varargs) - ret['result'] = False - return ret + if rarg in kwargs: + varargs = kwargs.pop(rarg) - args.extend(varargs) + if not isinstance(varargs, list): + msg = "'{0}' must be a list." + ret['comment'] = msg.format(aspec.varargs) + ret['result'] = False + return ret + + args.extend(varargs) nkwargs = {} if aspec.keywords and aspec.keywords in kwargs: @@ -463,7 +495,7 @@ def _run(name, **kwargs): 'id': __opts__['id'], 'ret': mret, 'fun': name, - 'jid': salt.utils.jid.gen_jid()} + 'jid': salt.utils.jid.gen_jid(__opts__)} returners = salt.loader.returners(__opts__, __salt__) if kwargs['returner'] in returners: returners[kwargs['returner']](ret_ret) diff --git a/salt/states/mongodb_database.py b/salt/states/mongodb_database.py index 788c5c4e929..be9a66612e3 100644 --- a/salt/states/mongodb_database.py +++ b/salt/states/mongodb_database.py @@ -8,7 +8,7 @@ and can be done using mongodb_user.present from __future__ import absolute_import -import salt.utils +import salt.utils.versions def absent(name, @@ -47,7 +47,7 @@ def absent(name, 'result': True, 'comment': ''} - salt.utils.warn_until( + salt.utils.versions.warn_until( 'Fluorine', 'The \'mongodb_database.absent\' function has been deprecated and will be removed in Salt ' '{version}. Please use \'mongodb.database_absent\' instead.' diff --git a/salt/states/mongodb_user.py b/salt/states/mongodb_user.py index bfa7ae1aafe..4a65cf23b75 100644 --- a/salt/states/mongodb_user.py +++ b/salt/states/mongodb_user.py @@ -9,7 +9,7 @@ Management of Mongodb users from __future__ import absolute_import -import salt.utils +import salt.utils.versions # Define the module's virtual name __virtualname__ = 'mongodb_user' @@ -85,7 +85,7 @@ def present(name, ''' - salt.utils.warn_until( + salt.utils.versions.warn_until( 'Fluorine', 'The \'mongodb_user.present\' function has been deprecated and will be removed in Salt ' '{version}. Please use \'mongodb.user_present\' instead.' @@ -189,7 +189,7 @@ def absent(name, The database in which to authenticate ''' - salt.utils.warn_until( + salt.utils.versions.warn_until( 'Fluorine', 'The \'mongodb_user.absent\' function has been deprecated and will be removed in Salt ' '{version}. Please use \'mongodb.user_absent\' instead.' diff --git a/salt/states/mount.py b/salt/states/mount.py index 3d9abe4fd3b..dd609a2c7a4 100644 --- a/salt/states/mount.py +++ b/salt/states/mount.py @@ -45,7 +45,7 @@ import re from salt.ext.six import string_types import logging -import salt.ext.six as six +from salt.ext import six log = logging.getLogger(__name__) @@ -75,7 +75,8 @@ def mounted(name, extra_mount_invisible_keys=None, extra_mount_ignore_fs_keys=None, extra_mount_translate_options=None, - hidden_opts=None): + hidden_opts=None, + **kwargs): ''' Verify that a device is mounted @@ -196,6 +197,8 @@ def mounted(name, 'result': True, 'comment': ''} + update_mount_cache = False + if device_name_regex is None: device_name_regex = [] @@ -438,6 +441,50 @@ def mounted(name, # don't write remount into fstab if 'remount' in opts: opts.remove('remount') + + # Update the cache + update_mount_cache = True + + mount_cache = __salt__['mount.read_mount_cache'](real_name) + if 'opts' in mount_cache: + _missing = [opt for opt in mount_cache['opts'] + if opt not in opts] + + if _missing: + if __opts__['test']: + ret['result'] = None + ret['comment'] = ('Remount would be forced because' + ' options ({0})' + 'changed'.format(','.join(_missing))) + return ret + else: + # Some file systems require umounting and mounting if options change + # add others to list that require similiar functionality + if fstype in ['nfs', 'cvfs'] or fstype.startswith('fuse'): + ret['changes']['umount'] = "Forced unmount and mount because " \ + + "options ({0}) changed".format(opt) + unmount_result = __salt__['mount.umount'](real_name) + if unmount_result is True: + mount_result = __salt__['mount.mount'](real_name, device, mkmnt=mkmnt, fstype=fstype, opts=opts) + ret['result'] = mount_result + else: + ret['result'] = False + ret['comment'] = 'Unable to unmount {0}: {1}.'.format(real_name, unmount_result) + return ret + else: + ret['changes']['umount'] = "Forced remount because " \ + + "options ({0}) changed".format(opt) + remount_result = __salt__['mount.remount'](real_name, device, mkmnt=mkmnt, fstype=fstype, opts=opts) + ret['result'] = remount_result + # Cleanup after the remount, so we + # don't write remount into fstab + if 'remount' in opts: + opts.remove('remount') + + update_mount_cache = True + else: + update_mount_cache = True + if real_device not in device_list: # name matches but device doesn't - need to umount _device_mismatch_is_ignored = None @@ -468,6 +515,7 @@ def mounted(name, ret['comment'] = "Unable to unmount" ret['result'] = None return ret + update_mount_cache = True else: ret['comment'] = 'Target was already mounted' # using a duplicate check so I can catch the results of a umount @@ -491,6 +539,7 @@ def mounted(name, out = __salt__['mount.mount'](name, device, mkmnt, fstype, opts, user=user) active = __salt__['mount.active'](extended=True) + update_mount_cache = True if isinstance(out, string_types): # Failed to (re)mount, the state has failed! ret['comment'] = out @@ -590,6 +639,13 @@ def mounted(name, config, match_on=match_on) + if update_mount_cache: + cache_result = __salt__['mount.write_mount_cache'](real_name, + device, + mkmnt=mkmnt, + fstype=fstype, + opts=opts) + if out == 'present': ret['comment'] += '. Entry already exists in the fstab.' return ret @@ -698,7 +754,8 @@ def unmounted(name, device=None, config='/etc/fstab', persist=False, - user=None): + user=None, + **kwargs): ''' .. versionadded:: 0.17.0 @@ -728,6 +785,8 @@ def unmounted(name, 'result': True, 'comment': ''} + update_mount_cache = False + # Get the active data active = __salt__['mount.active'](extended=True) if name not in active: @@ -742,8 +801,10 @@ def unmounted(name, return ret if device: out = __salt__['mount.umount'](name, device, user=user) + update_mount_cache = True else: out = __salt__['mount.umount'](name, user=user) + update_mount_cache = True if isinstance(out, string_types): # Failed to umount, the state has failed! ret['comment'] = out @@ -756,6 +817,9 @@ def unmounted(name, ret['comment'] = 'Execute set to False, Target was not unmounted' ret['result'] = True + if update_mount_cache: + cache_result = __salt__['mount.delete_mount_cache'](name) + if persist: # Override default for Mac OS if __grains__['os'] in ['MacOS', 'Darwin'] and config == '/etc/fstab': diff --git a/salt/states/mssql_database.py b/salt/states/mssql_database.py new file mode 100644 index 00000000000..da29a40650c --- /dev/null +++ b/salt/states/mssql_database.py @@ -0,0 +1,96 @@ +# -*- coding: utf-8 -*- +''' +Management of Microsoft SQLServer Databases +=========================================== + +The mssql_database module is used to create +and manage SQL Server Databases + +.. code-block:: yaml + + yolo: + mssql_database.present +''' +from __future__ import absolute_import +import collections + + +def __virtual__(): + ''' + Only load if the mssql module is present + ''' + return 'mssql.version' in __salt__ + + +def _normalize_options(options): + if type(options) in [dict, collections.OrderedDict]: + return ['{0}={1}'.format(k, v) for k, v in options.items()] + if type(options) is list and (not len(options) or type(options[0]) is str): + return options + # Invalid options + if type(options) is not list or type(options[0]) not in [dict, collections.OrderedDict]: + return [] + return [o for d in options for o in _normalize_options(d)] + + +def present(name, containment='NONE', options=None, **kwargs): + ''' + Ensure that the named database is present with the specified options + + name + The name of the database to manage + containment + Defaults to NONE + options + Can be a list of strings, a dictionary, or a list of dictionaries + ''' + ret = {'name': name, + 'changes': {}, + 'result': True, + 'comment': ''} + + if __salt__['mssql.db_exists'](name, **kwargs): + ret['comment'] = 'Database {0} is already present (Not going to try to set its options)'.format(name) + return ret + if __opts__['test']: + ret['result'] = None + ret['comment'] = 'Database {0} is set to be added'.format(name) + return ret + + db_created = __salt__['mssql.db_create'](name, containment=containment, new_database_options=_normalize_options(options), **kwargs) + if db_created is not True: # Non-empty strings are also evaluated to True, so we cannot use if not db_created: + ret['result'] = False + ret['comment'] += 'Database {0} failed to be created: {1}'.format(name, db_created) + return ret + ret['comment'] += 'Database {0} has been added'.format(name) + ret['changes'][name] = 'Present' + return ret + + +def absent(name, **kwargs): + ''' + Ensure that the named database is absent + + name + The name of the database to remove + ''' + ret = {'name': name, + 'changes': {}, + 'result': True, + 'comment': ''} + + if not __salt__['mssql.db_exists'](name): + ret['comment'] = 'Database {0} is not present'.format(name) + return ret + if __opts__['test']: + ret['result'] = None + ret['comment'] = 'Database {0} is set to be removed'.format(name) + return ret + if __salt__['mssql.db_remove'](name, **kwargs): + ret['comment'] = 'Database {0} has been removed'.format(name) + ret['changes'][name] = 'Absent' + return ret + # else: + ret['result'] = False + ret['comment'] = 'Database {0} failed to be removed'.format(name) + return ret diff --git a/salt/states/mssql_login.py b/salt/states/mssql_login.py new file mode 100644 index 00000000000..14d0ed6d0a6 --- /dev/null +++ b/salt/states/mssql_login.py @@ -0,0 +1,116 @@ +# -*- coding: utf-8 -*- +''' +Management of Microsoft SQLServer Logins +======================================== + +The mssql_login module is used to create +and manage SQL Server Logins + +.. code-block:: yaml + + frank: + mssql_login.present + - domain: mydomain +''' +from __future__ import absolute_import +import collections + + +def __virtual__(): + ''' + Only load if the mssql module is present + ''' + return 'mssql.version' in __salt__ + + +def _normalize_options(options): + if type(options) in [dict, collections.OrderedDict]: + return ['{0}={1}'.format(k, v) for k, v in options.items()] + if type(options) is list and (not len(options) or type(options[0]) is str): + return options + # Invalid options + if type(options) is not list or type(options[0]) not in [dict, collections.OrderedDict]: + return [] + return [o for d in options for o in _normalize_options(d)] + + +def present(name, password=None, domain=None, server_roles=None, options=None, **kwargs): + ''' + Checks existance of the named login. + If not present, creates the login with the specified roles and options. + + name + The name of the login to manage + password + Creates a SQL Server authentication login + Since hashed passwords are varbinary values, if the + new_login_password is 'long', it will be considered + to be HASHED. + domain + Creates a Windows authentication login. + Needs to be NetBIOS domain or hostname + server_roles + Add this login to all the server roles in the list + options + Can be a list of strings, a dictionary, or a list of dictionaries + ''' + ret = {'name': name, + 'changes': {}, + 'result': True, + 'comment': ''} + + if bool(password) == bool(domain): + ret['result'] = False + ret['comment'] = 'One and only one of password and domain should be specifies' + return ret + if __salt__['mssql.login_exists'](name, domain=domain, **kwargs): + ret['comment'] = 'Login {0} is already present (Not going to try to set its password)'.format(name) + return ret + if __opts__['test']: + ret['result'] = None + ret['comment'] = 'Login {0} is set to be added'.format(name) + return ret + + login_created = __salt__['mssql.login_create'](name, + new_login_password=password, + new_login_domain=domain, + new_login_roles=server_roles, + new_login_options=_normalize_options(options), + **kwargs) + # Non-empty strings are also evaluated to True, so we cannot use if not login_created: + if login_created is not True: + ret['result'] = False + ret['comment'] = 'Login {0} failed to be added: {1}'.format(name, login_created) + return ret + ret['comment'] = 'Login {0} has been added. '.format(name) + ret['changes'][name] = 'Present' + return ret + + +def absent(name, **kwargs): + ''' + Ensure that the named login is absent + + name + The name of the login to remove + ''' + ret = {'name': name, + 'changes': {}, + 'result': True, + 'comment': ''} + + if not __salt__['mssql.login_exists'](name): + ret['comment'] = 'Login {0} is not present'.format(name) + return ret + if __opts__['test']: + ret['result'] = None + ret['comment'] = 'Login {0} is set to be removed'.format(name) + return ret + if __salt__['mssql.login_remove'](name, **kwargs): + ret['comment'] = 'Login {0} has been removed'.format(name) + ret['changes'][name] = 'Absent' + return ret + # else: + ret['result'] = False + ret['comment'] = 'Login {0} failed to be removed'.format(name) + return ret diff --git a/salt/states/mssql_role.py b/salt/states/mssql_role.py new file mode 100644 index 00000000000..b515bb90683 --- /dev/null +++ b/salt/states/mssql_role.py @@ -0,0 +1,84 @@ +# -*- coding: utf-8 -*- +''' +Management of Microsoft SQLServer Databases +=========================================== + +The mssql_role module is used to create +and manage SQL Server Roles + +.. code-block:: yaml + + yolo: + mssql_role.present +''' +from __future__ import absolute_import + + +def __virtual__(): + ''' + Only load if the mssql module is present + ''' + return 'mssql.version' in __salt__ + + +def present(name, owner=None, grants=None, **kwargs): + ''' + Ensure that the named database is present with the specified options + + name + The name of the database to manage + owner + Adds owner using AUTHORIZATION option + Grants + Can only be a list of strings + ''' + ret = {'name': name, + 'changes': {}, + 'result': True, + 'comment': ''} + + if __salt__['mssql.role_exists'](name, **kwargs): + ret['comment'] = 'Role {0} is already present (Not going to try to set its grants)'.format(name) + return ret + if __opts__['test']: + ret['result'] = None + ret['comment'] = 'Role {0} is set to be added'.format(name) + return ret + + role_created = __salt__['mssql.role_create'](name, owner=owner, grants=grants, **kwargs) + if role_created is not True: # Non-empty strings are also evaluated to True, so we cannot use if not role_created: + ret['result'] = False + ret['comment'] += 'Role {0} failed to be created: {1}'.format(name, role_created) + return ret + ret['comment'] += 'Role {0} has been added'.format(name) + ret['changes'][name] = 'Present' + return ret + + +def absent(name, **kwargs): + ''' + Ensure that the named database is absent + + name + The name of the database to remove + ''' + ret = {'name': name, + 'changes': {}, + 'result': True, + 'comment': ''} + + if not __salt__['mssql.role_exists'](name): + ret['comment'] = 'Role {0} is not present'.format(name) + return ret + if __opts__['test']: + ret['result'] = None + ret['comment'] = 'Role {0} is set to be removed'.format(name) + return ret + if __salt__['mssql.role_remove'](name, **kwargs): + ret['comment'] = 'Role {0} has been removed'.format(name) + ret['changes'][name] = 'Absent' + return ret + # else: + ret['result'] = False + ret['comment'] = 'Role {0} failed to be removed'.format(name) + return ret diff --git a/salt/states/mssql_user.py b/salt/states/mssql_user.py new file mode 100644 index 00000000000..511293be2bf --- /dev/null +++ b/salt/states/mssql_user.py @@ -0,0 +1,114 @@ +# -*- coding: utf-8 -*- +''' +Management of Microsoft SQLServer Users +======================================= + +The mssql_user module is used to create +and manage SQL Server Users + +.. code-block:: yaml + + frank: + mssql_user.present: + - database: yolo +''' +from __future__ import absolute_import +import collections + + +def __virtual__(): + ''' + Only load if the mssql module is present + ''' + return 'mssql.version' in __salt__ + + +def _normalize_options(options): + if type(options) in [dict, collections.OrderedDict]: + return ['{0}={1}'.format(k, v) for k, v in options.items()] + if type(options) is list and (not len(options) or type(options[0]) is str): + return options + # Invalid options + if type(options) is not list or type(options[0]) not in [dict, collections.OrderedDict]: + return [] + return [o for d in options for o in _normalize_options(d)] + + +def present(name, login=None, domain=None, database=None, roles=None, options=None, **kwargs): + ''' + Checks existance of the named user. + If not present, creates the user with the specified roles and options. + + name + The name of the user to manage + login + If not specified, will be created WITHOUT LOGIN + domain + Creates a Windows authentication user. + Needs to be NetBIOS domain or hostname + database + The database of the user (not the login) + roles + Add this user to all the roles in the list + options + Can be a list of strings, a dictionary, or a list of dictionaries + ''' + ret = {'name': name, + 'changes': {}, + 'result': True, + 'comment': ''} + + if domain and not login: + ret['result'] = False + ret['comment'] = 'domain cannot be set without login' + return ret + if __salt__['mssql.user_exists'](name, domain=domain, database=database, **kwargs): + ret['comment'] = 'User {0} is already present (Not going to try to set its roles or options)'.format(name) + return ret + if __opts__['test']: + ret['result'] = None + ret['comment'] = 'User {0} is set to be added'.format(name) + return ret + + user_created = __salt__['mssql.user_create'](name, login=login, + domain=domain, + database=database, + roles=roles, + options=_normalize_options(options), + **kwargs) + if user_created is not True: # Non-empty strings are also evaluated to True, so we cannot use if not user_created: + ret['result'] = False + ret['comment'] += 'User {0} failed to be added: {1}'.format(name, user_created) + return ret + ret['comment'] += 'User {0} has been added'.format(name) + ret['changes'][name] = 'Present' + return ret + + +def absent(name, **kwargs): + ''' + Ensure that the named user is absent + + name + The username of the user to remove + ''' + ret = {'name': name, + 'changes': {}, + 'result': True, + 'comment': ''} + + if not __salt__['mssql.user_exists'](name): + ret['comment'] = 'User {0} is not present'.format(name) + return ret + if __opts__['test']: + ret['result'] = None + ret['comment'] = 'User {0} is set to be removed'.format(name) + return ret + if __salt__['mssql.user_remove'](name, **kwargs): + ret['comment'] = 'User {0} has been removed'.format(name) + ret['changes'][name] = 'Absent' + return ret + # else: + ret['result'] = False + ret['comment'] = 'User {0} failed to be removed'.format(name) + return ret diff --git a/salt/states/mysql_query.py b/salt/states/mysql_query.py index b2ea6c1c3cd..e5a1a046b4c 100644 --- a/salt/states/mysql_query.py +++ b/salt/states/mysql_query.py @@ -26,10 +26,10 @@ import sys import os.path # Import Salt libs -import salt.utils +import salt.utils.files # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six def __virtual__(): @@ -189,7 +189,7 @@ def run_file(name, + grain + ":" + key elif output is not None: ret['changes']['query'] = "Executed. Output into " + output - with salt.utils.fopen(output, 'w') as output_file: + with salt.utils.files.fopen(output, 'w') as output_file: if 'results' in query_result: for res in query_result['results']: for col, val in six.iteritems(res): @@ -329,7 +329,7 @@ def run(name, + grain + ":" + key elif output is not None: ret['changes']['query'] = "Executed. Output into " + output - with salt.utils.fopen(output, 'w') as output_file: + with salt.utils.files.fopen(output, 'w') as output_file: if 'results' in query_result: for res in query_result['results']: for col, val in six.iteritems(res): diff --git a/salt/states/netconfig.py b/salt/states/netconfig.py index 9318e227b41..7311baaf1f2 100644 --- a/salt/states/netconfig.py +++ b/salt/states/netconfig.py @@ -60,6 +60,7 @@ def _update_config(template_name, template_user='root', template_group='root', template_mode='755', + template_attrs='--------------e----', saltenv=None, template_engine='jinja', skip_verify=False, @@ -83,6 +84,7 @@ def _update_config(template_name, template_user=template_user, template_group=template_group, template_mode=template_mode, + template_attrs=template_attrs, saltenv=saltenv, template_engine=template_engine, skip_verify=skip_verify, @@ -107,9 +109,10 @@ def managed(name, template_user='root', template_group='root', template_mode='755', + template_attrs='--------------e----', saltenv=None, template_engine='jinja', - skip_verify=True, + skip_verify=False, defaults=None, test=False, commit=True, @@ -178,9 +181,14 @@ def managed(name, template_user: root Group owner of file. - template_user: 755 + template_mode: 755 Permissions of file + template_attrs: "--------------e----" + Attributes of file (see `man lsattr`) + + .. versionadded:: oxygen + saltenv: base Specifies the template environment. This will influence the relative imports inside the templates. @@ -194,10 +202,12 @@ def managed(name, - :mod:`py` - :mod:`wempy` - skip_verify: True + skip_verify: False If ``True``, hash verification of remote file sources (``http://``, ``https://``, ``ftp://``) will be skipped, and the ``source_hash`` argument will be ignored. + .. versionchanged:: 2017.7.1 + test: False Dry run? If set to ``True``, will apply the config, discard and return the changes. Default: ``False`` (will commit the changes on the device). @@ -337,6 +347,7 @@ def managed(name, template_user=template_user, template_group=template_group, template_mode=template_mode, + template_attrs=template_attrs, saltenv=saltenv, template_engine=template_engine, skip_verify=skip_verify, diff --git a/salt/states/netntp.py b/salt/states/netntp.py index a0d8c851caa..06cbdb66fa7 100644 --- a/salt/states/netntp.py +++ b/salt/states/netntp.py @@ -28,7 +28,9 @@ Dependencies from __future__ import absolute_import import logging -log = logging.getLogger(__name__) + +# Import 3rd-party libs +from salt.ext import six # import NAPALM utils import salt.utils.napalm @@ -52,6 +54,8 @@ except ImportError: __virtualname__ = 'netntp' +log = logging.getLogger(__name__) + # ---------------------------------------------------------------------------------------------------------------------- # global variables # ---------------------------------------------------------------------------------------------------------------------- @@ -105,7 +109,7 @@ def _check(peers): return False for peer in peers: - if not isinstance(peer, str): + if not isinstance(peer, six.string_types): return False if not HAS_NETADDR: # if does not have this lib installed, will simply try to load what user specified diff --git a/salt/states/network.py b/salt/states/network.py index 1411ea64533..4eb17f59f96 100644 --- a/salt/states/network.py +++ b/salt/states/network.py @@ -255,10 +255,12 @@ all interfaces are ignored unless specified. ''' from __future__ import absolute_import -# Import python libs +# Import Python libs import difflib -import salt.utils + +# Import Salt libs import salt.utils.network +import salt.utils.platform import salt.loader # Set up logging @@ -271,7 +273,7 @@ def __virtual__(): Confine this module to non-Windows systems with the required execution module available. ''' - if not salt.utils.is_windows() and 'ip.get_interface' in __salt__: + if not salt.utils.platform.is_windows() and 'ip.get_interface' in __salt__: return True return False diff --git a/salt/states/netyang.py b/salt/states/netyang.py index e39b09b3b97..71700090fcd 100644 --- a/salt/states/netyang.py +++ b/salt/states/netyang.py @@ -38,7 +38,7 @@ except ImportError: HAS_NAPALM_YANG = False # Import salt modules -from salt.utils import fopen +import salt.utils.files import salt.utils.napalm # ------------------------------------------------------------------------------ @@ -92,6 +92,13 @@ def managed(name, Use certain profiles to generate the config. If not specified, will use the platform default profile(s). + compliance_report: ``False`` + Return the compliance report in the comment. + The compliance report structured object can be found however + in the ``pchanges`` field of the output (not displayed on the CLI). + + .. versionadded:: 2017.7.3 + test: ``False`` Dry run? If set as ``True``, will apply the config, discard and return the changes. Default: ``False`` and will commit @@ -140,13 +147,14 @@ def managed(name, debug = kwargs.get('debug', False) or __opts__.get('debug', False) commit = kwargs.get('commit', True) or __opts__.get('commit', True) replace = kwargs.get('replace', False) or __opts__.get('replace', False) + return_compliance_report = kwargs.get('compliance_report', False) or __opts__.get('compliance_report', False) profiles = kwargs.get('profiles', []) temp_file = __salt__['temp.file']() log.debug('Creating temp file: {0}'.format(temp_file)) if 'to_dict' not in data: data = {'to_dict': data} data = [data] - with fopen(temp_file, 'w') as file_handle: + with salt.utils.files.fopen(temp_file, 'w') as file_handle: yaml.safe_dump(json.loads(json.dumps(data)), file_handle, encoding='utf-8', allow_unicode=True) device_config = __salt__['napalm_yang.parse'](models, config=True, @@ -180,7 +188,13 @@ def managed(name, log.debug('Loaded config result:') log.debug(loaded_changes) __salt__['file.remove'](temp_file) - return salt.utils.napalm.loaded_ret(ret, loaded_changes, test, debug) + loaded_changes['compliance_report'] = compliance_report + return salt.utils.napalm.loaded_ret(ret, + loaded_changes, + test, + debug, + opts=__opts__, + compliance_report=return_compliance_report) def configured(name, diff --git a/salt/states/nfs_export.py b/salt/states/nfs_export.py new file mode 100644 index 00000000000..7f4f488b0aa --- /dev/null +++ b/salt/states/nfs_export.py @@ -0,0 +1,218 @@ +# -*- coding: utf-8 -*- +''' +Management of NFS exports +=============================================== + +.. versionadded:: Oxygen + +To ensure an NFS export exists: + +.. code-block:: yaml + + add_simple_export: + nfs_export.present: + - name: '/srv/nfs' + - hosts: '10.0.2.0/24' + - options: + - 'rw' + +This creates the following in /etc/exports: + +.. code-block:: bash + + /srv/nfs 10.0.2.0/24(rw) + +For more complex exports with multiple groups of hosts, use 'clients': + +.. code-block:: yaml + + add_complex_export: + nfs_export.present: + - name: '/srv/nfs' + - clients: + # First export, same as simple one above + - hosts: '10.0.2.0/24' + options: + - 'rw' + # Second export + - hosts: '*.example.com' + options: + - 'ro' + - 'subtree_check' + +This creates the following in /etc/exports: + +.. code-block:: bash + + /srv/nfs 10.0.2.0/24(rw) 192.168.0.0/24,172.19.0.0/16(ro,subtree_check) + +Any export of the given path will be modified to match the one specified. + +To ensure an NFS export is absent: + +.. code-block:: yaml + + delete_export: + nfs_export.absent: + - name: '/srv/nfs' + +''' +from __future__ import absolute_import +import salt.utils.path + + +def __virtual__(): + ''' + Only work with nfs tools installed + ''' + cmd = 'exportfs' + if salt.utils.path.which(cmd): + return bool(cmd) + + return( + False, + 'The nfs_exports state module failed to load: ' + 'the exportfs binary is not in the path' + ) + + +def present(name, + clients=None, + hosts=None, + options=None, + exports='/etc/exports'): + ''' + Ensure that the named export is present with the given options + + name + The export path to configure + + clients + A list of hosts and the options applied to them. + This option may not be used in combination with + the 'hosts' or 'options' shortcuts. + + .. code-block:: yaml + + - clients: + # First export + - hosts: '10.0.2.0/24' + options: + - 'rw' + # Second export + - hosts: '*.example.com' + options: + - 'ro' + - 'subtree_check' + + hosts + A string matching a number of hosts, for example: + + .. code-block:: yaml + + hosts: '10.0.2.123' + + hosts: '10.0.2.0/24' + + hosts: 'minion1.example.com' + + hosts: '*.example.com' + + hosts: '*' + + options + A list of NFS options, for example: + + .. code-block:: yaml + + options: + - 'rw' + - 'subtree_check' + + ''' + path = name + ret = {'name': name, + 'changes': {}, + 'result': None, + 'comment': ''} + + if not clients: + if not hosts: + ret['result'] = False + ret['comment'] = 'Either \'clients\' or \'hosts\' must be defined' + return ret + # options being None is handled by add_export() + clients = [{'hosts': hosts, 'options': options}] + + old = __salt__['nfs3.list_exports'](exports) + if path in old: + if old[path] == clients: + ret['result'] = True + ret['comment'] = 'Export {0} already configured'.format(path) + return ret + + ret['changes']['new'] = clients + ret['changes']['old'] = old[path] + if __opts__['test']: + ret['result'] = None + ret['comment'] = 'Export {0} would be changed'.format(path) + return ret + + __salt__['nfs3.del_export'](exports, path) + + else: + ret['changes']['old'] = None + ret['changes']['new'] = clients + if __opts__['test']: + ret['result'] = None + ret['comment'] = 'Export {0} would be added'.format(path) + return ret + + add_export = __salt__['nfs3.add_export'] + for exp in clients: + add_export(exports, path, exp['hosts'], exp['options']) + + ret['changes']['new'] = clients + + try_reload = __salt__['nfs3.reload_exports']() + ret['comment'] = try_reload['stderr'] + ret['result'] = try_reload['result'] + return ret + + +def absent(name, exports='/etc/exports'): + ''' + Ensure that the named path is not exported + + name + The export path to remove + ''' + + path = name + ret = {'name': name, + 'changes': {}, + 'result': None, + 'comment': ''} + + old = __salt__['nfs3.list_exports'](exports) + if path in old: + if __opts__['test']: + ret['comment'] = 'Export {0} would be removed'.format(path) + ret['changes'][path] = old[path] + ret['result'] = None + return ret + + __salt__['nfs3.del_export'](exports, path) + try_reload = __salt__['nfs3.reload_exports']() + if not try_reload['result']: + ret['comment'] = try_reload['stderr'] + else: + ret['comment'] = 'Export {0} removed'.format(path) + + ret['result'] = try_reload['result'] + ret['changes'][path] = old[path] + else: + ret['comment'] = 'Export {0} already absent'.format(path) + ret['result'] = True + + return ret diff --git a/salt/states/npm.py b/salt/states/npm.py index 4c58d3ae03b..a323a5ae430 100644 --- a/salt/states/npm.py +++ b/salt/states/npm.py @@ -24,7 +24,7 @@ from __future__ import absolute_import from salt.exceptions import CommandExecutionError, CommandNotFoundError # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six def __virtual__(): @@ -296,7 +296,7 @@ def bootstrap(name, user=None, silent=True): return ret # npm.install will return a string if it can't parse a JSON result - if isinstance(call, str): + if isinstance(call, six.string_types): ret['result'] = False ret['changes'] = call ret['comment'] = 'Could not bootstrap directory' diff --git a/salt/states/ntp.py b/salt/states/ntp.py index dbc15838833..58f235911ab 100644 --- a/salt/states/ntp.py +++ b/salt/states/ntp.py @@ -17,12 +17,14 @@ This state is used to manage NTP servers. Currently only Windows is supported. ''' from __future__ import absolute_import -# Import python libs +# Import Python libs import logging -# Import salt libs -from salt.ext.six import string_types -import salt.utils +# Import Salt libs +import salt.utils.platform + +# Import 3rd-party libs +from salt.ext import six log = logging.getLogger(__name__) @@ -32,7 +34,7 @@ def __virtual__(): ''' This only supports Windows ''' - if not salt.utils.is_windows(): + if not salt.utils.platform.is_windows(): return False return 'ntp' @@ -41,7 +43,7 @@ def _check_servers(servers): if not isinstance(servers, list): return False for server in servers: - if not isinstance(server, string_types): + if not isinstance(server, six.string_types): return False return True diff --git a/salt/states/panos.py b/salt/states/panos.py new file mode 100644 index 00000000000..f941ff157ad --- /dev/null +++ b/salt/states/panos.py @@ -0,0 +1,1069 @@ +# -*- coding: utf-8 -*- +''' +A state module to manage Palo Alto network devices. + +:codeauthor: :email:`Spencer Ervin ` +:maturity: new +:depends: none +:platform: unix + + +About +===== +This state module was designed to handle connections to a Palo Alto based +firewall. This module relies on the Palo Alto proxy module to interface with the devices. + +This state module is designed to give extreme flexibility in the control over XPATH values on the PANOS device. It +exposes the core XML API commands and allows state modules to chain complex XPATH commands. + +Below is an example of how to construct a security rule and move to the top of the policy. This will take a config +lock to prevent execution during the operation, then remove the lock. After the XPATH has been deployed, it will +commit to the device. + +.. code-block:: yaml + + panos/takelock: + panos.add_config_lock + panos/service_tcp_22: + panos.set_config: + - xpath: /config/devices/entry[@name='localhost.localdomain']/vsys/entry[@name='vsys1']/service + - value: 22 + - commit: False + panos/create_rule1: + panos.set_config: + - xpath: /config/devices/entry[@name='localhost.localdomain']/vsys/entry[@name='vsys1']/rulebase/security/rules + - value: ' + + trust + untrust + 10.0.0.1 + 10.0.1.1 + tcp-22 + any + allow + no + ' + - commit: False + panos/moveruletop: + panos.move_config: + - xpath: /config/devices/entry[@name='localhost.localdomain']/vsys/entry[@name='vsys1']/rulebase/security/rules/entry[@name='rule1'] + - where: top + - commit: False + panos/removelock: + panos.remove_config_lock + panos/commit: + panos.commit + +Version Specific Configurations +=============================== +Palo Alto devices running different versions will have different supported features and different command structures. In +order to account for this, the proxy module can be leveraged to check if the panos device is at a specific revision +level. + +The proxy['panos.is_required_version'] method will check if a panos device is currently running a version equal or +greater than the passed version. For example, proxy['panos.is_required_version']('7.0.0') would match both 7.1.0 and +8.0.0. + +.. code-block:: yaml + + {% if proxy['panos.is_required_version']('8.0.0') %} + panos/deviceconfig/system/motd-and-banner: + panos.set_config: + - xpath: /config/devices/entry[@name='localhost.localdomain']/deviceconfig/system/motd-and-banner + - value: | + BANNER TEXT + color2 + color18 + yes + - commit: False + {% endif %} + +.. seealso:: + :prox:`Palo Alto Proxy Module ` + +''' + +# Import Python Libs +from __future__ import absolute_import +import logging + +log = logging.getLogger(__name__) + + +def __virtual__(): + return 'panos.commit' in __salt__ + + +def _build_members(members, anycheck=False): + ''' + Builds a member formatted string for XML operation. + + ''' + if isinstance(members, list): + + # This check will strip down members to a single any statement + if anycheck and 'any' in members: + return "any" + response = "" + for m in members: + response += "{0}".format(m) + return response + else: + return "{0}".format(members) + + +def _default_ret(name): + ''' + Set the default response values. + + ''' + ret = { + 'name': name, + 'changes': {}, + 'commit': None, + 'result': False, + 'comment': '' + } + return ret + + +def _edit_config(xpath, element): + ''' + Sends an edit request to the device. + + ''' + query = {'type': 'config', + 'action': 'edit', + 'xpath': xpath, + 'element': element} + + response = __proxy__['panos.call'](query) + + return _validate_response(response) + + +def _get_config(xpath): + ''' + Retrieves an xpath from the device. + + ''' + query = {'type': 'config', + 'action': 'get', + 'xpath': xpath} + + response = __proxy__['panos.call'](query) + + return response + + +def _move_after(xpath, target): + ''' + Moves an xpath to the after of its section. + + ''' + query = {'type': 'config', + 'action': 'move', + 'xpath': xpath, + 'where': 'after', + 'dst': target} + + response = __proxy__['panos.call'](query) + + return _validate_response(response) + + +def _move_before(xpath, target): + ''' + Moves an xpath to the bottom of its section. + + ''' + query = {'type': 'config', + 'action': 'move', + 'xpath': xpath, + 'where': 'before', + 'dst': target} + + response = __proxy__['panos.call'](query) + + return _validate_response(response) + + +def _move_bottom(xpath): + ''' + Moves an xpath to the bottom of its section. + + ''' + query = {'type': 'config', + 'action': 'move', + 'xpath': xpath, + 'where': 'bottom'} + + response = __proxy__['panos.call'](query) + + return _validate_response(response) + + +def _move_top(xpath): + ''' + Moves an xpath to the top of its section. + + ''' + query = {'type': 'config', + 'action': 'move', + 'xpath': xpath, + 'where': 'top'} + + response = __proxy__['panos.call'](query) + + return _validate_response(response) + + +def _set_config(xpath, element): + ''' + Sends a set request to the device. + + ''' + query = {'type': 'config', + 'action': 'set', + 'xpath': xpath, + 'element': element} + + response = __proxy__['panos.call'](query) + + return _validate_response(response) + + +def _validate_response(response): + ''' + Validates a response from a Palo Alto device. Used to verify success of commands. + + ''' + if not response: + return False, "Error during move configuration. Verify connectivity to device." + elif 'msg' in response: + if response['msg'] == 'command succeeded': + return True, response['msg'] + else: + return False, response['msg'] + elif 'line' in response: + if response['line'] == 'already at the top': + return True, response['line'] + elif response['line'] == 'already at the bottom': + return True, response['line'] + else: + return False, response['line'] + else: + return False, "Error during move configuration. Verify connectivity to device." + + +def add_config_lock(name): + ''' + Prevent other users from changing configuration until the lock is released. + + name: The name of the module function to execute. + + SLS Example: + + .. code-block:: yaml + + panos/takelock: + panos.add_config_lock + + ''' + ret = _default_ret(name) + + ret.update({ + 'changes': __salt__['panos.add_config_lock'](), + 'result': True + }) + + return ret + + +def clone_config(name, xpath=None, newname=None, commit=False): + ''' + Clone a specific XPATH and set it to a new name. + + name: The name of the module function to execute. + + xpath(str): The XPATH of the configuration API tree to clone. + + newname(str): The new name of the XPATH clone. + + commit(bool): If true the firewall will commit the changes, if false do not commit changes. + + SLS Example: + + .. code-block:: yaml + + panos/clonerule: + panos.clone_config: + - xpath: /config/devices/entry/vsys/entry[@name='vsys1']/rulebase/security/rules&from=/config/devices/ + entry/vsys/entry[@name='vsys1']/rulebase/security/rules/entry[@name='rule1'] + - value: rule2 + - commit: True + + ''' + ret = _default_ret(name) + + if not xpath: + return ret + + if not newname: + return ret + + query = {'type': 'config', + 'action': 'clone', + 'xpath': xpath, + 'newname': newname} + + response = __proxy__['panos.call'](query) + + ret.update({ + 'changes': response, + 'result': True + }) + + if commit is True: + ret.update({ + 'commit': __salt__['panos.commit'](), + 'result': True + }) + + return ret + + +def commit_config(name): + ''' + Commits the candidate configuration to the running configuration. + + name: The name of the module function to execute. + + SLS Example: + + .. code-block:: yaml + + panos/commit: + panos.commit_config + + ''' + ret = _default_ret(name) + + ret.update({ + 'commit': __salt__['panos.commit'](), + 'result': True + }) + + return ret + + +def delete_config(name, xpath=None, commit=False): + ''' + Deletes a Palo Alto XPATH to a specific value. + + Use the xpath parameter to specify the location of the object to be deleted. + + name: The name of the module function to execute. + + xpath(str): The XPATH of the configuration API tree to control. + + commit(bool): If true the firewall will commit the changes, if false do not commit changes. + + SLS Example: + + .. code-block:: yaml + + panos/deletegroup: + panos.delete_config: + - xpath: /config/devices/entry/vsys/entry[@name='vsys1']/address-group/entry[@name='test'] + - commit: True + + ''' + ret = _default_ret(name) + + if not xpath: + return ret + + query = {'type': 'config', + 'action': 'delete', + 'xpath': xpath} + + response = __proxy__['panos.call'](query) + + ret.update({ + 'changes': response, + 'result': True + }) + + if commit is True: + ret.update({ + 'commit': __salt__['panos.commit'](), + 'result': True + }) + + return ret + + +def download_software(name, version=None, synch=False, check=False): + ''' + Ensures that a software version is downloaded. + + name: The name of the module function to execute. + + version(str): The software version to check. If this version is not already downloaded, it will attempt to download + the file from Palo Alto. + + synch(bool): If true, after downloading the file it will be synched to its peer. + + check(bool): If true, the PANOS device will first attempt to pull the most recent software inventory list from Palo + Alto. + + SLS Example: + + .. code-block:: yaml + + panos/version8.0.0: + panos.download_software: + - version: 8.0.0 + - synch: False + - check: True + + ''' + ret = _default_ret(name) + + if check is True: + __salt__['panos.check_software']() + + versions = __salt__['panos.get_software_info']() + + if 'sw-updates' not in versions \ + or 'versions' not in versions['sw-updates'] \ + or 'entry' not in versions['sw-updates']['versions']: + ret.update({ + 'comment': 'Software version is not found in the local software list.', + 'result': False + }) + return ret + + for entry in versions['sw-updates']['versions']['entry']: + if entry['version'] == version and entry['downloaded'] == "yes": + ret.update({ + 'comment': 'Software version is already downloaded.', + 'result': True + }) + return ret + + ret.update({ + 'changes': __salt__['panos.download_software_version'](version=version, synch=synch) + }) + + versions = __salt__['panos.get_software_info']() + + if 'sw-updates' not in versions \ + or 'versions' not in versions['sw-updates'] \ + or 'entry' not in versions['sw-updates']['versions']: + ret.update({ + 'result': False + }) + return ret + + for entry in versions['sw-updates']['versions']['entry']: + if entry['version'] == version and entry['downloaded'] == "yes": + ret.update({ + 'result': True + }) + return ret + + return ret + + +def edit_config(name, xpath=None, value=None, commit=False): + ''' + Edits a Palo Alto XPATH to a specific value. This will always overwrite the existing value, even if it is not + changed. + + You can replace an existing object hierarchy at a specified location in the configuration with a new value. Use + the xpath parameter to specify the location of the object, including the node to be replaced. + + This is the recommended state to enforce configurations on a xpath. + + name: The name of the module function to execute. + + xpath(str): The XPATH of the configuration API tree to control. + + value(str): The XML value to edit. This must be a child to the XPATH. + + commit(bool): If true the firewall will commit the changes, if false do not commit changes. + + SLS Example: + + .. code-block:: yaml + + panos/addressgroup: + panos.edit_config: + - xpath: /config/devices/entry/vsys/entry[@name='vsys1']/address-group/entry[@name='test'] + - value: abcxyz + - commit: True + + ''' + ret = _default_ret(name) + + result, msg = _edit_config(xpath, value) + + ret.update({ + 'comment': msg, + 'result': result + }) + + # Ensure we do not commit after a failed action + if not result: + return ret + + if commit is True: + ret.update({ + 'commit': __salt__['panos.commit'](), + 'result': True + }) + + return ret + + +def move_config(name, xpath=None, where=None, dst=None, commit=False): + ''' + Moves a XPATH value to a new location. + + Use the xpath parameter to specify the location of the object to be moved, the where parameter to + specify type of move, and dst parameter to specify the destination path. + + name: The name of the module function to execute. + + xpath(str): The XPATH of the configuration API tree to move. + + where(str): The type of move to execute. Valid options are after, before, top, bottom. The after and before + options will require the dst option to specify the destination of the action. The top action will move the + XPATH to the top of its structure. The botoom action will move the XPATH to the bottom of its structure. + + dst(str): Optional. Specifies the destination to utilize for a move action. This is ignored for the top + or bottom action. + + commit(bool): If true the firewall will commit the changes, if false do not commit changes. If the operation is + not successful, it will not commit. + + SLS Example: + + .. code-block:: yaml + + panos/moveruletop: + panos.move_config: + - xpath: /config/devices/entry/vsys/entry[@name='vsys1']/rulebase/security/rules/entry[@name='rule1'] + - where: top + - commit: True + + panos/moveruleafter: + panos.move_config: + - xpath: /config/devices/entry/vsys/entry[@name='vsys1']/rulebase/security/rules/entry[@name='rule1'] + - where: after + - dst: rule2 + - commit: True + + ''' + ret = _default_ret(name) + + if not xpath: + return ret + + if not where: + return ret + + if where == 'after': + result, msg = _move_after(xpath, dst) + elif where == 'before': + result, msg = _move_before(xpath, dst) + elif where == 'top': + result, msg = _move_top(xpath) + elif where == 'bottom': + result, msg = _move_bottom(xpath) + + ret.update({ + 'result': result + }) + + if not result: + return ret + + if commit is True: + ret.update({ + 'commit': __salt__['panos.commit'](), + 'result': True + }) + + return ret + + +def remove_config_lock(name): + ''' + Release config lock previously held. + + name: The name of the module function to execute. + + SLS Example: + + .. code-block:: yaml + + panos/takelock: + panos.remove_config_lock + + ''' + ret = _default_ret(name) + + ret.update({ + 'changes': __salt__['panos.remove_config_lock'](), + 'result': True + }) + + return ret + + +def rename_config(name, xpath=None, newname=None, commit=False): + ''' + Rename a Palo Alto XPATH to a specific value. This will always rename the value even if a change is not needed. + + name: The name of the module function to execute. + + xpath(str): The XPATH of the configuration API tree to control. + + newname(str): The new name of the XPATH value. + + commit(bool): If true the firewall will commit the changes, if false do not commit changes. + + SLS Example: + + .. code-block:: yaml + + panos/renamegroup: + panos.rename_config: + - xpath: /config/devices/entry/vsys/entry[@name='vsys1']/address/entry[@name='old_address'] + - value: new_address + - commit: True + + ''' + ret = _default_ret(name) + + if not xpath: + return ret + + if not newname: + return ret + + query = {'type': 'config', + 'action': 'rename', + 'xpath': xpath, + 'newname': newname} + + response = __proxy__['panos.call'](query) + + ret.update({ + 'changes': response, + 'result': True + }) + + if commit is True: + ret.update({ + 'commit': __salt__['panos.commit'](), + 'result': True + }) + + return ret + + +def security_rule_exists(name, + rulename=None, + vsys='1', + action=None, + disabled=None, + sourcezone=None, + destinationzone=None, + source=None, + destination=None, + application=None, + service=None, + description=None, + logsetting=None, + logstart=None, + logend=None, + negatesource=None, + negatedestination=None, + profilegroup=None, + datafilter=None, + fileblock=None, + spyware=None, + urlfilter=None, + virus=None, + vulnerability=None, + wildfire=None, + move=None, + movetarget=None, + commit=False): + ''' + Ensures that a security rule exists on the device. Also, ensure that all configurations are set appropriately. + + This method will create the rule if it does not exist. If the rule does exist, it will ensure that the + configurations are set appropriately. + + If the rule does not exist and is created, any value that is not provided will be provided as the default. + The action, to, from, source, destination, application, and service fields are mandatory and must be provided. + + This will enforce the exact match of the rule. For example, if the rule is currently configured with the log-end + option, but this option is not specified in the state method, it will be removed and reset to the system default. + + It is strongly recommended to specify all options to ensure proper operation. + + When defining the profile group settings, the device can only support either a profile group or individual settings. + If both are specified, the profile group will be preferred and the individual settings are ignored. If neither are + specified, the value will be set to system default of none. + + name: The name of the module function to execute. + + rulename(str): The name of the security rule. The name is case-sensitive and can have up to 31 characters, which + can be letters, numbers, spaces, hyphens, and underscores. The name must be unique on a firewall and, on Panorama, + unique within its device group and any ancestor or descendant device groups. + + vsys(str): The string representation of the VSYS ID. Defaults to VSYS 1. + + action(str): The action that the security rule will enforce. Valid options are: allow, deny, drop, reset-client, + reset-server, reset-both. + + disabled(bool): Controls if the rule is disabled. Set 'True' to disable and 'False' to enable. + + sourcezone(str, list): The source zone(s). The value 'any' will match all zones. + + destinationzone(str, list): The destination zone(s). The value 'any' will match all zones. + + source(str, list): The source address(es). The value 'any' will match all addresses. + + destination(str, list): The destination address(es). The value 'any' will match all addresses. + + application(str, list): The application(s) matched. The value 'any' will match all applications. + + service(str, list): The service(s) matched. The value 'any' will match all services. The value + 'application-default' will match based upon the application defined ports. + + description(str): A description for the policy (up to 255 characters). + + logsetting(str): The name of a valid log forwarding profile. + + logstart(bool): Generates a traffic log entry for the start of a session (disabled by default). + + logend(bool): Generates a traffic log entry for the end of a session (enabled by default). + + negatesource(bool): Match all but the specified source addresses. + + negatedestination(bool): Match all but the specified destination addresses. + + profilegroup(str): A valid profile group name. + + datafilter(str): A valid data filter profile name. Ignored with the profilegroup option set. + + fileblock(str): A valid file blocking profile name. Ignored with the profilegroup option set. + + spyware(str): A valid spyware profile name. Ignored with the profilegroup option set. + + urlfilter(str): A valid URL filtering profile name. Ignored with the profilegroup option set. + + virus(str): A valid virus profile name. Ignored with the profilegroup option set. + + vulnerability(str): A valid vulnerability profile name. Ignored with the profilegroup option set. + + wildfire(str): A valid vulnerability profile name. Ignored with the profilegroup option set. + + move(str): An optional argument that ensure the rule is moved to a specific location. Valid options are 'top', + 'bottom', 'before', or 'after'. The 'before' and 'after' options require the use of the 'movetarget' argument + to define the location of the move request. + + movetarget(str): An optional argument that defines the target of the move operation if the move argument is + set to 'before' or 'after'. + + commit(bool): If true the firewall will commit the changes, if false do not commit changes. + + SLS Example: + + .. code-block:: yaml + + panos/rulebase/security/rule01: + panos.security_rule_exists: + - rulename: rule01 + - vsys: 1 + - action: allow + - disabled: False + - sourcezone: untrust + - destinationzone: trust + - source: + - 10.10.10.0/24 + - 1.1.1.1 + - destination: + - 2.2.2.2-2.2.2.4 + - application: + - any + - service: + - tcp-25 + - description: My test security rule + - logsetting: logprofile + - logstart: False + - logend: True + - negatesource: False + - negatedestination: False + - profilegroup: myprofilegroup + - move: top + - commit: False + + panos/rulebase/security/rule01: + panos.security_rule_exists: + - rulename: rule01 + - vsys: 1 + - action: allow + - disabled: False + - sourcezone: untrust + - destinationzone: trust + - source: + - 10.10.10.0/24 + - 1.1.1.1 + - destination: + - 2.2.2.2-2.2.2.4 + - application: + - any + - service: + - tcp-25 + - description: My test security rule + - logsetting: logprofile + - logstart: False + - logend: False + - datafilter: foobar + - fileblock: foobar + - spyware: foobar + - urlfilter: foobar + - virus: foobar + - vulnerability: foobar + - wildfire: foobar + - move: after + - movetarget: rule02 + - commit: False + ''' + ret = _default_ret(name) + + if not rulename: + return ret + + # Check if rule currently exists + rule = __salt__['panos.get_security_rule'](rulename, vsys) + + # Build the rule element + element = "" + if sourcezone: + element += "{0}".format(_build_members(sourcezone, True)) + else: + ret.update({'comment': "The sourcezone field must be provided."}) + return ret + + if destinationzone: + element += "{0}".format(_build_members(destinationzone, True)) + else: + ret.update({'comment': "The destinationzone field must be provided."}) + return ret + + if source: + element += "{0}".format(_build_members(source, True)) + else: + ret.update({'comment': "The source field must be provided."}) + return + + if destination: + element += "{0}".format(_build_members(destination, True)) + else: + ret.update({'comment': "The destination field must be provided."}) + return ret + + if application: + element += "{0}".format(_build_members(application, True)) + else: + ret.update({'comment': "The application field must be provided."}) + return ret + + if service: + element += "{0}".format(_build_members(service, True)) + else: + ret.update({'comment': "The service field must be provided."}) + return ret + + if action: + element += "{0}".format(action) + else: + ret.update({'comment': "The action field must be provided."}) + return ret + + if disabled is not None: + if disabled: + element += "yes" + else: + element += "no" + + if description: + element += "{0}".format(description) + + if logsetting: + element += "{0}".format(logsetting) + + if logstart is not None: + if logstart: + element += "yes" + else: + element += "no" + + if logend is not None: + if logend: + element += "yes" + else: + element += "no" + + if negatesource is not None: + if negatesource: + element += "yes" + else: + element += "no" + + if negatedestination is not None: + if negatedestination: + element += "yes" + else: + element += "no" + + # Build the profile settings + profile_string = None + if profilegroup: + profile_string = "{0}".format(profilegroup) + else: + member_string = "" + if datafilter: + member_string += "{0}".format(datafilter) + if fileblock: + member_string += "{0}".format(fileblock) + if spyware: + member_string += "{0}".format(spyware) + if urlfilter: + member_string += "{0}".format(urlfilter) + if virus: + member_string += "{0}".format(virus) + if vulnerability: + member_string += "{0}".format(vulnerability) + if wildfire: + member_string += "{0}".format(wildfire) + if member_string != "": + profile_string = "{0}".format(member_string) + + if profile_string: + element += "{0}".format(profile_string) + + full_element = "{1}".format(rulename, element) + + create_rule = False + + if 'result' in rule: + if rule['result'] == "None": + create_rule = True + + if create_rule: + xpath = "/config/devices/entry[@name=\'localhost.localdomain\']/vsys/entry[@name=\'vsys{0}\']/rulebase/" \ + "security/rules".format(vsys) + + result, msg = _set_config(xpath, full_element) + if not result: + ret['changes']['set'] = msg + return ret + else: + xpath = "/config/devices/entry[@name=\'localhost.localdomain\']/vsys/entry[@name=\'vsys{0}\']/rulebase/" \ + "security/rules/entry[@name=\'{1}\']".format(vsys, rulename) + + result, msg = _edit_config(xpath, full_element) + if not result: + ret['changes']['edit'] = msg + return ret + + if move: + movepath = "/config/devices/entry[@name=\'localhost.localdomain\']/vsys/entry[@name=\'vsys{0}\']/rulebase/" \ + "security/rules/entry[@name=\'{1}\']".format(vsys, rulename) + move_result = False + move_msg = '' + if move == "before" and movetarget: + move_result, move_msg = _move_before(movepath, movetarget) + elif move == "after": + move_result, move_msg = _move_after(movepath, movetarget) + elif move == "top": + move_result, move_msg = _move_top(movepath) + elif move == "bottom": + move_result, move_msg = _move_bottom(movepath) + + if not move_result: + ret['changes']['move'] = move_msg + return ret + + if commit is True: + ret.update({ + 'commit': __salt__['panos.commit'](), + 'comment': 'Security rule verified successfully.', + 'result': True + }) + else: + ret.update({ + 'comment': 'Security rule verified successfully.', + 'result': True + }) + + return ret + + +def set_config(name, xpath=None, value=None, commit=False): + ''' + Sets a Palo Alto XPATH to a specific value. This will always overwrite the existing value, even if it is not + changed. + + You can add or create a new object at a specified location in the configuration hierarchy. Use the xpath parameter + to specify the location of the object in the configuration + + name: The name of the module function to execute. + + xpath(str): The XPATH of the configuration API tree to control. + + value(str): The XML value to set. This must be a child to the XPATH. + + commit(bool): If true the firewall will commit the changes, if false do not commit changes. + + SLS Example: + + .. code-block:: yaml + + panos/hostname: + panos.set_config: + - xpath: /config/devices/entry[@name='localhost.localdomain']/deviceconfig/system + - value: foobar + - commit: True + + ''' + ret = _default_ret(name) + + result, msg = _set_config(xpath, value) + + ret.update({ + 'comment': msg, + 'result': result + }) + + # Ensure we do not commit after a failed action + if not result: + return ret + + if commit is True: + ret.update({ + 'commit': __salt__['panos.commit'](), + 'result': True + }) + + return ret diff --git a/salt/states/pcs.py b/salt/states/pcs.py index 54b92cc3a89..9d0815ade26 100644 --- a/salt/states/pcs.py +++ b/salt/states/pcs.py @@ -164,13 +164,16 @@ Create a cluster from scratch: ''' from __future__ import absolute_import -# Import python libs +# Import Python libs import logging import os -# Import salt libs -import salt.utils -import salt.ext.six as six +# Import Salt libs +import salt.utils.files +import salt.utils.path + +# Import 3rd-party libs +from salt.ext import six log = logging.getLogger(__name__) @@ -179,7 +182,7 @@ def __virtual__(): ''' Only load if pcs package is installed ''' - if salt.utils.which('pcs'): + if salt.utils.path.which('pcs'): return 'pcs' return False @@ -190,7 +193,7 @@ def _file_read(path): ''' content = False if os.path.exists(path): - with salt.utils.fopen(path, 'r+') as fp_: + with salt.utils.files.fopen(path, 'r+') as fp_: content = fp_.read() fp_.close() return content @@ -200,7 +203,7 @@ def _file_write(path, content): ''' Write content to a file ''' - with salt.utils.fopen(path, 'w+') as fp_: + with salt.utils.files.fopen(path, 'w+') as fp_: fp_.write(content) fp_.close() diff --git a/salt/states/pip_state.py b/salt/states/pip_state.py index 1fc3817fa6d..8460bc8293c 100644 --- a/salt/states/pip_state.py +++ b/salt/states/pip_state.py @@ -25,12 +25,12 @@ import re import logging # Import salt libs -import salt.utils +import salt.utils.versions from salt.version import SaltStackVersion as _SaltStackVersion from salt.exceptions import CommandExecutionError, CommandNotFoundError # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six # pylint: disable=import-error try: import pip @@ -94,7 +94,7 @@ def _fulfills_version_spec(version, version_spec): for oper, spec in version_spec: if oper is None: continue - if not salt.utils.compare_versions(ver1=version, oper=oper, ver2=spec): + if not salt.utils.versions.compare(ver1=version, oper=oper, ver2=spec): return False return True @@ -527,7 +527,7 @@ def installed(name, # prepro = lambda pkg: pkg if type(pkg) == str else \ # ' '.join((pkg.items()[0][0], pkg.items()[0][1].replace(',', ';'))) # pkgs = ','.join([prepro(pkg) for pkg in pkgs]) - prepro = lambda pkg: pkg if isinstance(pkg, str) else \ + prepro = lambda pkg: pkg if isinstance(pkg, six.string_types) else \ ' '.join((six.iteritems(pkg)[0][0], six.iteritems(pkg)[0][1])) pkgs = [prepro(pkg) for pkg in pkgs] @@ -538,7 +538,7 @@ def installed(name, if use_wheel: min_version = '1.4' cur_version = __salt__['pip.version'](bin_env) - if not salt.utils.compare_versions(ver1=cur_version, oper='>=', + if not salt.utils.versions.compare(ver1=cur_version, oper='>=', ver2=min_version): ret['result'] = False ret['comment'] = ('The \'use_wheel\' option is only supported in ' @@ -550,7 +550,7 @@ def installed(name, if no_use_wheel: min_version = '1.4' cur_version = __salt__['pip.version'](bin_env) - if not salt.utils.compare_versions(ver1=cur_version, oper='>=', + if not salt.utils.versions.compare(ver1=cur_version, oper='>=', ver2=min_version): ret['result'] = False ret['comment'] = ('The \'no_use_wheel\' option is only supported in ' @@ -568,7 +568,7 @@ def installed(name, repo, version=_SaltStackVersion.from_name('Lithium').formatted_version )) - salt.utils.warn_until('Lithium', msg) + salt.utils.versions.warn_until('Lithium', msg) ret.setdefault('warnings', []).append(msg) name = repo diff --git a/salt/states/pkg.py b/salt/states/pkg.py index ef9f8700215..159f110cbc2 100644 --- a/salt/states/pkg.py +++ b/salt/states/pkg.py @@ -80,9 +80,10 @@ import logging import os import re -# Import salt libs -import salt.utils +# Import Salt libs import salt.utils.pkg +import salt.utils.platform +import salt.utils.versions from salt.output import nested from salt.utils import namespaced_function as _namespaced_function from salt.utils.odict import OrderedDict as _OrderedDict @@ -92,12 +93,12 @@ from salt.exceptions import ( from salt.modules.pkg_resource import _repack_pkgs # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six # pylint: disable=invalid-name _repack_pkgs = _namespaced_function(_repack_pkgs, globals()) -if salt.utils.is_windows(): +if salt.utils.platform.is_windows(): # pylint: disable=import-error,no-name-in-module,unused-import from salt.ext.six.moves.urllib.parse import urlparse as _urlparse from salt.exceptions import SaltRenderError @@ -105,6 +106,7 @@ if salt.utils.is_windows(): import datetime import errno import time + from functools import cmp_to_key # pylint: disable=import-error # pylint: enable=unused-import from salt.modules.win_pkg import _get_package_info @@ -172,19 +174,16 @@ def _fulfills_version_spec(versions, oper, desired_version, ''' cmp_func = __salt__.get('pkg.version_cmp') # stripping "with_origin" dict wrapper - if salt.utils.is_freebsd(): + if salt.utils.platform.is_freebsd(): if isinstance(versions, dict) and 'version' in versions: versions = versions['version'] for ver in versions: - if oper == '==': - if fnmatch.fnmatch(ver, desired_version): - return True - - elif salt.utils.compare_versions(ver1=ver, - oper=oper, - ver2=desired_version, - cmp_func=cmp_func, - ignore_epoch=ignore_epoch): + if (oper == '==' and fnmatch.fnmatch(ver, desired_version)) \ + or salt.utils.versions.compare(ver1=ver, + oper=oper, + ver2=desired_version, + cmp_func=cmp_func, + ignore_epoch=ignore_epoch): return True return False @@ -504,7 +503,7 @@ def _find_install_targets(name=None, if __grains__['os'] == 'FreeBSD': kwargs['with_origin'] = True - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): # Windows requires a refresh to establish a pkg db if refresh=True, so # add it to the kwargs. kwargs['refresh'] = refresh @@ -517,7 +516,7 @@ def _find_install_targets(name=None, 'result': False, 'comment': exc.strerror} - if salt.utils.is_windows() and kwargs.pop('refresh', False): + if salt.utils.platform.is_windows() and kwargs.pop('refresh', False): # We already refreshed when we called pkg.list_pkgs was_refreshed = True refresh = False @@ -541,7 +540,7 @@ def _find_install_targets(name=None, else 'sources')} to_unpurge = _find_unpurge_targets(desired) else: - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): pkginfo = _get_package_info(name, saltenv=kwargs['saltenv']) if not pkginfo: return {'name': name, @@ -1257,8 +1256,11 @@ def installed( ``NOTE:`` For :mod:`apt `, :mod:`ebuild `, - :mod:`pacman `, :mod:`yumpkg `, - and :mod:`zypper `, version numbers can be specified + :mod:`pacman `, + :mod:`winrepo `, + :mod:`yumpkg `, and + :mod:`zypper `, + version numbers can be specified in the ``pkgs`` argument. For example: .. code-block:: yaml @@ -1434,6 +1436,15 @@ def installed( 'result': True, 'comment': 'No packages to install provided'} + # If just a name (and optionally a version) is passed, just pack them into + # the pkgs argument. + if name and not any((pkgs, sources)): + if version: + pkgs = [{name: version}] + version = None + else: + pkgs = [name] + kwargs['saltenv'] = __env__ refresh = salt.utils.pkg.check_refresh(__opts__, refresh) if not isinstance(pkg_verify, list): @@ -1597,10 +1608,10 @@ def installed( failed_hold = None if targets or to_reinstall: force = False - if salt.utils.is_freebsd(): + if salt.utils.platform.is_freebsd(): force = True # Downgrades need to be forced. try: - pkg_ret = __salt__['pkg.install'](name, + pkg_ret = __salt__['pkg.install'](name=None, refresh=refresh, version=version, force=force, @@ -1871,7 +1882,7 @@ def downloaded(name, ignore_epoch=None, **kwargs): ''' - .. versionadded:: Oxygen + .. versionadded:: 2017.7.0 Ensure that the package is downloaded, and that it is the correct version (if specified). @@ -2008,7 +2019,7 @@ def downloaded(name, def patch_installed(name, advisory_ids=None, downloadonly=None, **kwargs): ''' - .. versionadded:: Oxygen + .. versionadded:: 2017.7.0 Ensure that packages related to certain advisory ids are installed. @@ -2088,7 +2099,7 @@ def patch_installed(name, advisory_ids=None, downloadonly=None, **kwargs): def patch_downloaded(name, advisory_ids=None, **kwargs): ''' - .. versionadded:: Oxygen + .. versionadded:: 2017.7.0 Ensure that packages related to certain advisory ids are downloaded. @@ -3059,7 +3070,7 @@ def mod_aggregate(low, chunks, running): if low.get('fun') not in agg_enabled: return low for chunk in chunks: - tag = salt.utils.gen_state_tag(chunk) + tag = __utils__['state.gen_tag'](chunk) if tag in running: # Already ran the pkg state, skip aggregation continue diff --git a/salt/states/pkgrepo.py b/salt/states/pkgrepo.py index 82064e42a5d..0c47abae1de 100644 --- a/salt/states/pkgrepo.py +++ b/salt/states/pkgrepo.py @@ -79,10 +79,9 @@ these states. Here is some example SLS: ``python-pycurl`` will need to be manually installed if it is not present once ``python-software-properties`` is installed. - On Ubuntu & Debian systems, the ```python-apt`` package is required to be - installed. To check if this package is installed, run ``dpkg -l - python-software-properties``. ``python-apt`` will need to be manually - installed if it is not present. + On Ubuntu & Debian systems, the ``python-apt`` package is required to be + installed. To check if this package is installed, run ``dpkg -l python-apt``. + ``python-apt`` will need to be manually installed if it is not present. ''' @@ -95,7 +94,10 @@ from salt.exceptions import CommandExecutionError, SaltInvocationError from salt.modules.aptpkg import _strip_uri from salt.state import STATE_INTERNAL_KEYWORDS as _STATE_INTERNAL_KEYWORDS import salt.utils +import salt.utils.files import salt.utils.pkg.deb +import salt.utils.pkg.rpm +import salt.utils.versions def __virtual__(): @@ -131,7 +133,7 @@ def managed(name, ppa=None, **kwargs): disabled : False Included to reduce confusion due to APT's use of the ``disabled`` - argument. If this is passed for a yum/dnf/zypper-based distro, then the + argument. If this is passed for a YUM/DNF/Zypper-based distro, then the reverse will be passed as ``enabled``. For example passing ``disabled=True`` will assume ``enabled=False``. @@ -150,7 +152,7 @@ def managed(name, ppa=None, **kwargs): enabled configuration. Anything supplied for this list will be saved in the repo configuration with a comment marker (#) in front. - Additional configuration values seen in yum repo files, such as ``gpgkey`` or + Additional configuration values seen in repo files, such as ``gpgkey`` or ``gpgcheck``, will be used directly as key-value pairs. For example: .. code-block:: yaml @@ -257,29 +259,45 @@ def managed(name, ppa=None, **kwargs): Use either ``keyid``/``keyserver`` or ``key_url``, but not both. - consolidate - If set to true, this will consolidate all sources definitions to - the sources.list file, cleanup the now unused files, consolidate - components (e.g. main) for the same URI, type, and architecture - to a single line, and finally remove comments from the sources.list - file. The consolidate will run every time the state is processed. The - option only needs to be set on one repo managed by salt to take effect. + consolidate : False + If set to ``True``, this will consolidate all sources definitions to the + sources.list file, cleanup the now unused files, consolidate components + (e.g. main) for the same URI, type, and architecture to a single line, + and finally remove comments from the sources.list file. The consolidate + will run every time the state is processed. The option only needs to be + set on one repo managed by salt to take effect. - clean_file - If set to true, empty file before config repo, dangerous if use - multiple sources in one file. + clean_file : False + If set to ``True``, empty the file before config repo + + .. note:: + Use with care. This can be dangerous if multiple sources are + configured in the same file. .. versionadded:: 2015.8.0 - refresh_db - If set to false this will skip refreshing the apt package database on - debian based systems. + refresh : True + If set to ``False`` this will skip refreshing the apt package database + on debian based systems. + + refresh_db : True + .. deprecated:: Oxygen + Use ``refresh`` instead. require_in Set this to a list of pkg.installed or pkg.latest to trigger the running of apt-get update prior to attempting to install these - packages. Setting a require in the pkg will not work for this. + packages. Setting a require in the pkg state will not work for this. ''' + if 'refresh_db' in kwargs: + salt.utils.versions.warn_until( + 'Neon', + 'The \'refresh_db\' argument to \'pkg.mod_repo\' has been ' + 'renamed to \'refresh\'. Support for using \'refresh_db\' will be ' + 'removed in the Neon release of Salt.' + ) + kwargs['refresh'] = kwargs.pop('refresh_db') + ret = {'name': name, 'changes': {}, 'result': None, @@ -403,6 +421,12 @@ def managed(name, ppa=None, **kwargs): salt.utils.pkg.deb.combine_comments(kwargs['comments']) if pre_comments != post_comments: break + elif kwarg == 'comments' and os_family == 'redhat': + precomments = salt.utils.pkg.rpm.combine_comments(pre[kwarg]) + kwargcomments = salt.utils.pkg.rpm.combine_comments( + sanitizedkwargs[kwarg]) + if precomments != kwargcomments: + break else: if os_family in ('redhat', 'suse') \ and any(isinstance(x, bool) for x in @@ -431,7 +455,7 @@ def managed(name, ppa=None, **kwargs): # empty file before configure if kwargs.get('clean_file', False): - with salt.utils.fopen(kwargs['file'], 'w'): + with salt.utils.files.fopen(kwargs['file'], 'w'): pass try: diff --git a/salt/states/ports.py b/salt/states/ports.py index 252c2410a41..1dbf2ed2c10 100644 --- a/salt/states/ports.py +++ b/salt/states/ports.py @@ -28,7 +28,7 @@ from salt.modules.freebsdports import _normalize, _options_file_exists # Needed by imported function _options_file_exists import os # pylint: disable=W0611 -import salt.ext.six as six +from salt.ext import six log = logging.getLogger(__name__) diff --git a/salt/states/postgres_privileges.py b/salt/states/postgres_privileges.py index 67c6fbbdf33..451f3f7eec1 100644 --- a/salt/states/postgres_privileges.py +++ b/salt/states/postgres_privileges.py @@ -110,6 +110,8 @@ def present(name, - group - function + View permissions should specify `object_type: table`. + privileges List of privileges to grant, from the list below: @@ -231,6 +233,8 @@ def absent(name, - group - function + View permissions should specify `object_type: table`. + privileges Comma separated list of privileges to revoke, from the list below: diff --git a/salt/states/postgres_schema.py b/salt/states/postgres_schema.py index 0f157563398..4acc52c46ce 100644 --- a/salt/states/postgres_schema.py +++ b/salt/states/postgres_schema.py @@ -29,7 +29,7 @@ def __virtual__(): def present(dbname, name, - owner=None, + owner=None, user=None, db_user=None, db_password=None, db_host=None, db_port=None): ''' @@ -41,8 +41,8 @@ def present(dbname, name, name The name of the schema to manage - owner - The database user that will be the owner of the schema + user + system user all operations should be performed on behalf of db_user database username if different from config or default @@ -67,7 +67,8 @@ def present(dbname, name, 'db_user': db_user, 'db_password': db_password, 'db_host': db_host, - 'db_port': db_port + 'db_port': db_port, + 'user': user } # check if schema exists @@ -105,7 +106,7 @@ def present(dbname, name, return ret -def absent(dbname, name, +def absent(dbname, name, user=None, db_user=None, db_password=None, db_host=None, db_port=None): ''' @@ -117,6 +118,9 @@ def absent(dbname, name, name The name of the schema to remove + user + system user all operations should be performed on behalf of + db_user database username if different from config or default @@ -139,7 +143,8 @@ def absent(dbname, name, 'db_user': db_user, 'db_password': db_password, 'db_host': db_host, - 'db_port': db_port + 'db_port': db_port, + 'user': user } # check if schema exists and remove it diff --git a/salt/states/postgres_tablespace.py b/salt/states/postgres_tablespace.py index 661be6812ff..634814e47ad 100644 --- a/salt/states/postgres_tablespace.py +++ b/salt/states/postgres_tablespace.py @@ -20,7 +20,7 @@ A module used to create and manage PostgreSQL tablespaces. from __future__ import absolute_import # Import salt libs -from salt.utils import dictupdate +import salt.utils.dictupdate as dictupdate # Import 3rd-party libs from salt.ext.six import iteritems diff --git a/salt/states/postgres_user.py b/salt/states/postgres_user.py index d9f686871c3..b2b255e751a 100644 --- a/salt/states/postgres_user.py +++ b/salt/states/postgres_user.py @@ -13,13 +13,14 @@ The postgres_users module is used to create and manage Postgres users. from __future__ import absolute_import # Import Python libs +import datetime +import logging # Import salt libs -import logging # Salt imports from salt.modules import postgres -import salt.ext.six as six +from salt.ext import six log = logging.getLogger(__name__) @@ -45,6 +46,7 @@ def present(name, password=None, default_password=None, refresh_password=None, + valid_until=None, groups=None, user=None, maintenance_db=None, @@ -112,6 +114,9 @@ def present(name, This behaviour makes it possible to execute in environments without superuser access available, e.g. Amazon RDS for PostgreSQL + valid_until + A date and time after which the role's password is no longer valid. + groups A string of comma separated groups the user should be in @@ -168,7 +173,6 @@ def present(name, if user_attr is not None: mode = 'update' - # The user is not present, make it! cret = None update = {} if mode == 'update': @@ -199,6 +203,18 @@ def present(name, update['superuser'] = superuser if password is not None and (refresh_password or user_attr['password'] != password): update['password'] = True + if valid_until is not None: + valid_until_dt = __salt__['postgres.psql_query']( + 'SELECT \'{0}\'::timestamp(0) as dt;'.format( + valid_until.replace('\'', '\'\'')), + **db_args)[0]['dt'] + try: + valid_until_dt = datetime.datetime.strptime( + valid_until_dt, '%Y-%m-%d %H:%M:%S') + except ValueError: + valid_until_dt = None + if valid_until_dt != user_attr['expiry time']: + update['valid_until'] = valid_until if groups is not None: lgroups = groups if isinstance(groups, (six.string_types, six.text_type)): @@ -228,6 +244,7 @@ def present(name, inherit=inherit, replication=replication, rolepassword=password, + valid_until=valid_until, groups=groups, **db_args) else: diff --git a/salt/states/proxy.py b/salt/states/proxy.py index f284d4e5ee3..6c128c290e3 100644 --- a/salt/states/proxy.py +++ b/salt/states/proxy.py @@ -14,13 +14,12 @@ Setup proxy settings on minions - localhost - 127.0.0.1 ''' - -# Import python libs +# Import Python libs from __future__ import absolute_import import logging -# Import salt libs -import salt.utils +# Import Salt libs +import salt.utils.platform log = logging.getLogger(__name__) __virtualname__ = 'proxy' @@ -30,7 +29,7 @@ def __virtual__(): ''' Only work on Mac OS and Windows ''' - if salt.utils.is_darwin() or salt.utils.is_windows(): + if salt.utils.platform.is_darwin() or salt.utils.platform.is_windows(): return True return False diff --git a/salt/states/rabbitmq_cluster.py b/salt/states/rabbitmq_cluster.py index e8d0ed33843..22316e99e07 100644 --- a/salt/states/rabbitmq_cluster.py +++ b/salt/states/rabbitmq_cluster.py @@ -19,6 +19,7 @@ import logging # Import salt libs import salt.utils +import salt.utils.path log = logging.getLogger(__name__) @@ -27,7 +28,7 @@ def __virtual__(): ''' Only load if RabbitMQ is installed. ''' - return salt.utils.which('rabbitmqctl') is not None + return salt.utils.path.which('rabbitmqctl') is not None def joined(name, host, user='rabbit', ram_node=None, runas='root'): diff --git a/salt/states/rabbitmq_policy.py b/salt/states/rabbitmq_policy.py index 37c4c8ff778..c438161a31e 100644 --- a/salt/states/rabbitmq_policy.py +++ b/salt/states/rabbitmq_policy.py @@ -21,7 +21,7 @@ from __future__ import absolute_import # Import python libs import logging -import salt.utils +import salt.utils.path log = logging.getLogger(__name__) @@ -30,7 +30,7 @@ def __virtual__(): ''' Only load if RabbitMQ is installed. ''' - return salt.utils.which('rabbitmqctl') is not None + return salt.utils.path.which('rabbitmqctl') is not None def present(name, diff --git a/salt/states/rabbitmq_user.py b/salt/states/rabbitmq_user.py index a990b5a2d2c..b431a229b3f 100644 --- a/salt/states/rabbitmq_user.py +++ b/salt/states/rabbitmq_user.py @@ -27,8 +27,8 @@ from __future__ import absolute_import import logging # Import salt libs -import salt.utils -import salt.ext.six as six +import salt.utils.path +from salt.ext import six from salt.exceptions import CommandExecutionError log = logging.getLogger(__name__) @@ -38,7 +38,7 @@ def __virtual__(): ''' Only load if RabbitMQ is installed. ''' - return salt.utils.which('rabbitmqctl') is not None + return salt.utils.path.which('rabbitmqctl') is not None def _check_perms_changes(name, newperms, runas=None, existing=None): @@ -175,7 +175,7 @@ def present(name, if tags is not None: current_tags = _get_current_tags(name, runas=runas) - if isinstance(tags, str): + if isinstance(tags, six.string_types): tags = tags.split() # Diff the tags sets. Symmetric difference operator ^ will give us # any element in one set, but not both diff --git a/salt/states/rabbitmq_vhost.py b/salt/states/rabbitmq_vhost.py index cb8332df3b2..c35c7810f5d 100644 --- a/salt/states/rabbitmq_vhost.py +++ b/salt/states/rabbitmq_vhost.py @@ -20,7 +20,7 @@ from __future__ import absolute_import import logging # Import salt libs -import salt.utils +import salt.utils.path log = logging.getLogger(__name__) @@ -29,7 +29,7 @@ def __virtual__(): ''' Only load if RabbitMQ is installed. ''' - return salt.utils.which('rabbitmqctl') is not None + return salt.utils.path.which('rabbitmqctl') is not None def present(name): diff --git a/salt/states/rbenv.py b/salt/states/rbenv.py index f4889be8a2c..50a66e1ef41 100644 --- a/salt/states/rbenv.py +++ b/salt/states/rbenv.py @@ -77,7 +77,7 @@ def _ruby_installed(ret, ruby, user=None): for version in __salt__['rbenv.versions'](user): if version == ruby: ret['result'] = True - ret['comment'] = 'Requested ruby exists.' + ret['comment'] = 'Requested ruby exists' ret['default'] = default == ruby break @@ -97,7 +97,7 @@ def _check_and_install_ruby(ret, ruby, default=False, user=None): ret['default'] = default else: ret['result'] = False - ret['comment'] = 'Could not install ruby.' + ret['comment'] = 'Failed to install ruby' return ret if default: @@ -131,7 +131,11 @@ def installed(name, default=False, user=None): name = re.sub(r'^ruby-', '', name) if __opts__['test']: - ret['comment'] = 'Ruby {0} is set to be installed'.format(name) + ret = _ruby_installed(ret, name, user=user) + if not ret['result']: + ret['comment'] = 'Ruby {0} is set to be installed'.format(name) + else: + ret['comment'] = 'Ruby {0} is already installed'.format(name) return ret rbenv_installed_ret = _check_and_install_rbenv(rbenv_installed_ret, user) @@ -188,16 +192,22 @@ def absent(name, user=None): if name.startswith('ruby-'): name = re.sub(r'^ruby-', '', name) - if __opts__['test']: - ret['comment'] = 'Ruby {0} is set to be uninstalled'.format(name) - return ret - ret = _check_rbenv(ret, user) if ret['result'] is False: ret['result'] = True ret['comment'] = 'Rbenv not installed, {0} not either'.format(name) return ret else: + if __opts__['test']: + ret = _ruby_installed(ret, name, user=user) + if ret['result']: + ret['result'] = None + ret['comment'] = 'Ruby {0} is set to be uninstalled'.format(name) + else: + ret['result'] = True + ret['comment'] = 'Ruby {0} is already uninstalled'.format(name) + return ret + return _check_and_uninstall_ruby(ret, name, user=user) @@ -205,7 +215,6 @@ def _check_and_install_rbenv(ret, user=None): ''' Verify that rbenv is installed, install if unavailable ''' - ret = _check_rbenv(ret, user) if ret['result'] is False: if __salt__['rbenv.install'](user): @@ -216,7 +225,7 @@ def _check_and_install_rbenv(ret, user=None): ret['comment'] = 'Rbenv failed to install' else: ret['result'] = True - ret['comment'] = 'Rbenv already installed' + ret['comment'] = 'Rbenv is already installed' return ret @@ -237,7 +246,13 @@ def install_rbenv(name, user=None): ret = {'name': name, 'result': None, 'comment': '', 'changes': {}} if __opts__['test']: - ret['comment'] = 'Rbenv is set to be installed' + ret = _check_rbenv(ret, user=user) + if ret['result'] is False: + ret['result'] = None + ret['comment'] = 'Rbenv is set to be installed' + else: + ret['result'] = True + ret['comment'] = 'Rbenv is already installed' return ret return _check_and_install_rbenv(ret, user) diff --git a/salt/states/rsync.py b/salt/states/rsync.py index eb72a0d8eaa..01d8711f2b4 100644 --- a/salt/states/rsync.py +++ b/salt/states/rsync.py @@ -28,9 +28,10 @@ State to synchronize files and directories with rsync. ''' from __future__ import absolute_import -import salt.utils import os +import salt.utils.path + def __virtual__(): ''' @@ -38,7 +39,7 @@ def __virtual__(): :return: ''' - return salt.utils.which('rsync') and 'rsync' or False + return salt.utils.path.which('rsync') and 'rsync' or False def _get_summary(rsync_out): diff --git a/salt/states/saltmod.py b/salt/states/saltmod.py index d9cc6ee1965..0095401356a 100644 --- a/salt/states/saltmod.py +++ b/salt/states/saltmod.py @@ -30,10 +30,9 @@ import time # Import salt libs import salt.syspaths -import salt.utils import salt.utils.event -import salt.ext.six as six -from salt.ext.six import string_types +import salt.utils.versions +from salt.ext import six log = logging.getLogger(__name__) @@ -83,7 +82,8 @@ def state(name, batch=None, queue=False, subset=None, - orchestration_jid=None): + orchestration_jid=None, + **kwargs): ''' Invoke a state run on a given target @@ -227,7 +227,7 @@ def state(name, # remember to remove the expr_form argument from this function when # performing the cleanup on this deprecation. if expr_form is not None: - salt.utils.warn_until( + salt.utils.versions.warn_until( 'Fluorine', 'the target type should be passed using the \'tgt_type\' ' 'argument instead of \'expr_form\'. Support for using ' @@ -259,14 +259,12 @@ def state(name, if pillar: cmd_kw['kwarg']['pillar'] = pillar - # If pillarenv is directly defined, use it - if pillarenv: + if pillarenv is not None: cmd_kw['kwarg']['pillarenv'] = pillarenv - # Use pillarenv if it's passed from __opts__ (via state.orchestrate for example) - elif __opts__.get('pillarenv'): - cmd_kw['kwarg']['pillarenv'] = __opts__['pillarenv'] - cmd_kw['kwarg']['saltenv'] = saltenv if saltenv is not None else __env__ + if saltenv is not None: + cmd_kw['kwarg']['saltenv'] = saltenv + cmd_kw['kwarg']['queue'] = queue if isinstance(concurrent, bool): @@ -311,7 +309,7 @@ def state(name, if fail_minions is None: fail_minions = () - elif isinstance(fail_minions, string_types): + elif isinstance(fail_minions, six.string_types): fail_minions = [minion.strip() for minion in fail_minions.split(',')] elif not isinstance(fail_minions, list): state_ret.setdefault('warnings', []).append( @@ -343,7 +341,7 @@ def state(name, except KeyError: m_state = False if m_state: - m_state = salt.utils.check_state_result(m_ret, recurse=True) + m_state = __utils__['state.check_result'](m_ret, recurse=True) if not m_state: if minion not in fail_minions: @@ -468,7 +466,7 @@ def function( 'result': True} if kwarg is None: kwarg = {} - if isinstance(arg, str): + if isinstance(arg, six.string_types): func_ret['warnings'] = ['Please specify \'arg\' as a list, not a string. ' 'Modifying in place, but please update SLS file ' 'to remove this warning.'] @@ -479,7 +477,7 @@ def function( # remember to remove the expr_form argument from this function when # performing the cleanup on this deprecation. if expr_form is not None: - salt.utils.warn_until( + salt.utils.versions.warn_until( 'Fluorine', 'the target type should be passed using the \'tgt_type\' ' 'argument instead of \'expr_form\'. Support for using ' @@ -529,7 +527,7 @@ def function( if fail_minions is None: fail_minions = () - elif isinstance(fail_minions, string_types): + elif isinstance(fail_minions, six.string_types): fail_minions = [minion.strip() for minion in fail_minions.split(',')] elif not isinstance(fail_minions, list): func_ret.setdefault('warnings', []).append( @@ -743,18 +741,26 @@ def runner(name, **kwargs): if isinstance(runner_return, dict) and 'Error' in runner_return: out['success'] = False if not out.get('success', True): + cmt = "Runner function '{0}' failed{1}.".format( + name, + ' with return {0}'.format(runner_return) if runner_return else '', + ) ret = { 'name': name, 'result': False, 'changes': {}, - 'comment': runner_return if runner_return else "Runner function '{0}' failed without comment.".format(name) + 'comment': cmt, } else: + cmt = "Runner function '{0}' executed{1}.".format( + name, + ' with return {0}'.format(runner_return) if runner_return else '', + ) ret = { 'name': name, 'result': True, - 'changes': runner_return if runner_return else {}, - 'comment': "Runner function '{0}' executed.".format(name) + 'changes': {}, + 'comment': cmt, } ret['__orchestration__'] = True @@ -803,14 +809,14 @@ def wheel(name, **kwargs): **kwargs) ret['result'] = True - ret['comment'] = "Wheel function '{0}' executed.".format(name) - ret['__orchestration__'] = True if 'jid' in out: ret['__jid__'] = out['jid'] runner_return = out.get('return') - if runner_return: - ret['changes'] = runner_return + ret['comment'] = "Wheel function '{0}' executed{1}.".format( + name, + ' with return {0}'.format(runner_return) if runner_return else '', + ) return ret diff --git a/salt/states/selinux.py b/salt/states/selinux.py index 868dbc16e33..8187ea8338d 100644 --- a/salt/states/selinux.py +++ b/salt/states/selinux.py @@ -171,8 +171,14 @@ def boolean(name, value, persist=False): name, rvalue) return ret - if __salt__['selinux.setsebool'](name, rvalue, persist): + ret['result'] = __salt__['selinux.setsebool'](name, rvalue, persist) + if ret['result']: ret['comment'] = 'Boolean {0} has been set to {1}'.format(name, rvalue) + ret['changes'].update({'State': {'old': bools[name]['State'], + 'new': rvalue}}) + if persist and not default: + ret['changes'].update({'Default': {'old': bools[name]['Default'], + 'new': rvalue}}) return ret ret['comment'] = 'Failed to set the boolean {0} to {1}'.format(name, rvalue) return ret @@ -262,7 +268,7 @@ def module_install(name): name Path to file with module to install - .. versionadded:: develop + .. versionadded:: 2016.11.6 ''' ret = {'name': name, 'result': True, @@ -283,7 +289,7 @@ def module_remove(name): name The name of the module to remove - .. versionadded:: develop + .. versionadded:: 2016.11.6 ''' ret = {'name': name, 'result': True, diff --git a/salt/states/serverdensity_device.py b/salt/states/serverdensity_device.py index a2395a2bf9d..a2427e2de8d 100644 --- a/salt/states/serverdensity_device.py +++ b/salt/states/serverdensity_device.py @@ -53,7 +53,7 @@ import json import logging # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six import json # TODO: diff --git a/salt/states/service.py b/salt/states/service.py index 895d6b49608..0b0850d9a32 100644 --- a/salt/states/service.py +++ b/salt/states/service.py @@ -57,18 +57,18 @@ service, then set the reload value to True: :ref:`Requisites ` documentation. ''' - # Import Python libs from __future__ import absolute_import import time # Import Salt libs import salt.utils +import salt.utils.platform from salt.utils.args import get_function_argspec as _argspec from salt.exceptions import CommandExecutionError # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six SYSTEMD_ONLY = ('no_block', 'unmask', 'unmask_runtime') @@ -432,7 +432,7 @@ def running(name, ret['comment'] = 'Service {0} is set to start'.format(name) return ret - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): if enable is True: ret.update(_enable(name, False, result=False, **kwargs)) @@ -900,7 +900,7 @@ def mod_watch(name, try: result = func(name, **func_kwargs) except CommandExecutionError as exc: - ret['result'] = True + ret['result'] = False ret['comment'] = exc.strerror return ret diff --git a/salt/states/smartos.py b/salt/states/smartos.py index ecab7b1a00b..15d440b7539 100644 --- a/salt/states/smartos.py +++ b/salt/states/smartos.py @@ -122,7 +122,7 @@ def _load_config(): config = {} if os.path.isfile('/usbkey/config'): - with salt.utils.fopen('/usbkey/config', 'r') as config_file: + with salt.utils.files.fopen('/usbkey/config', 'r') as config_file: for optval in config_file: if optval[0] == '#': continue diff --git a/salt/states/solrcloud.py b/salt/states/solrcloud.py index 7e738d2c6f5..6fc969af279 100644 --- a/salt/states/solrcloud.py +++ b/salt/states/solrcloud.py @@ -11,7 +11,7 @@ from __future__ import absolute_import import json # Import 3rd party libs -import salt.ext.six as six +from salt.ext import six def alias(name, collections, **kwargs): diff --git a/salt/states/sqlite3.py b/salt/states/sqlite3.py index 8251e04e71d..98fe38a6ef8 100644 --- a/salt/states/sqlite3.py +++ b/salt/states/sqlite3.py @@ -96,7 +96,7 @@ can be approximated with sqlite3's module functions and module.run: from __future__ import absolute_import # Import Salt libs -import salt.ext.six as six +from salt.ext import six try: import sqlite3 @@ -160,9 +160,13 @@ def row_absent(name, db, table, where_sql, where_args=None): changes['changes']['old'] = rows[0] else: - cursor = conn.execute("DELETE FROM `" + - table + "` WHERE " + where_sql, - where_args) + if where_args is None: + cursor = conn.execute("DELETE FROM `" + + table + "` WHERE " + where_sql) + else: + cursor = conn.execute("DELETE FROM `" + + table + "` WHERE " + where_sql, + where_args) conn.commit() if cursor.rowcount == 1: changes['result'] = True @@ -406,7 +410,7 @@ def table_present(name, db, schema, force=False): if len(tables) == 1: sql = None - if isinstance(schema, str): + if isinstance(schema, six.string_types): sql = schema.strip() else: sql = _get_sql_from_schema(name, schema) @@ -437,7 +441,7 @@ def table_present(name, db, schema, force=False): elif len(tables) == 0: # Create the table sql = None - if isinstance(schema, str): + if isinstance(schema, six.string_types): sql = schema else: sql = _get_sql_from_schema(name, schema) diff --git a/salt/states/ssh_auth.py b/salt/states/ssh_auth.py index 120a7d078a8..aa78786033e 100644 --- a/salt/states/ssh_auth.py +++ b/salt/states/ssh_auth.py @@ -52,10 +52,10 @@ import re import sys # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six -def _present_test(user, name, enc, comment, options, source, config): +def _present_test(user, name, enc, comment, options, source, config, fingerprint_hash_type): ''' Run checks for "present" ''' @@ -65,7 +65,8 @@ def _present_test(user, name, enc, comment, options, source, config): user, source, config, - saltenv=__env__) + saltenv=__env__, + fingerprint_hash_type=fingerprint_hash_type) if keys: comment = '' for key, status in six.iteritems(keys): @@ -111,7 +112,8 @@ def _present_test(user, name, enc, comment, options, source, config): enc, comment, options, - config) + config=config, + fingerprint_hash_type=fingerprint_hash_type) if check == 'update': comment = ( 'Key {0} for user {1} is set to be updated' @@ -128,7 +130,7 @@ def _present_test(user, name, enc, comment, options, source, config): return result, comment -def _absent_test(user, name, enc, comment, options, source, config): +def _absent_test(user, name, enc, comment, options, source, config, fingerprint_hash_type): ''' Run checks for "absent" ''' @@ -138,7 +140,8 @@ def _absent_test(user, name, enc, comment, options, source, config): user, source, config, - saltenv=__env__) + saltenv=__env__, + fingerprint_hash_type=fingerprint_hash_type) if keys: comment = '' for key, status in list(keys.items()): @@ -184,7 +187,8 @@ def _absent_test(user, name, enc, comment, options, source, config): enc, comment, options, - config) + config=config, + fingerprint_hash_type=fingerprint_hash_type) if check == 'update' or check == 'exists': comment = ('Key {0} for user {1} is set for removal').format(name, user) else: @@ -202,6 +206,7 @@ def present( source='', options=None, config='.ssh/authorized_keys', + fingerprint_hash_type=None, **kwargs): ''' Verifies that the specified SSH key is present for the specified user @@ -243,6 +248,17 @@ def present( The location of the authorized keys file relative to the user's home directory, defaults to ".ssh/authorized_keys". Token expansion %u and %h for username and home path supported. + + fingerprint_hash_type + The public key fingerprint hash type that the public key fingerprint + was originally hashed with. This defaults to ``md5`` if not specified. + + .. versionadded:: 2016.11.7 + + .. note:: + + The default value of the ``fingerprint_hash_type`` will change to + ``sha256`` in Salt 2017.7.0. ''' ret = {'name': name, 'changes': {}, @@ -279,7 +295,7 @@ def present( options or [], source, config, - ) + fingerprint_hash_type) return ret # Get only the path to the file without env referrences to check if exists @@ -305,10 +321,11 @@ def present( data = __salt__['ssh.set_auth_key_from_file']( user, source, - config, - saltenv=__env__) + config=config, + saltenv=__env__, + fingerprint_hash_type=fingerprint_hash_type) else: - # Split keyline to get key und commen + # Split keyline to get key und comment keyline = keyline.split(' ') key_type = keyline[0] key_value = keyline[1] @@ -316,18 +333,20 @@ def present( data = __salt__['ssh.set_auth_key']( user, key_value, - key_type, - key_comment, - options or [], - config) + enc=key_type, + comment=key_comment, + options=options or [], + config=config, + fingerprint_hash_type=fingerprint_hash_type) else: data = __salt__['ssh.set_auth_key']( user, name, - enc, - comment, - options or [], - config) + enc=enc, + comment=comment, + options=options or [], + config=config, + fingerprint_hash_type=fingerprint_hash_type) if data == 'replace': ret['changes'][name] = 'Updated' @@ -369,7 +388,8 @@ def absent(name, comment='', source='', options=None, - config='.ssh/authorized_keys'): + config='.ssh/authorized_keys', + fingerprint_hash_type=None): ''' Verifies that the specified SSH key is absent @@ -401,6 +421,17 @@ def absent(name, directory, defaults to ".ssh/authorized_keys". Token expansion %u and %h for username and home path supported. + fingerprint_hash_type + The public key fingerprint hash type that the public key fingerprint + was originally hashed with. This defaults to ``md5`` if not specified. + + .. versionadded:: 2016.11.7 + + .. note:: + + The default value of the ``fingerprint_hash_type`` will change to + ``sha256`` in Salt 2017.7.0. + ''' ret = {'name': name, 'changes': {}, @@ -416,7 +447,7 @@ def absent(name, options or [], source, config, - ) + fingerprint_hash_type) return ret # Extract Key from file if source is present @@ -434,13 +465,15 @@ def absent(name, ret['comment'] = __salt__['ssh.rm_auth_key_from_file'](user, source, config, - saltenv=__env__) + saltenv=__env__, + fingerprint_hash_type=fingerprint_hash_type) else: # Split keyline to get key keyline = keyline.split(' ') ret['comment'] = __salt__['ssh.rm_auth_key'](user, keyline[1], - config) + config=config, + fingerprint_hash_type=fingerprint_hash_type) else: # Get just the key sshre = re.compile(r'^(.*?)\s?((?:ssh\-|ecds)[\w-]+\s.+)$') @@ -461,7 +494,10 @@ def absent(name, name = comps[1] if len(comps) == 3: comment = comps[2] - ret['comment'] = __salt__['ssh.rm_auth_key'](user, name, config) + ret['comment'] = __salt__['ssh.rm_auth_key'](user, + name, + config=config, + fingerprint_hash_type=fingerprint_hash_type) if ret['comment'] == 'User authorized keys file not present': ret['result'] = False diff --git a/salt/states/ssh_known_hosts.py b/salt/states/ssh_known_hosts.py index 0fe24cb8138..93325aa18a6 100644 --- a/salt/states/ssh_known_hosts.py +++ b/salt/states/ssh_known_hosts.py @@ -12,6 +12,7 @@ Manage the information stored in the known_hosts files. - present - user: root - fingerprint: 16:27:ac:a5:76:28:2d:36:63:1b:56:4d:eb:df:a6:48 + - fingerprint_hash_type: md5 example.com: ssh_known_hosts: @@ -20,12 +21,12 @@ Manage the information stored in the known_hosts files. ''' from __future__ import absolute_import -# Import python libs +# Import Python libs import os -# Import salt libs +# Import Salt libs +import salt.utils.platform from salt.exceptions import CommandNotFoundError -import salt.utils # Define the state's virtual name __virtualname__ = 'ssh_known_hosts' @@ -35,7 +36,7 @@ def __virtual__(): ''' Does not work on Windows, requires ssh module functions ''' - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): return False, 'ssh_known_hosts: Does not support Windows' return __virtualname__ diff --git a/salt/states/stormpath_account.py b/salt/states/stormpath_account.py deleted file mode 100644 index a23f2b057da..00000000000 --- a/salt/states/stormpath_account.py +++ /dev/null @@ -1,193 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Support for Stormpath. - -.. versionadded:: 2015.8.0 -''' - -# Import python libs -from __future__ import absolute_import -import pprint - - -def __virtual__(): - ''' - Only load if the stormpath module is available in __salt__ - ''' - return 'stormpath.create_account' in __salt__ - - -def present(name, **kwargs): - ''' - Ensure that an account is present and properly configured - - name - The email address associated with the Stormpath account - - directory_id - The ID of a directory which the account belongs to. Required. - - password - Required when creating a new account. If specified, it is advisable to - reference the password in another database using an ``sdb://`` URL. - Will NOT update the password if an account already exists. - - givenName - Required when creating a new account. - - surname - Required when creating a new account. - - username - Optional. Must be unique across the owning directory. If not specified, - the username will default to the email field. - - middleName - Optional. - - status - ``enabled`` accounts are able to login to their assigned applications, - ``disabled`` accounts may not login to applications, ``unverified`` - accounts are disabled and have not verified their email address. - - customData. - Optional. Must be specified as a dict. - ''' - # Because __opts__ is not available outside of functions - backend = __opts__.get('backend', False) - if not backend: - backend = 'requests' - - if backend == 'requests': - from requests.exceptions import HTTPError - elif backend == 'urrlib2': - from urllib2 import HTTPError - else: - from tornado.httpclient import HTTPError - - ret = {'name': name, - 'changes': {}, - 'result': None, - 'comment': ''} - info = {} - try: - result = __salt__['stormpath.show_account'](email=name, **kwargs) - if len(result['items']) > 0: - info = result['items'][0] - except HTTPError: - pass - needs_update = {} - if info.get('email', False): - for field in kwargs: - if info.get(field, None) != kwargs[field]: - needs_update[field] = kwargs[field] - del needs_update['directory_id'] - if 'password' in needs_update: - del needs_update['password'] - if len(needs_update.keys()) < 1: - ret['result'] = True - ret['comment'] = 'Stormpath account {0} already exists and is correct'.format(name) - return ret - if __opts__['test']: - if len(needs_update.keys()) < 1: - ret['comment'] = 'Stormpath account {0} needs to be created'.format(name) - else: - if 'password' in needs_update: - needs_update['password'] = '**HIDDEN**' - ret['comment'] = ('Stormpath account {0} needs the following ' - 'fields to be updated: '.format(', '.join(needs_update))) - return ret - if len(needs_update.keys()) < 1: - info = __salt__['stormpath.create_account'](email=name, **kwargs) - comps = info['href'].split('/') - account_id = comps[-1] - ret['changes'] = info - ret['result'] = True - kwargs['password'] = '**HIDDEN**' - ret['comment'] = 'Created account ID {0} ({1}): {2}'.format( - account_id, name, pprint.pformat(kwargs)) - return ret - comps = info['href'].split('/') - account_id = comps[-1] - result = __salt__['stormpath.update_account'](account_id, items=needs_update) - if result.get('href', None): - ret['changes'] = needs_update - ret['result'] = True - if 'password' in needs_update: - needs_update['password'] = '**HIDDEN**' - ret['comment'] = 'Set the following fields for account ID {0} ({1}): {2}'.format( - account_id, name, pprint.pformat(needs_update)) - return ret - else: - ret['result'] = False - ret['comment'] = 'Failed to set the following fields for account ID {0} ({1}): {2}'.format( - account_id, name, pprint.pformat(needs_update)) - return ret - - -def absent(name, directory_id=None): - ''' - Ensure that an account associated with the given email address is absent. - Will search all directories for the account, unless a directory_id is - specified. - - name - The email address of the account to delete. - - directory_id - Optional. The ID of the directory that the account is expected to belong - to. If not specified, then a list of directories will be retrieved, and - each will be scanned for the account. Specifying a directory_id will - therefore cut down on the number of requests to Stormpath, and increase - performance of this state. - ''' - # Because __opts__ is not available outside of functions - backend = __opts__.get('backend', False) - if not backend: - backend = 'requests' - - if backend == 'requests': - from requests.exceptions import HTTPError - elif backend == 'urrlib2': - from urllib2 import HTTPError - else: - from tornado.httpclient import HTTPError - - ret = {'name': name, - 'changes': {}, - 'result': None, - 'comment': ''} - info = {} - if directory_id is None: - dirs = __salt__['stormpath.list_directories']() - for dir_ in dirs.get('items', []): - try: - comps = dir_.get('href', '').split('/') - directory_id = comps[-1] - info = __salt__['stormpath.show_account'](email=name, directory_id=directory_id) - if len(info.get('items', [])) > 0: - info = info['items'][0] - break - except HTTPError: - pass - else: - info = __salt__['stormpath.show_account'](email=name, directory_id=directory_id) - info = info['items'][0] - if 'items' in info: - ret['result'] = True - ret['comment'] = 'Stormpath account {0} already absent'.format(name) - return ret - if __opts__['test']: - ret['comment'] = 'Stormpath account {0} needs to be deleted'.format(name) - return ret - comps = info['href'].split('/') - account_id = comps[-1] - if __salt__['stormpath.delete_account'](account_id): - ret['changes'] = {'deleted': account_id} - ret['result'] = True - ret['comment'] = 'Stormpath account {0} was deleted'.format(name) - return ret - else: - ret['result'] = False - ret['comment'] = 'Failed to delete Stormpath account {0}'.format(name) - return ret diff --git a/salt/states/supervisord.py b/salt/states/supervisord.py index c886d3f699f..c114b864d7e 100644 --- a/salt/states/supervisord.py +++ b/salt/states/supervisord.py @@ -16,7 +16,7 @@ from __future__ import absolute_import # Import python libs import logging -import salt.ext.six as six +from salt.ext import six log = logging.getLogger(__name__) diff --git a/salt/states/sysrc.py b/salt/states/sysrc.py index 5390b4f3d39..223bf56db34 100644 --- a/salt/states/sysrc.py +++ b/salt/states/sysrc.py @@ -4,7 +4,7 @@ from __future__ import absolute_import # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six # define the module's virtual name __virtualname__ = 'sysrc' diff --git a/salt/states/testinframod.py b/salt/states/testinframod.py index 0350f4d3856..a31a72c3e00 100644 --- a/salt/states/testinframod.py +++ b/salt/states/testinframod.py @@ -52,8 +52,12 @@ def _to_snake_case(pascal_case): def _generate_functions(): - for module in modules.__all__: - module_name = _to_snake_case(module) + try: + modules_ = [_to_snake_case(module_) for module_ in modules.__all__] + except AttributeError: + modules_ = [module_ for module_ in modules.modules] + + for module_name in modules_: func_name = 'testinfra.{0}'.format(module_name) __all__.append(module_name) log.debug('Generating state for module %s as function %s', diff --git a/salt/states/trafficserver.py b/salt/states/trafficserver.py index 13b15da7b4c..8d0ec4354a0 100644 --- a/salt/states/trafficserver.py +++ b/salt/states/trafficserver.py @@ -10,7 +10,7 @@ Control Apache Traffic Server from __future__ import absolute_import # Import Salt libs -import salt.utils +import salt.utils.versions def __virtual__(): @@ -260,7 +260,7 @@ def set_var(name, value): - value: cdn.site.domain.tld ''' - salt.utils.warn_until( + salt.utils.versions.warn_until( 'Fluorine', 'The \'set_var\' function has been deprecated and will be removed in Salt ' '{version}. Please use \'trafficserver.config\' instead.' diff --git a/salt/states/user.py b/salt/states/user.py index 8a731cc2a10..027b587802b 100644 --- a/salt/states/user.py +++ b/salt/states/user.py @@ -23,14 +23,14 @@ as either absent or present testuser: user.absent ''' - -# Import python libs +# Import Python libs from __future__ import absolute_import import os import logging -# Import salt libs +# Import Salt libs import salt.utils +import salt.utils.platform from salt.utils.locales import sdecode, sdecode_if_string # Import 3rd-party libs @@ -146,7 +146,7 @@ def _changes(name, change['warndays'] = warndays if expire and lshad['expire'] != expire: change['expire'] = expire - elif 'shadow.info' in __salt__ and salt.utils.is_windows(): + elif 'shadow.info' in __salt__ and salt.utils.platform.is_windows(): if expire and expire is not -1 and salt.utils.date_format(lshad['expire']) != salt.utils.date_format(expire): change['expire'] = expire @@ -633,7 +633,7 @@ def present(name, # Setup params specific to Linux and Windows to be passed to the # add.user function - if not salt.utils.is_windows(): + if not salt.utils.platform.is_windows(): params = {'name': name, 'uid': uid, 'gid': gid, @@ -667,8 +667,8 @@ def present(name, # pwd incorrectly reports presence of home ret['changes']['home'] = '' if 'shadow.info' in __salt__ \ - and not salt.utils.is_windows()\ - and not salt.utils.is_darwin(): + and not salt.utils.platform.is_windows() \ + and not salt.utils.platform.is_darwin(): if password and not empty_password: __salt__['shadow.set_password'](name, password) spost = __salt__['shadow.info'](name) @@ -740,7 +740,7 @@ def present(name, ' {1}'.format(name, expire) ret['result'] = False ret['changes']['expire'] = expire - elif salt.utils.is_windows(): + elif salt.utils.platform.is_windows(): if password and not empty_password: if not __salt__['user.setpassword'](name, password): ret['comment'] = 'User {0} created but failed to set' \ @@ -757,7 +757,7 @@ def present(name, ' {1}'.format(name, expire) ret['result'] = False ret['changes']['expiration_date'] = spost['expire'] - elif salt.utils.is_darwin() and password and not empty_password: + elif salt.utils.platform.is_darwin() and password and not empty_password: if not __salt__['shadow.set_password'](name, password): ret['comment'] = 'User {0} created but failed to set' \ ' password to' \ diff --git a/salt/states/virt.py b/salt/states/virt.py index 89fa6f61aa0..c0286674701 100644 --- a/salt/states/virt.py +++ b/salt/states/virt.py @@ -13,10 +13,9 @@ for the generation and signing of certificates for systems running libvirt: ''' from __future__ import absolute_import -# Import python libs +# Import Python libs import fnmatch import os -from salt.ext import six try: import libvirt # pylint: disable=import-error @@ -24,10 +23,14 @@ try: except ImportError: HAS_LIBVIRT = False -# Import salt libs -import salt.utils +# Import Salt libs +import salt.utils.args +import salt.utils.files from salt.exceptions import CommandExecutionError +# Import 3rd-party libs +from salt.ext import six + __virtualname__ = 'virt' @@ -114,7 +117,7 @@ def keys(name, basepath='/etc/pki', **kwargs): if not os.path.exists(os.path.dirname(paths[key])): os.makedirs(os.path.dirname(paths[key])) if os.path.isfile(paths[key]): - with salt.utils.fopen(paths[key], 'r') as fp_: + with salt.utils.files.fopen(paths[key], 'r') as fp_: if fp_.read() != pillar[p_key]: ret['changes'][key] = 'update' else: @@ -128,7 +131,7 @@ def keys(name, basepath='/etc/pki', **kwargs): ret['changes'] = {} else: for key in ret['changes']: - with salt.utils.fopen(paths[key], 'w+') as fp_: + with salt.utils.files.fopen(paths[key], 'w+') as fp_: fp_.write(pillar['libvirt.{0}.pem'.format(key)]) ret['comment'] = 'Updated libvirt certs and keys' @@ -227,7 +230,7 @@ def running(name, **kwargs): 'comment': '{0} is running'.format(name) } - kwargs = salt.utils.clean_kwargs(**kwargs) + kwargs = salt.utils.args.clean_kwargs(**kwargs) cpu = kwargs.pop('cpu', False) mem = kwargs.pop('mem', False) image = kwargs.pop('image', False) @@ -240,7 +243,7 @@ def running(name, **kwargs): ret['changes'][name] = 'Domain started' ret['comment'] = 'Domain {0} started'.format(name) except CommandExecutionError: - kwargs = salt.utils.clean_kwargs(**kwargs) + kwargs = salt.utils.args.clean_kwargs(**kwargs) __salt__['virt.init'](name, cpu=cpu, mem=mem, image=image, **kwargs) ret['changes'][name] = 'Domain defined and started' ret['comment'] = 'Domain {0} defined and started'.format(name) diff --git a/salt/states/virtualenv_mod.py b/salt/states/virtualenv_mod.py index 81ee1707e87..364699051fa 100644 --- a/salt/states/virtualenv_mod.py +++ b/salt/states/virtualenv_mod.py @@ -6,17 +6,20 @@ Setup of Python virtualenv sandboxes. ''' from __future__ import absolute_import -# Import python libs +# Import Python libs import logging import os -# Import salt libs +# Import Salt libs import salt.version -import salt.utils - +import salt.utils # Can be removed once alias_function is moved +import salt.utils.platform +import salt.utils.versions from salt.exceptions import CommandExecutionError, CommandNotFoundError +# Import 3rd-party libs from salt.ext import six + log = logging.getLogger(__name__) # Define the module's virtual name @@ -134,7 +137,7 @@ def managed(name, ret['comment'] = 'Virtualenv was not detected on this system' return ret - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): venv_py = os.path.join(name, 'Scripts', 'python.exe') else: venv_py = os.path.join(name, 'bin', 'python') @@ -225,7 +228,7 @@ def managed(name, if use_wheel: min_version = '1.4' cur_version = __salt__['pip.version'](bin_env=name) - if not salt.utils.compare_versions(ver1=cur_version, oper='>=', + if not salt.utils.versions.compare(ver1=cur_version, oper='>=', ver2=min_version): ret['result'] = False ret['comment'] = ('The \'use_wheel\' option is only supported in ' @@ -236,7 +239,7 @@ def managed(name, if no_use_wheel: min_version = '1.4' cur_version = __salt__['pip.version'](bin_env=name) - if not salt.utils.compare_versions(ver1=cur_version, oper='>=', + if not salt.utils.versions.compare(ver1=cur_version, oper='>=', ver2=min_version): ret['result'] = False ret['comment'] = ('The \'no_use_wheel\' option is only supported ' diff --git a/salt/states/win_certutil.py b/salt/states/win_certutil.py index ea65622b374..94f5ba95d59 100644 --- a/salt/states/win_certutil.py +++ b/salt/states/win_certutil.py @@ -11,13 +11,12 @@ Install certificates to the Windows Certificate Manager certutil.add_store: - store: TrustedPublisher ''' - -# Import python libs +# Import Python libs from __future__ import absolute_import import logging -# Import salt libs -import salt.utils +# Import Salt libs +import salt.utils.platform log = logging.getLogger(__name__) __virtualname__ = "certutil" @@ -27,7 +26,7 @@ def __virtual__(): ''' Only work on Windows ''' - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): return __virtualname__ return False diff --git a/salt/states/win_dism.py b/salt/states/win_dism.py index 4b73d4e7112..c7575f1b102 100644 --- a/salt/states/win_dism.py +++ b/salt/states/win_dism.py @@ -15,12 +15,13 @@ Install windows features/capabilties with DISM ''' from __future__ import absolute_import -# Import python libs +# Import Python libs import logging import os -# Import salt libs +# Import Salt libs import salt.utils +import salt.utils.platform log = logging.getLogger(__name__) __virtualname__ = "dism" @@ -30,7 +31,7 @@ def __virtual__(): ''' Only work on Windows where the DISM module is available ''' - if not salt.utils.is_windows(): + if not salt.utils.platform.is_windows(): return False, 'Module only available on Windows' return __virtualname__ diff --git a/salt/states/win_iis.py b/salt/states/win_iis.py index 614236afc13..6407315c216 100644 --- a/salt/states/win_iis.py +++ b/salt/states/win_iis.py @@ -481,7 +481,6 @@ def container_setting(name, container, settings=None): :param str container: The type of IIS container. The container types are: AppPools, Sites, SslBindings :param str settings: A dictionary of the setting names and their values. - Example of usage for the ``AppPools`` container: .. code-block:: yaml @@ -495,6 +494,7 @@ def container_setting(name, container, settings=None): processModel.maxProcesses: 1 processModel.userName: TestUser processModel.password: TestPassword + processModel.identityType: SpecificUser Example of usage for the ``Sites`` container: @@ -509,6 +509,8 @@ def container_setting(name, container, settings=None): logFile.period: Daily limits.maxUrlSegments: 32 ''' + + identityType_map2string = {0: 'LocalSystem', 1: 'LocalService', 2: 'NetworkService', 3: 'SpecificUser', 4: 'ApplicationPoolIdentity'} ret = {'name': name, 'changes': {}, 'comment': str(), @@ -528,6 +530,10 @@ def container_setting(name, container, settings=None): container=container, settings=settings.keys()) for setting in settings: + # map identity type from numeric to string for comparing + if setting == 'processModel.identityType' and settings[setting] in identityType_map2string.keys(): + settings[setting] = identityType_map2string[settings[setting]] + if str(settings[setting]) != str(current_settings[setting]): ret_settings['changes'][setting] = {'old': current_settings[setting], 'new': settings[setting]} @@ -540,8 +546,8 @@ def container_setting(name, container, settings=None): ret['changes'] = ret_settings return ret - __salt__['win_iis.set_container_setting'](name=name, container=container, - settings=settings) + __salt__['win_iis.set_container_setting'](name=name, container=container, settings=settings) + new_settings = __salt__['win_iis.get_container_setting'](name=name, container=container, settings=settings.keys()) diff --git a/salt/states/win_lgpo.py b/salt/states/win_lgpo.py index 96e6ff729d1..fcedaae0106 100644 --- a/salt/states/win_lgpo.py +++ b/salt/states/win_lgpo.py @@ -112,7 +112,7 @@ import json import salt.utils.dictdiffer # Import 3rd party libs -import salt.ext.six as six +from salt.ext import six log = logging.getLogger(__name__) __virtualname__ = 'lgpo' diff --git a/salt/states/win_license.py b/salt/states/win_license.py index 34d7a7a0db9..3f2238c2f00 100644 --- a/salt/states/win_license.py +++ b/salt/states/win_license.py @@ -10,13 +10,12 @@ Install and activate windows licenses XXXXX-XXXXX-XXXXX-XXXXX-XXXXX: license.activate ''' - -# Import python libs +# Import Python libs from __future__ import absolute_import import logging # Import Salt Libs -import salt.utils +import salt.utils.platform log = logging.getLogger(__name__) __virtualname__ = 'license' @@ -26,7 +25,7 @@ def __virtual__(): ''' Only work on Windows ''' - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): return __virtualname__ return False diff --git a/salt/states/win_network.py b/salt/states/win_network.py index 63710658a74..c68a5d0a00c 100644 --- a/salt/states/win_network.py +++ b/salt/states/win_network.py @@ -61,11 +61,12 @@ default gateway using the ``gateway`` parameter: ''' from __future__ import absolute_import -# Import python libs +# Import Python libs import logging -# Import salt libs +# Import Salt libs import salt.utils +import salt.utils.platform import salt.utils.validate.net from salt.ext.six.moves import range from salt.exceptions import CommandExecutionError @@ -84,7 +85,7 @@ def __virtual__(): Confine this module to Windows systems with the required execution module available. ''' - if salt.utils.is_windows() and 'ip.get_interface' in __salt__: + if salt.utils.platform.is_windows() and 'ip.get_interface' in __salt__: return __virtualname__ return False diff --git a/salt/states/win_servermanager.py b/salt/states/win_servermanager.py index 8c0e7eadf21..b50a180f353 100644 --- a/salt/states/win_servermanager.py +++ b/salt/states/win_servermanager.py @@ -1,11 +1,18 @@ # -*- coding: utf-8 -*- ''' -Manage Windows features via the ServerManager powershell module +Manage Windows features via the ServerManager powershell module. Can install and +remove roles/features. + +:maintainer: Shane Lee +:platform: Windows Server 2008R2 or greater +:depends: win_servermanager.install +:depends: win_servermanager.remove ''' from __future__ import absolute_import # Import salt modules import salt.utils +import salt.utils.versions def __virtual__(): @@ -16,160 +23,308 @@ def __virtual__(): def installed(name, + features=None, recurse=False, - force=False, restart=False, source=None, - exclude=None): + exclude=None, + **kwargs): ''' - Install the windows feature + Install the windows feature. To install a single feature, use the ``name`` + parameter. To install multiple features, use the ``features`` parameter. - Args: - name (str): Short name of the feature (the right column in - win_servermanager.list_available) - recurse (Optional[bool]): install all sub-features as well - force (Optional[bool]): if the feature is installed but one of its - sub-features are not installed set this to True to force the - installation of the sub-features - source (Optional[str]): Path to the source files if missing from the - target system. None means that the system will use windows update - services to find the required files. Default is None - restart (Optional[bool]): Restarts the computer when installation is - complete, if required by the role/feature installed. Default is - False - exclude (Optional[str]): The name of the feature to exclude when - installing the named feature. - - restart: - Restarts the computer when installation is complete, if restarting is required by the role feature installed. - - Note: + .. note:: Some features require reboot after un/installation. If so, until the server is restarted other features can not be installed! + Args: + + name (str): + Short name of the feature (the right column in + win_servermanager.list_available). This can be a single feature or a + string of features in a comma delimited list (no spaces) + + .. note:: + A list is not allowed in the name parameter of any state. Use + the ``features`` parameter if you want to pass the features as a + list + + features (Optional[list]): + A list of features to install. If this is passed it will be used + instead of the ``name`` parameter. + + .. versionadded:: Oxygen + + recurse (Optional[bool]): + Install all sub-features as well. If the feature is installed but + one of its sub-features are not installed set this will install + additional sub-features + + source (Optional[str]): + Path to the source files if missing from the target system. None + means that the system will use windows update services to find the + required files. Default is None + + restart (Optional[bool]): + Restarts the computer when installation is complete, if required by + the role/feature installed. Default is False + + exclude (Optional[str]): + The name of the feature to exclude when installing the named + feature. This can be a single feature, a string of features in a + comma-delimited list (no spaces), or a list of features. + + .. warning:: + As there is no exclude option for the ``Add-WindowsFeature`` + or ``Install-WindowsFeature`` PowerShell commands the features + named in ``exclude`` will be installed with other sub-features + and will then be removed. **If the feature named in ``exclude`` + is not a sub-feature of one of the installed items it will still + be removed.** + Example: - Run ``salt MinionName win_servermanager.list_available`` to get a list - of available roles and features. Use the name in the right column. Do - not use the role or feature names mentioned in the PKGMGR documentation. - In this example for IIS-WebServerRole the name to be used is Web-Server. + + Do not use the role or feature names mentioned in the PKGMGR + documentation. To get a list of available roles and features run the + following command: + + .. code-block:: bash + + salt win_servermanager.list_available + + Use the name in the right column of the results. .. code-block:: yaml - ISWebserverRole: + # Installs the IIS Web Server Role (Web-Server) + IIS-WebServerRole: win_servermanager.installed: - - force: True - recurse: True - name: Web-Server + + # Install multiple features, exclude the Web-Service + install_multiple_features: + win_servermanager.installed: + - recurse: True + - features: + - RemoteAccess + - XPS-Viewer + - SNMP-Service + - exclude: + - Web-Service ''' + if 'force' in kwargs: + salt.utils.versions.warn_until( + 'Flourine', + 'Parameter \'force\' has been detected in the argument list. This' + 'parameter is no longer used and has been replaced by \'recurse\'' + 'as of Salt Oxygen. This warning will be removed in Salt Flourine.' + ) + kwargs.pop('force') + ret = {'name': name, 'result': True, 'changes': {}, 'comment': ''} + # Check if features is not passed, use name. Split commas + if features is None: + features = name.split(',') + + # Make sure features is a list, split commas + if not isinstance(features, list): + features = features.split(',') + # Determine if the feature is installed old = __salt__['win_servermanager.list_installed']() - if name not in old: - ret['changes']['feature'] = \ - '{0} will be installed recurse={1}'.format(name, recurse) - elif force and recurse: - ret['changes']['feature'] = \ - '{0} already installed but might install sub-features'.format(name) - else: - ret['comment'] = 'The feature {0} is already installed'.format(name) + + cur_feat = [] + for feature in features: + + if feature not in old: + ret['changes'][feature] = \ + 'Will be installed recurse={0}'.format(recurse) + elif recurse: + ret['changes'][feature] = \ + 'Already installed but might install sub-features' + else: + cur_feat.append(feature) + + if cur_feat: + cur_feat.insert(0, 'The following features are already installed:') + ret['comment'] = '\n- '.join(cur_feat) + + if not ret['changes']: return ret if __opts__['test']: ret['result'] = None return ret - if ret['changes']['feature']: - ret['comment'] = ret['changes']['feature'] - - ret['changes'] = {} - # Install the features - status = __salt__['win_servermanager.install'](name, - recurse=recurse, - restart=restart, - source=source, - exclude=exclude) + status = __salt__['win_servermanager.install']( + features, recurse=recurse, restart=restart, source=source, + exclude=exclude) ret['result'] = status['Success'] - if not ret['result']: - ret['comment'] = 'Failed to install {0}: {1}'\ - .format(name, status['ExitCode']) + # Show items failed to install + fail_feat = [] + new_feat = [] + rem_feat = [] + for feature in status['Features']: + # Features that failed to install or be removed + if not status['Features'][feature].get('Success', True): + fail_feat.append('- {0}'.format(feature)) + # Features that installed + elif '(exclude)' not in status['Features'][feature]['Message']: + new_feat.append('- {0}'.format(feature)) + # Show items that were removed because they were part of `exclude` + elif '(exclude)' in status['Features'][feature]['Message']: + rem_feat.append('- {0}'.format(feature)) + + if fail_feat: + fail_feat.insert(0, 'Failed to install the following:') + if new_feat: + new_feat.insert(0, 'Installed the following:') + if rem_feat: + rem_feat.insert(0, 'Removed the following (exclude):') + + ret['comment'] = '\n'.join(fail_feat + new_feat + rem_feat) + + # Get the changes new = __salt__['win_servermanager.list_installed']() - changes = salt.utils.compare_dicts(old, new) - - if changes: - ret['comment'] = 'Installed {0}'.format(name) - ret['changes'] = status - ret['changes']['feature'] = changes + ret['changes'] = salt.utils.compare_dicts(old, new) return ret -def removed(name, remove_payload=False, restart=False): +def removed(name, features=None, remove_payload=False, restart=False): ''' - Remove the windows feature + Remove the windows feature To remove a single feature, use the ``name`` + parameter. To remove multiple features, use the ``features`` parameter. Args: - name (str): Short name of the feature (the right column in - win_servermanager.list_available) - remove_payload (Optional[bool]): True will case the feature to be - removed from the side-by-side store - restart (Optional[bool]): Restarts the computer when uninstall is - complete, if required by the role/feature removed. Default is False + name (str): + Short name of the feature (the right column in + win_servermanager.list_available). This can be a single feature or a + string of features in a comma-delimited list (no spaces) - Note: - Some features require a reboot after uninstallation. If so the feature - will not be completely uninstalled until the server is restarted. + .. note:: + A list is not allowed in the name parameter of any state. Use + the ``features`` parameter if you want to pass the features as a + list + + features (Optional[list]): + A list of features to remove. If this is passed it will be used + instead of the ``name`` parameter. + + .. versionadded:: Oxygen + + remove_payload (Optional[bool]): + True will cause the feature to be removed from the side-by-side + store. To install the feature in the future you will need to + specify the ``source`` + + restart (Optional[bool]): + Restarts the computer when uninstall is complete if required by the + role/feature uninstall. Default is False + + .. note:: + Some features require a reboot after uninstall. If so the feature will + not be completely uninstalled until the server is restarted. Example: - Run ``salt MinionName win_servermanager.list_installed`` to get a list - of all features installed. Use the top name listed for each feature, not - the indented one. Do not use the role or feature names mentioned in the - PKGMGR documentation. + + Do not use the role or feature names mentioned in the PKGMGR + documentation. To get a list of available roles and features run the + following command: + + .. code-block:: bash + + salt win_servermanager.list_available + + Use the name in the right column of the results. .. code-block:: yaml - ISWebserverRole: + # Uninstall the IIS Web Server Rol (Web-Server) + IIS-WebserverRole: win_servermanager.removed: - name: Web-Server + + # Uninstall multiple features, reboot if required + uninstall_multiple_features: + win_servermanager.removed: + - features: + - RemoteAccess + - XPX-Viewer + - SNMP-Service + - restart: True ''' ret = {'name': name, 'result': True, 'changes': {}, 'comment': ''} + + # Check if features is not passed, use name. Split commas + if features is None: + features = name.split(',') + + # Make sure features is a list, split commas + if not isinstance(features, list): + features = features.split(',') + # Determine if the feature is installed old = __salt__['win_servermanager.list_installed']() - if name in old: - ret['changes']['feature'] = '{0} will be removed'.format(name) - else: - ret['comment'] = 'The feature {0} is not installed'.format(name) + + rem_feat = [] + for feature in features: + + if feature in old: + ret['changes'][feature] = 'Will be removed' + else: + rem_feat.append(feature) + + if rem_feat: + rem_feat.insert(0, 'The following features are not installed:') + ret['comment'] = '\n- '.join(rem_feat) + + if not ret['changes']: return ret if __opts__['test']: ret['result'] = None return ret - ret['changes'] = {} - # Remove the features - status = __salt__['win_servermanager.remove'](name, remove_payload, restart) + status = __salt__['win_servermanager.remove']( + features, remove_payload=remove_payload, restart=restart) ret['result'] = status['Success'] - if not ret['result']: - ret['comment'] = 'Failed to uninstall the feature {0}'\ - .format(status['ExitCode']) + # Some items failed to uninstall + fail_feat = [] + rem_feat = [] + for feature in status['Features']: + # Use get because sometimes 'Success' isn't defined such as when the + # feature is already uninstalled + if not status['Features'][feature].get('Success', True): + # Show items that failed to uninstall + fail_feat.append('- {0}'.format(feature)) + else: + # Show items that uninstalled + rem_feat.append('- {0}'.format(feature)) + + if fail_feat: + fail_feat.insert(0, 'Failed to remove the following:') + if rem_feat: + rem_feat.insert(0, 'Removed the following:') + + ret['comment'] = '\n'.join(fail_feat + rem_feat) + + # Get the changes new = __salt__['win_servermanager.list_installed']() - changes = salt.utils.compare_dicts(old, new) - - if changes: - ret['comment'] = 'Removed {0}'.format(name) - ret['changes'] = status - ret['changes']['feature'] = changes + ret['changes'] = salt.utils.compare_dicts(old, new) return ret diff --git a/salt/states/win_smtp_server.py b/salt/states/win_smtp_server.py index a6591e255ee..79a2470c76f 100644 --- a/salt/states/win_smtp_server.py +++ b/salt/states/win_smtp_server.py @@ -7,10 +7,10 @@ Module for managing IIS SMTP server configuration on Windows servers. from __future__ import absolute_import # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six # Import salt libs -import salt.utils +import salt.utils.args _DEFAULT_SERVER = 'SmtpSvc/1' @@ -39,7 +39,7 @@ def _normalize_server_settings(**settings): Convert setting values that has been improperly converted to a dict back to a string. ''' ret = dict() - settings = salt.utils.clean_kwargs(**settings) + settings = salt.utils.args.clean_kwargs(**settings) for setting in settings: if isinstance(settings[setting], dict): diff --git a/salt/states/win_system.py b/salt/states/win_system.py index 5e11dfbe281..59a6d23fcd3 100644 --- a/salt/states/win_system.py +++ b/salt/states/win_system.py @@ -16,14 +16,14 @@ description. This is Erik's computer, don't touch!: system.computer_desc: [] ''' - from __future__ import absolute_import -# Import python libs +# Import Python libs import logging -# Import salt libs +# Import Salt libs import salt.utils +import salt.utils.platform log = logging.getLogger(__name__) @@ -35,7 +35,7 @@ def __virtual__(): ''' This only supports Windows ''' - if salt.utils.is_windows() and 'system.get_computer_desc' in __salt__: + if salt.utils.platform.is_windows() and 'system.get_computer_desc' in __salt__: return __virtualname__ return False diff --git a/salt/states/win_update.py b/salt/states/win_update.py index fca29fe252a..327a1449325 100644 --- a/salt/states/win_update.py +++ b/salt/states/win_update.py @@ -71,7 +71,7 @@ import logging # Import 3rd-party libs # pylint: disable=import-error -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import range # pylint: disable=redefined-builtin try: import win32com.client @@ -81,8 +81,9 @@ except ImportError: HAS_DEPENDENCIES = False # pylint: enable=import-error -# Import salt libs -import salt.utils +# Import Salt libs +import salt.utils.platform +import salt.utils.versions log = logging.getLogger(__name__) @@ -91,7 +92,7 @@ def __virtual__(): ''' Only works on Windows systems ''' - if salt.utils.is_windows() and HAS_DEPENDENCIES: + if salt.utils.platform.is_windows() and HAS_DEPENDENCIES: return True return False @@ -465,7 +466,7 @@ def installed(name, categories=None, skips=None, retries=10): deprecation_msg = 'The \'win_update\' module is deprecated, and will be ' \ 'removed in Salt Fluorine. Please use the \'win_wua\' ' \ 'module instead.' - salt.utils.warn_until('Fluorine', deprecation_msg) + salt.utils.versions.warn_until('Fluorine', deprecation_msg) ret.setdefault('warnings', []).append(deprecation_msg) if not categories: categories = [name] @@ -549,7 +550,7 @@ def downloaded(name, categories=None, skips=None, retries=10): deprecation_msg = 'The \'win_update\' module is deprecated, and will be ' \ 'removed in Salt Fluorine. Please use the \'win_wua\' ' \ 'module instead.' - salt.utils.warn_until('Fluorine', deprecation_msg) + salt.utils.versions.warn_until('Fluorine', deprecation_msg) ret.setdefault('warnings', []).append(deprecation_msg) if not categories: diff --git a/salt/states/win_wua.py b/salt/states/win_wua.py index 835dea28cee..ab43b656544 100644 --- a/salt/states/win_wua.py +++ b/salt/states/win_wua.py @@ -56,6 +56,7 @@ import logging # Import Salt libs from salt.ext import six import salt.utils +import salt.utils.platform import salt.utils.win_update log = logging.getLogger(__name__) @@ -67,7 +68,7 @@ def __virtual__(): ''' Only valid on Windows machines ''' - if not salt.utils.is_windows(): + if not salt.utils.platform.is_windows(): return False, 'WUA: Only available on Window systems' if not salt.utils.win_update.HAS_PYWIN32: diff --git a/salt/states/x509.py b/salt/states/x509.py index bcd08972b7d..58fbabac347 100644 --- a/salt/states/x509.py +++ b/salt/states/x509.py @@ -166,7 +166,7 @@ import salt.exceptions import salt.utils # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six try: from M2Crypto.RSA import RSAError diff --git a/salt/states/zabbix_usergroup.py b/salt/states/zabbix_usergroup.py index a5d68b1546a..292a2aafc9c 100644 --- a/salt/states/zabbix_usergroup.py +++ b/salt/states/zabbix_usergroup.py @@ -84,7 +84,7 @@ def present(name, **kwargs): for right in kwargs['rights']: for key in right: right[key] = str(right[key]) - if cmp(sorted(kwargs['rights']), sorted(usergroup['rights'])) != 0: + if sorted(kwargs['rights']) != sorted(usergroup['rights']): update_rights = True else: update_rights = True diff --git a/salt/states/zone.py b/salt/states/zone.py index 22900f89c54..02a324111ef 100644 --- a/salt/states/zone.py +++ b/salt/states/zone.py @@ -116,6 +116,7 @@ import logging # Import Salt libs import salt.utils +import salt.utils.args import salt.utils.files import salt.utils.atomicfile from salt.modules.zonecfg import _parse_value, _zonecfg_resource_default_selectors @@ -284,7 +285,7 @@ def resource_present(name, resource_type, resource_selector_property, resource_s 'comment': ''} # sanitize input - kwargs = salt.utils.clean_kwargs(**kwargs) + kwargs = salt.utils.args.clean_kwargs(**kwargs) resource_selector_value = _parse_value(resource_selector_value) for k, v in kwargs.items(): kwargs[k] = _parse_value(kwargs[k]) diff --git a/salt/syspaths.py b/salt/syspaths.py index 9145fff2603..7aeb6e45e46 100644 --- a/salt/syspaths.py +++ b/salt/syspaths.py @@ -22,6 +22,9 @@ from __future__ import absolute_import import sys import os.path +# Import Salt libs +from salt.utils.locales import sdecode + __PLATFORM = sys.platform.lower() @@ -30,10 +33,10 @@ try: # installation time. import salt._syspaths as __generated_syspaths # pylint: disable=no-name-in-module except ImportError: - import imp - __generated_syspaths = imp.new_module('salt._syspaths') + import types + __generated_syspaths = types.ModuleType('salt._syspaths') for key in ('ROOT_DIR', 'CONFIG_DIR', 'CACHE_DIR', 'SOCK_DIR', - 'SRV_ROOT_DIR', 'BASE_FILE_ROOTS_DIR', + 'SRV_ROOT_DIR', 'BASE_FILE_ROOTS_DIR', 'HOME_DIR', 'BASE_PILLAR_ROOTS_DIR', 'BASE_THORIUM_ROOTS_DIR', 'BASE_MASTER_ROOTS_DIR', 'LOGS_DIR', 'PIDFILE_DIR', 'SPM_FORMULA_PATH', 'SPM_PILLAR_PATH', 'SPM_REACTOR_PATH', @@ -42,118 +45,122 @@ except ImportError: # Let's find out the path of this module -if 'SETUP_DIRNAME' in globals(): +if u'SETUP_DIRNAME' in globals(): # This is from the exec() call in Salt's setup.py - __THIS_FILE = os.path.join(SETUP_DIRNAME, 'salt', 'syspaths.py') # pylint: disable=E0602 + __THIS_FILE = os.path.join(SETUP_DIRNAME, u'salt', u'syspaths.py') # pylint: disable=E0602 else: __THIS_FILE = __file__ # These values are always relative to salt's installation directory INSTALL_DIR = os.path.dirname(os.path.realpath(__THIS_FILE)) -CLOUD_DIR = os.path.join(INSTALL_DIR, 'cloud') -BOOTSTRAP = os.path.join(CLOUD_DIR, 'deploy', 'bootstrap-salt.sh') +CLOUD_DIR = os.path.join(INSTALL_DIR, u'cloud') +BOOTSTRAP = os.path.join(CLOUD_DIR, u'deploy', u'bootstrap-salt.sh') ROOT_DIR = __generated_syspaths.ROOT_DIR if ROOT_DIR is None: # The installation time value was not provided, let's define the default - if __PLATFORM.startswith('win'): - ROOT_DIR = r'c:\salt' + if __PLATFORM.startswith(u'win'): + ROOT_DIR = sdecode(r'c:\salt') # future lint: disable=non-unicode-string else: - ROOT_DIR = '/' + ROOT_DIR = u'/' CONFIG_DIR = __generated_syspaths.CONFIG_DIR if CONFIG_DIR is None: - if __PLATFORM.startswith('win'): - CONFIG_DIR = os.path.join(ROOT_DIR, 'conf') - elif 'freebsd' in __PLATFORM: - CONFIG_DIR = os.path.join(ROOT_DIR, 'usr', 'local', 'etc', 'salt') - elif 'netbsd' in __PLATFORM: - CONFIG_DIR = os.path.join(ROOT_DIR, 'usr', 'pkg', 'etc', 'salt') - elif 'sunos5' in __PLATFORM: - CONFIG_DIR = os.path.join(ROOT_DIR, 'opt', 'local', 'etc', 'salt') + if __PLATFORM.startswith(u'win'): + CONFIG_DIR = os.path.join(ROOT_DIR, u'conf') + elif u'freebsd' in __PLATFORM: + CONFIG_DIR = os.path.join(ROOT_DIR, u'usr', u'local', u'etc', u'salt') + elif u'netbsd' in __PLATFORM: + CONFIG_DIR = os.path.join(ROOT_DIR, u'usr', u'pkg', u'etc', u'salt') + elif u'sunos5' in __PLATFORM: + CONFIG_DIR = os.path.join(ROOT_DIR, u'opt', u'local', u'etc', u'salt') else: - CONFIG_DIR = os.path.join(ROOT_DIR, 'etc', 'salt') + CONFIG_DIR = os.path.join(ROOT_DIR, u'etc', u'salt') SHARE_DIR = __generated_syspaths.SHARE_DIR if SHARE_DIR is None: - if __PLATFORM.startswith('win'): - SHARE_DIR = os.path.join(ROOT_DIR, 'share') - elif 'freebsd' in __PLATFORM: - SHARE_DIR = os.path.join(ROOT_DIR, 'usr', 'local', 'share', 'salt') - elif 'netbsd' in __PLATFORM: - SHARE_DIR = os.path.join(ROOT_DIR, 'usr', 'share', 'salt') - elif 'sunos5' in __PLATFORM: - SHARE_DIR = os.path.join(ROOT_DIR, 'usr', 'share', 'salt') + if __PLATFORM.startswith(u'win'): + SHARE_DIR = os.path.join(ROOT_DIR, u'share') + elif u'freebsd' in __PLATFORM: + SHARE_DIR = os.path.join(ROOT_DIR, u'usr', u'local', u'share', u'salt') + elif u'netbsd' in __PLATFORM: + SHARE_DIR = os.path.join(ROOT_DIR, u'usr', u'share', u'salt') + elif u'sunos5' in __PLATFORM: + SHARE_DIR = os.path.join(ROOT_DIR, u'usr', u'share', u'salt') else: - SHARE_DIR = os.path.join(ROOT_DIR, 'usr', 'share', 'salt') + SHARE_DIR = os.path.join(ROOT_DIR, u'usr', u'share', u'salt') CACHE_DIR = __generated_syspaths.CACHE_DIR if CACHE_DIR is None: - CACHE_DIR = os.path.join(ROOT_DIR, 'var', 'cache', 'salt') + CACHE_DIR = os.path.join(ROOT_DIR, u'var', u'cache', u'salt') SOCK_DIR = __generated_syspaths.SOCK_DIR if SOCK_DIR is None: - SOCK_DIR = os.path.join(ROOT_DIR, 'var', 'run', 'salt') + SOCK_DIR = os.path.join(ROOT_DIR, u'var', u'run', u'salt') SRV_ROOT_DIR = __generated_syspaths.SRV_ROOT_DIR if SRV_ROOT_DIR is None: - SRV_ROOT_DIR = os.path.join(ROOT_DIR, 'srv') + SRV_ROOT_DIR = os.path.join(ROOT_DIR, u'srv') BASE_FILE_ROOTS_DIR = __generated_syspaths.BASE_FILE_ROOTS_DIR if BASE_FILE_ROOTS_DIR is None: - BASE_FILE_ROOTS_DIR = os.path.join(SRV_ROOT_DIR, 'salt') + BASE_FILE_ROOTS_DIR = os.path.join(SRV_ROOT_DIR, u'salt') BASE_PILLAR_ROOTS_DIR = __generated_syspaths.BASE_PILLAR_ROOTS_DIR if BASE_PILLAR_ROOTS_DIR is None: - BASE_PILLAR_ROOTS_DIR = os.path.join(SRV_ROOT_DIR, 'pillar') + BASE_PILLAR_ROOTS_DIR = os.path.join(SRV_ROOT_DIR, u'pillar') BASE_THORIUM_ROOTS_DIR = __generated_syspaths.BASE_THORIUM_ROOTS_DIR if BASE_THORIUM_ROOTS_DIR is None: - BASE_THORIUM_ROOTS_DIR = os.path.join(SRV_ROOT_DIR, 'thorium') + BASE_THORIUM_ROOTS_DIR = os.path.join(SRV_ROOT_DIR, u'thorium') BASE_MASTER_ROOTS_DIR = __generated_syspaths.BASE_MASTER_ROOTS_DIR if BASE_MASTER_ROOTS_DIR is None: - BASE_MASTER_ROOTS_DIR = os.path.join(SRV_ROOT_DIR, 'salt-master') + BASE_MASTER_ROOTS_DIR = os.path.join(SRV_ROOT_DIR, u'salt-master') LOGS_DIR = __generated_syspaths.LOGS_DIR if LOGS_DIR is None: - LOGS_DIR = os.path.join(ROOT_DIR, 'var', 'log', 'salt') + LOGS_DIR = os.path.join(ROOT_DIR, u'var', u'log', u'salt') PIDFILE_DIR = __generated_syspaths.PIDFILE_DIR if PIDFILE_DIR is None: - PIDFILE_DIR = os.path.join(ROOT_DIR, 'var', 'run') + PIDFILE_DIR = os.path.join(ROOT_DIR, u'var', u'run') SPM_FORMULA_PATH = __generated_syspaths.SPM_FORMULA_PATH if SPM_FORMULA_PATH is None: - SPM_FORMULA_PATH = os.path.join(SRV_ROOT_DIR, 'spm', 'salt') + SPM_FORMULA_PATH = os.path.join(SRV_ROOT_DIR, u'spm', u'salt') SPM_PILLAR_PATH = __generated_syspaths.SPM_PILLAR_PATH if SPM_PILLAR_PATH is None: - SPM_PILLAR_PATH = os.path.join(SRV_ROOT_DIR, 'spm', 'pillar') + SPM_PILLAR_PATH = os.path.join(SRV_ROOT_DIR, u'spm', u'pillar') SPM_REACTOR_PATH = __generated_syspaths.SPM_REACTOR_PATH if SPM_REACTOR_PATH is None: - SPM_REACTOR_PATH = os.path.join(SRV_ROOT_DIR, 'spm', 'reactor') + SPM_REACTOR_PATH = os.path.join(SRV_ROOT_DIR, u'spm', u'reactor') + +HOME_DIR = __generated_syspaths.HOME_DIR +if HOME_DIR is None: + HOME_DIR = os.path.expanduser('~') __all__ = [ - 'ROOT_DIR', - 'SHARE_DIR', - 'CONFIG_DIR', - 'CACHE_DIR', - 'SOCK_DIR', - 'SRV_ROOT_DIR', - 'BASE_FILE_ROOTS_DIR', - 'BASE_PILLAR_ROOTS_DIR', - 'BASE_MASTER_ROOTS_DIR', - 'BASE_THORIUM_ROOTS_DIR', - 'LOGS_DIR', - 'PIDFILE_DIR', - 'INSTALL_DIR', - 'CLOUD_DIR', - 'BOOTSTRAP', - 'SPM_FORMULA_PATH', - 'SPM_PILLAR_PATH', - 'SPM_REACTOR_PATH' + u'ROOT_DIR', + u'SHARE_DIR', + u'CONFIG_DIR', + u'CACHE_DIR', + u'SOCK_DIR', + u'SRV_ROOT_DIR', + u'BASE_FILE_ROOTS_DIR', + u'BASE_PILLAR_ROOTS_DIR', + u'BASE_MASTER_ROOTS_DIR', + u'BASE_THORIUM_ROOTS_DIR', + u'LOGS_DIR', + u'PIDFILE_DIR', + u'INSTALL_DIR', + u'CLOUD_DIR', + u'BOOTSTRAP', + u'SPM_FORMULA_PATH', + u'SPM_PILLAR_PATH', + u'SPM_REACTOR_PATH' ] diff --git a/salt/template.py b/salt/template.py index a6e6085cb68..c2a3de7582f 100644 --- a/salt/template.py +++ b/salt/template.py @@ -5,17 +5,20 @@ Manage basic template commands from __future__ import absolute_import -# Import python libs +# Import Python libs import time import os import codecs import logging -# Import salt libs -import salt.utils +# Import Salt libs import salt.utils.files +import salt.utils.locales import salt.utils.stringio -import salt.ext.six as six +import salt.utils.versions + +# Import 3rd-party libs +from salt.ext import six from salt.ext.six.moves import StringIO log = logging.getLogger(__name__) @@ -24,7 +27,7 @@ log = logging.getLogger(__name__) # FIXME: we should make the default encoding of a .sls file a configurable # option in the config, and default it to 'utf-8'. # -SLS_ENCODING = 'utf-8' # this one has no BOM. +SLS_ENCODING = u'utf-8' # this one has no BOM. SLS_ENCODER = codecs.getencoder(SLS_ENCODING) @@ -33,9 +36,9 @@ def compile_template(template, default, blacklist, whitelist, - saltenv='base', - sls='', - input_data='', + saltenv=u'base', + sls=u'', + input_data=u'', **kwargs): ''' Take the path to a template and return the high data structure @@ -45,29 +48,24 @@ def compile_template(template, # if any error occurs, we return an empty dictionary ret = {} - log.debug('compile template: {0}'.format(template)) + log.debug(u'compile template: %s', template) - if 'env' in kwargs: - salt.utils.warn_until( - 'Oxygen', - 'Parameter \'env\' has been detected in the argument list. This ' - 'parameter is no longer used and has been replaced by \'saltenv\' ' - 'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.' - ) - kwargs.pop('env') + if u'env' in kwargs: + # "env" is not supported; Use "saltenv". + kwargs.pop(u'env') - if template != ':string:': + if template != u':string:': # Template was specified incorrectly if not isinstance(template, six.string_types): - log.error('Template was specified incorrectly: {0}'.format(template)) + log.error(u'Template was specified incorrectly: %s', template) return ret # Template does not exist if not os.path.isfile(template): - log.error('Template does not exist: {0}'.format(template)) + log.error(u'Template does not exist: %s', template) return ret # Template is an empty file - if salt.utils.is_empty(template): - log.debug('Template is an empty file: {0}'.format(template)) + if salt.utils.files.is_empty(template): + log.debug(u'Template is an empty file: %s', template) return ret with codecs.open(template, encoding=SLS_ENCODING) as ifile: @@ -75,13 +73,13 @@ def compile_template(template, input_data = ifile.read() if not input_data.strip(): # Template is nothing but whitespace - log.error('Template is nothing but whitespace: {0}'.format(template)) + log.error(u'Template is nothing but whitespace: %s', template) return ret # Get the list of render funcs in the render pipe line. render_pipe = template_shebang(template, renderers, default, blacklist, whitelist, input_data) - windows_newline = '\r\n' in input_data + windows_newline = u'\r\n' in input_data input_data = StringIO(input_data) for render, argline in render_pipe: @@ -90,15 +88,14 @@ def compile_template(template, render_kwargs = dict(renderers=renderers, tmplpath=template) render_kwargs.update(kwargs) if argline: - render_kwargs['argline'] = argline + render_kwargs[u'argline'] = argline start = time.time() ret = render(input_data, saltenv, sls, **render_kwargs) log.profile( - 'Time (in seconds) to render \'{0}\' using \'{1}\' renderer: {2}'.format( - template, - render.__module__.split('.')[-1], - time.time() - start - ) + u'Time (in seconds) to render \'%s\' using \'%s\' renderer: %s', + template, + render.__module__.split(u'.')[-1], + time.time() - start ) if ret is None: # The file is empty or is being written elsewhere @@ -110,10 +107,11 @@ def compile_template(template, # yaml, mako, or another engine which renders to a data # structure) we don't want to log this. if salt.utils.stringio.is_readable(ret): - log.debug('Rendered data from file: {0}:\n{1}'.format( + log.debug( + u'Rendered data from file: %s:\n%s', template, - ret.read())) # pylint: disable=no-member - ret.seek(0) # pylint: disable=no-member + salt.utils.locales.sdecode(ret.read())) # pylint: disable=no-member + ret.seek(0) # pylint: disable=no-member # Preserve newlines from original template if windows_newline: @@ -125,8 +123,8 @@ def compile_template(template, contents = ret if isinstance(contents, six.string_types): - if '\r\n' not in contents: - contents = contents.replace('\n', '\r\n') + if u'\r\n' not in contents: + contents = contents.replace(u'\n', u'\r\n') ret = StringIO(contents) if is_stringio else contents else: if is_stringio: @@ -140,7 +138,7 @@ def compile_template_str(template, renderers, default, blacklist, whitelist): derived from the template. ''' fn_ = salt.utils.files.mkstemp() - with salt.utils.fopen(fn_, 'wb') as ofile: + with salt.utils.files.fopen(fn_, u'wb') as ofile: ofile.write(SLS_ENCODER(template)[0]) return compile_template(fn_, renderers, default, blacklist, whitelist) @@ -163,16 +161,16 @@ def template_shebang(template, renderers, default, blacklist, whitelist, input_d #!mako|yaml_odict|stateconf ''' - line = '' + line = u'' # Open up the first line of the sls template - if template == ':string:': + if template == u':string:': line = input_data.split()[0] else: - with salt.utils.fopen(template, 'r') as ifile: + with salt.utils.files.fopen(template, u'r') as ifile: line = ifile.readline() # Check if it starts with a shebang and not a path - if line.startswith('#!') and not line.startswith('#!/'): + if line.startswith(u'#!') and not line.startswith(u'#!/'): # pull out the shebang data # If the shebang does not contain recognized/not-blacklisted/whitelisted # renderers, do not fall back to the default renderer @@ -186,18 +184,18 @@ def template_shebang(template, renderers, default, blacklist, whitelist, input_d # OLD_STYLE_RENDERERS = {} -for comb in ('yaml_jinja', - 'yaml_mako', - 'yaml_wempy', - 'json_jinja', - 'json_mako', - 'json_wempy', - 'yamlex_jinja', - 'yamlexyamlex_mako', - 'yamlexyamlex_wempy'): +for comb in (u'yaml_jinja', + u'yaml_mako', + u'yaml_wempy', + u'json_jinja', + u'json_mako', + u'json_wempy', + u'yamlex_jinja', + u'yamlexyamlex_mako', + u'yamlexyamlex_wempy'): - fmt, tmpl = comb.split('_') - OLD_STYLE_RENDERERS[comb] = '{0}|{1}'.format(tmpl, fmt) + fmt, tmpl = comb.split(u'_') + OLD_STYLE_RENDERERS[comb] = u'{0}|{1}'.format(tmpl, fmt) def check_render_pipe_str(pipestr, renderers, blacklist, whitelist): @@ -208,24 +206,25 @@ def check_render_pipe_str(pipestr, renderers, blacklist, whitelist): ''' if pipestr is None: return [] - parts = [r.strip() for r in pipestr.split('|')] + parts = [r.strip() for r in pipestr.split(u'|')] # Note: currently, | is not allowed anywhere in the shebang line except # as pipes between renderers. results = [] try: if parts[0] == pipestr and pipestr in OLD_STYLE_RENDERERS: - parts = OLD_STYLE_RENDERERS[pipestr].split('|') + parts = OLD_STYLE_RENDERERS[pipestr].split(u'|') for part in parts: - name, argline = (part + ' ').split(' ', 1) + name, argline = (part + u' ').split(u' ', 1) if whitelist and name not in whitelist or \ blacklist and name in blacklist: log.warning( - 'The renderer "{0}" is disallowed by configuration and ' - 'will be skipped.'.format(name)) + u'The renderer "%s" is disallowed by configuration and ' + u'will be skipped.', name + ) continue results.append((renderers[name], argline.strip())) return results except KeyError: - log.error('The renderer "{0}" is not available'.format(pipestr)) + log.error(u'The renderer "%s" is not available', pipestr) return [] diff --git a/salt/templates/rh_ip/rh7_eth.jinja b/salt/templates/rh_ip/rh7_eth.jinja index c9c50cd8cd7..741b9649466 100644 --- a/salt/templates/rh_ip/rh7_eth.jinja +++ b/salt/templates/rh_ip/rh7_eth.jinja @@ -15,6 +15,8 @@ DEVICE="{{name}}" {%endif%}{% if onparent %}ONPARENT={{onparent}} {%endif%}{% if ipv4_failure_fatal %}IPV4_FAILURE_FATAL="{{ipv4_failure_fatal}}" {%endif%}{% if ipaddr %}IPADDR="{{ipaddr}}" +{%endif%}{% if ipaddr_start %}IPADDR_START="{{ipaddr_start}}" +{%endif%}{% if ipaddr_end %}IPADDR_END="{{ipaddr_end}}" {%endif%}{% if netmask %}NETMASK="{{netmask}}" {%endif%}{% if prefix %}PREFIX="{{prefix}}" {%endif%}{% if gateway %}GATEWAY="{{gateway}}" diff --git a/salt/textformat.py b/salt/textformat.py index b50a9f9c985..828b6282459 100644 --- a/salt/textformat.py +++ b/salt/textformat.py @@ -3,98 +3,102 @@ ANSI escape code utilities, see http://www.ecma-international.org/publications/files/ECMA-ST/Ecma-048.pdf ''' +from __future__ import absolute_import -graph_prefix = '\x1b[' -graph_suffix = 'm' +# Import 3rd-party libs +from salt.ext import six + +graph_prefix = u'\x1b[' +graph_suffix = u'm' codes = { - 'reset': '0', + u'reset': u'0', - 'bold': '1', - 'faint': '2', - 'italic': '3', - 'underline': '4', - 'blink': '5', - 'slow_blink': '5', - 'fast_blink': '6', - 'inverse': '7', - 'conceal': '8', - 'strike': '9', + u'bold': u'1', + u'faint': u'2', + u'italic': u'3', + u'underline': u'4', + u'blink': u'5', + u'slow_blink': u'5', + u'fast_blink': u'6', + u'inverse': u'7', + u'conceal': u'8', + u'strike': u'9', - 'primary_font': '10', - 'reset_font': '10', - 'font_0': '10', - 'font_1': '11', - 'font_2': '12', - 'font_3': '13', - 'font_4': '14', - 'font_5': '15', - 'font_6': '16', - 'font_7': '17', - 'font_8': '18', - 'font_9': '19', - 'fraktur': '20', + u'primary_font': u'10', + u'reset_font': u'10', + u'font_0': u'10', + u'font_1': u'11', + u'font_2': u'12', + u'font_3': u'13', + u'font_4': u'14', + u'font_5': u'15', + u'font_6': u'16', + u'font_7': u'17', + u'font_8': u'18', + u'font_9': u'19', + u'fraktur': u'20', - 'double_underline': '21', - 'end_bold': '21', - 'normal_intensity': '22', - 'end_italic': '23', - 'end_fraktur': '23', - 'end_underline': '24', # single or double - 'end_blink': '25', - 'end_inverse': '27', - 'end_conceal': '28', - 'end_strike': '29', + u'double_underline': u'21', + u'end_bold': u'21', + u'normal_intensity': u'22', + u'end_italic': u'23', + u'end_fraktur': u'23', + u'end_underline': u'24', # single or double + u'end_blink': u'25', + u'end_inverse': u'27', + u'end_conceal': u'28', + u'end_strike': u'29', - 'black': '30', - 'red': '31', - 'green': '32', - 'yellow': '33', - 'blue': '34', - 'magenta': '35', - 'cyan': '36', - 'white': '37', - 'extended': '38', - 'default': '39', + u'black': u'30', + u'red': u'31', + u'green': u'32', + u'yellow': u'33', + u'blue': u'34', + u'magenta': u'35', + u'cyan': u'36', + u'white': u'37', + u'extended': u'38', + u'default': u'39', - 'fg_black': '30', - 'fg_red': '31', - 'fg_green': '32', - 'fg_yellow': '33', - 'fg_blue': '34', - 'fg_magenta': '35', - 'fg_cyan': '36', - 'fg_white': '37', - 'fg_extended': '38', - 'fg_default': '39', + u'fg_black': u'30', + u'fg_red': u'31', + u'fg_green': u'32', + u'fg_yellow': u'33', + u'fg_blue': u'34', + u'fg_magenta': u'35', + u'fg_cyan': u'36', + u'fg_white': u'37', + u'fg_extended': u'38', + u'fg_default': u'39', - 'bg_black': '40', - 'bg_red': '41', - 'bg_green': '42', - 'bg_yellow': '44', - 'bg_blue': '44', - 'bg_magenta': '45', - 'bg_cyan': '46', - 'bg_white': '47', - 'bg_extended': '48', - 'bg_default': '49', + u'bg_black': u'40', + u'bg_red': u'41', + u'bg_green': u'42', + u'bg_yellow': u'44', + u'bg_blue': u'44', + u'bg_magenta': u'45', + u'bg_cyan': u'46', + u'bg_white': u'47', + u'bg_extended': u'48', + u'bg_default': u'49', - 'frame': '51', - 'encircle': '52', - 'overline': '53', - 'end_frame': '54', - 'end_encircle': '54', - 'end_overline': '55', + u'frame': u'51', + u'encircle': u'52', + u'overline': u'53', + u'end_frame': u'54', + u'end_encircle': u'54', + u'end_overline': u'55', - 'ideogram_underline': '60', - 'right_line': '60', - 'ideogram_double_underline': '61', - 'right_double_line': '61', - 'ideogram_overline': '62', - 'left_line': '62', - 'ideogram_double_overline': '63', - 'left_double_line': '63', - 'ideogram_stress': '64', - 'reset_ideogram': '65' + u'ideogram_underline': u'60', + u'right_line': u'60', + u'ideogram_double_underline': u'61', + u'right_double_line': u'61', + u'ideogram_overline': u'62', + u'left_line': u'62', + u'ideogram_double_overline': u'63', + u'left_double_line': u'63', + u'ideogram_stress': u'64', + u'reset_ideogram': u'65' } @@ -138,10 +142,10 @@ class TextFormat(object): '{0}Can you read this?{1}' ).format(magenta_on_green, TextFormat('reset')) ''' - self.codes = [codes[attr.lower()] for attr in attrs if isinstance(attr, str)] + self.codes = [codes[attr.lower()] for attr in attrs if isinstance(attr, six.string_types)] - if kwargs.get('reset', True): - self.codes[:0] = [codes['reset']] + if kwargs.get(u'reset', True): + self.codes[:0] = [codes[u'reset']] def qualify_int(i): if isinstance(i, int): @@ -151,20 +155,20 @@ class TextFormat(object): if isinstance(t, (list, tuple)) and len(t) == 3: return qualify_int(t[0]), qualify_int(t[1]), qualify_int(t[2]) - if kwargs.get('x', None) is not None: - self.codes.extend((codes['extended'], '5', qualify_int(kwargs['x']))) - elif kwargs.get('rgb', None) is not None: - self.codes.extend((codes['extended'], '2')) - self.codes.extend(*qualify_triple_int(kwargs['rgb'])) + if kwargs.get(u'x', None) is not None: + self.codes.extend((codes[u'extended'], u'5', qualify_int(kwargs[u'x']))) + elif kwargs.get(u'rgb', None) is not None: + self.codes.extend((codes[u'extended'], u'2')) + self.codes.extend(*qualify_triple_int(kwargs[u'rgb'])) - if kwargs.get('bg_x', None) is not None: - self.codes.extend((codes['extended'], '5', qualify_int(kwargs['bg_x']))) - elif kwargs.get('bg_rgb', None) is not None: - self.codes.extend((codes['extended'], '2')) - self.codes.extend(*qualify_triple_int(kwargs['bg_rgb'])) + if kwargs.get(u'bg_x', None) is not None: + self.codes.extend((codes[u'extended'], u'5', qualify_int(kwargs[u'bg_x']))) + elif kwargs.get(u'bg_rgb', None) is not None: + self.codes.extend((codes[u'extended'], u'2')) + self.codes.extend(*qualify_triple_int(kwargs[u'bg_rgb'])) - self.sequence = '%s%s%s' % (graph_prefix, # pylint: disable=E1321 - ';'.join(self.codes), + self.sequence = u'%s%s%s' % (graph_prefix, # pylint: disable=E1321 + u';'.join(self.codes), graph_suffix) def __call__(self, text, reset=True): @@ -179,8 +183,8 @@ class TextFormat(object): green_blink_text = TextFormat('blink', 'green') 'The answer is: {0}'.format(green_blink_text(42)) ''' - end = TextFormat('reset') if reset else '' - return '%s%s%s' % (self.sequence, text, end) # pylint: disable=E1321 + end = TextFormat(u'reset') if reset else u'' + return u'%s%s%s' % (self.sequence, text, end) # pylint: disable=E1321 def __str__(self): return self.sequence diff --git a/salt/thorium/__init__.py b/salt/thorium/__init__.py index 72e854a36b5..48bef8580b3 100644 --- a/salt/thorium/__init__.py +++ b/salt/thorium/__init__.py @@ -23,6 +23,9 @@ import salt.loader import salt.payload from salt.exceptions import SaltRenderError +# Import 3rd-party libs +from salt.ext import six + log = logging.getLogger(__name__) @@ -69,7 +72,7 @@ class ThorState(salt.state.HighState): cache = {'grains': {}, 'pillar': {}} if self.grains or self.pillar: if self.opts.get('minion_data_cache'): - minions = self.cache.ls('minions') + minions = self.cache.list('minions') if not minions: return cache for minion in minions: @@ -130,7 +133,7 @@ class ThorState(salt.state.HighState): matches = self.matches_whitelist(matches, whitelist) high, errors = self.render_highstate(matches) if exclude: - if isinstance(exclude, str): + if isinstance(exclude, six.string_types): exclude = exclude.split(',') if '__exclude__' in high: high['__exclude__'].extend(exclude) diff --git a/salt/thorium/check.py b/salt/thorium/check.py index 6bde8eb8d37..1196280cbe4 100644 --- a/salt/thorium/check.py +++ b/salt/thorium/check.py @@ -7,8 +7,12 @@ of having a command execution get gated by a check state via a requisite. ''' # import python libs from __future__ import absolute_import +import logging + import salt.utils +log = logging.getLogger(__file__) + def gt(name, value): ''' @@ -208,7 +212,14 @@ def ne(name, value): return ret -def contains(name, value): +def contains(name, + value, + count_lt=None, + count_lte=None, + count_eq=None, + count_gte=None, + count_gt=None, + count_ne=None): ''' Only succeed if the value in the given register location contains the given value @@ -237,8 +248,27 @@ def contains(name, value): ret['comment'] = 'Value {0} not in register'.format(name) return ret try: - if value in __reg__[name]['val']: + count_compare = count_lt or count_lte or count_eq or\ + count_gte or count_gt or count_ne + if count_compare: + occurrences = __reg__[name]['val'].count(value) + log.debug('{} appears {} times'.format(value, occurrences)) ret['result'] = True + if count_lt: + ret['result'] &= occurrences < count_lt + if count_lte: + ret['result'] &= occurrences <= count_lte + if count_eq: + ret['result'] &= occurrences == count_eq + if count_gte: + ret['result'] &= occurrences >= count_gte + if count_gt: + ret['result'] &= occurrences > count_gt + if count_ne: + ret['result'] &= occurrences != count_ne + else: + if value in __reg__[name]['val']: + ret['result'] = True except TypeError: pass return ret @@ -273,3 +303,201 @@ def event(name): ret['result'] = True return ret + + +def len_gt(name, value): + ''' + Only succeed if length of the given register location is greater than + the given value. + + USAGE: + + .. code-block:: yaml + + foo: + check.len_gt: + - value: 42 + + run_remote_ex: + local.cmd: + - tgt: '*' + - func: test.ping + - require: + - check: foo + ''' + ret = {'name': name, + 'result': False, + 'comment': '', + 'changes': {}} + if name not in __reg__: + ret['result'] = False + ret['comment'] = 'Value {0} not in register'.format(name) + return ret + if len(__reg__[name]['val']) > value: + ret['result'] = True + return ret + + +def len_gte(name, value): + ''' + Only succeed if the length of the given register location is greater or equal + than the given value + + USAGE: + + .. code-block:: yaml + + foo: + check.len_gte: + - value: 42 + + run_remote_ex: + local.cmd: + - tgt: '*' + - func: test.ping + - require: + - check: foo + ''' + ret = {'name': name, + 'result': False, + 'comment': '', + 'changes': {}} + if name not in __reg__: + ret['result'] = False + ret['comment'] = 'Value {0} not in register'.format(name) + return ret + if len(__reg__[name]['val']) >= value: + ret['result'] = True + return ret + + +def len_lt(name, value): + ''' + Only succeed if the lenght of the given register location is less than + the given value. + + USAGE: + + .. code-block:: yaml + + foo: + check.len_lt: + - value: 42 + + run_remote_ex: + local.cmd: + - tgt: '*' + - func: test.ping + - require: + - check: foo + ''' + ret = {'name': name, + 'result': False, + 'comment': '', + 'changes': {}} + if name not in __reg__: + ret['result'] = False + ret['comment'] = 'Value {0} not in register'.format(name) + return ret + if len(__reg__[name]['val']) < value: + ret['result'] = True + return ret + + +def len_lte(name, value): + ''' + Only succeed if the length of the given register location is less than + or equal the given value + + USAGE: + + .. code-block:: yaml + + foo: + check.len_lte: + - value: 42 + + run_remote_ex: + local.cmd: + - tgt: '*' + - func: test.ping + - require: + - check: foo + ''' + ret = {'name': name, + 'result': False, + 'comment': '', + 'changes': {}} + if name not in __reg__: + ret['result'] = False + ret['comment'] = 'Value {0} not in register'.format(name) + return ret + if len(__reg__[name]['val']) <= value: + ret['result'] = True + return ret + + +def len_eq(name, value): + ''' + Only succeed if the length of the given register location is equal to + the given value. + + USAGE: + + .. code-block:: yaml + + foo: + check.len_eq: + - value: 42 + + run_remote_ex: + local.cmd: + - tgt: '*' + - func: test.ping + - require: + - check: foo + ''' + ret = {'name': name, + 'result': False, + 'comment': '', + 'changes': {}} + if name not in __reg__: + ret['result'] = False + ret['comment'] = 'Value {0} not in register'.format(name) + return ret + if __reg__[name]['val'] == value: + ret['result'] = True + return ret + + +def len_ne(name, value): + ''' + Only succeed if the length of the given register location is not equal to + the given value. + + USAGE: + + .. code-block:: yaml + + foo: + check.len_ne: + - value: 42 + + run_remote_ex: + local.cmd: + - tgt: '*' + - func: test.ping + - require: + - check: foo + ''' + ret = {'name': name, + 'result': False, + 'comment': '', + 'changes': {}} + if name not in __reg__: + ret['result'] = False + ret['comment'] = 'Value {0} not in register'.format(name) + return ret + if len(__reg__[name]['val']) != value: + ret['result'] = True + return ret diff --git a/salt/thorium/file.py b/salt/thorium/file.py index 0d2f4a2e6d1..931bb819ddc 100644 --- a/salt/thorium/file.py +++ b/salt/thorium/file.py @@ -45,7 +45,7 @@ import json # Import salt libs import salt.utils -from salt.utils import simple_types_filter +import salt.utils.files def save(name, filter=False): @@ -78,9 +78,11 @@ def save(name, filter=False): fn_ = os.path.join(tgt_dir, name) if not os.path.isdir(tgt_dir): os.makedirs(tgt_dir) - with salt.utils.fopen(fn_, 'w+') as fp_: + with salt.utils.files.fopen(fn_, 'w+') as fp_: if filter is True: - fp_.write(json.dumps(simple_types_filter(__reg__))) + fp_.write(json.dumps( + salt.utils.simple_types_filter(__reg__)) + ) else: fp_.write(json.dumps(__reg__)) return ret diff --git a/salt/tokens/__init__.py b/salt/tokens/__init__.py new file mode 100644 index 00000000000..bdd5d4ff196 --- /dev/null +++ b/salt/tokens/__init__.py @@ -0,0 +1,17 @@ +# -*- coding: utf-8 -*- +''' + salt.tokens + ~~~~~~~~~~~~ + + This module implements all the token stores used by salt during eauth authentication. + Each store must implement the following methods: + + :mk_token: function to mint a new unique token and store it + + :get_token: function to get data of a given token if it exists + + :rm_token: remove the given token from storage + + :list_tokens: list all tokens in storage + +''' diff --git a/salt/tokens/localfs.py b/salt/tokens/localfs.py new file mode 100644 index 00000000000..e2c45c254aa --- /dev/null +++ b/salt/tokens/localfs.py @@ -0,0 +1,97 @@ +# -*- coding: utf-8 -*- + +''' +Stores eauth tokens in the filesystem of the master. Location is configured by the master config option 'token_dir' +''' + +from __future__ import absolute_import + +import hashlib +import os +import logging + +import salt.utils +import salt.payload + +log = logging.getLogger(__name__) + +__virtualname__ = 'localfs' + + +def mk_token(opts, tdata): + ''' + Mint a new token using the config option hash_type and store tdata with 'token' attribute set + to the token. + This module uses the hash of random 512 bytes as a token. + + :param opts: Salt master config options + :param tdata: Token data to be stored with 'token' attirbute of this dict set to the token. + :returns: tdata with token if successful. Empty dict if failed. + ''' + hash_type = getattr(hashlib, opts.get('hash_type', 'md5')) + tok = str(hash_type(os.urandom(512)).hexdigest()) + t_path = os.path.join(opts['token_dir'], tok) + while os.path.isfile(t_path): + tok = str(hash_type(os.urandom(512)).hexdigest()) + t_path = os.path.join(opts['token_dir'], tok) + tdata['token'] = tok + serial = salt.payload.Serial(opts) + try: + with salt.utils.files.set_umask(0o177): + with salt.utils.files.fopen(t_path, 'w+b') as fp_: + fp_.write(serial.dumps(tdata)) + except (IOError, OSError): + log.warning('Authentication failure: can not write token file "{0}".'.format(t_path)) + return {} + return tdata + + +def get_token(opts, tok): + ''' + Fetch the token data from the store. + + :param opts: Salt master config options + :param tok: Token value to get + :returns: Token data if successful. Empty dict if failed. + ''' + t_path = os.path.join(opts['token_dir'], tok) + if not os.path.isfile(t_path): + return {} + serial = salt.payload.Serial(opts) + try: + with salt.utils.files.fopen(t_path, 'rb') as fp_: + tdata = serial.loads(fp_.read()) + return tdata + except (IOError, OSError): + log.warning('Authentication failure: can not read token file "{0}".'.format(t_path)) + return {} + + +def rm_token(opts, tok): + ''' + Remove token from the store. + + :param opts: Salt master config options + :param tok: Token to remove + :returns: Empty dict if successful. None if failed. + ''' + t_path = os.path.join(opts['token_dir'], tok) + try: + os.remove(t_path) + return {} + except (IOError, OSError): + log.warning('Could not remove token {0}'.format(tok)) + + +def list_tokens(opts): + ''' + List all tokens in the store. + + :param opts: Salt master config options + :returns: List of dicts (tokens) + ''' + ret = [] + for (dirpath, dirnames, filenames) in os.walk(opts['token_dir']): + for token in filenames: + ret.append(token) + return ret diff --git a/salt/tokens/rediscluster.py b/salt/tokens/rediscluster.py new file mode 100644 index 00000000000..5609884f8ca --- /dev/null +++ b/salt/tokens/rediscluster.py @@ -0,0 +1,145 @@ +# -*- coding: utf-8 -*- + +''' +Provide token storage in Redis cluster. + +To get started simply start a redis cluster and assign all hashslots to the connected nodes. +Add the redis hostname and port to master configs as eauth_redis_host and eauth_redis_port. +Default values for these configs are as follow: + +.. code-block:: yaml + + eauth_redis_host: localhost + eauth_redis_port: 6379 + +:depends: - redis-py-cluster Python package +''' + +from __future__ import absolute_import + + +try: + import rediscluster + HAS_REDIS = True +except ImportError: + HAS_REDIS = False + + +import os +import logging +import hashlib + +import salt.payload + +log = logging.getLogger(__name__) + +__virtualname__ = 'rediscluster' + + +def __virtual__(): + if not HAS_REDIS: + return False, 'Could not use redis for tokens; '\ + 'rediscluster python client is not installed.' + return __virtualname__ + + +def _redis_client(opts): + ''' + Connect to the redis host and return a StrictRedisCluster client object. + If connection fails then return None. + ''' + redis_host = opts.get("eauth_redis_host", "localhost") + redis_port = opts.get("eauth_redis_port", 6379) + try: + return rediscluster.StrictRedisCluster(host=redis_host, port=redis_port) + except rediscluster.exceptions.RedisClusterException as err: + log.warning("Failed to connect to redis at {0}:{1} - {2}".format(redis_host, redis_port, err)) + return None + + +def mk_token(opts, tdata): + ''' + Mint a new token using the config option hash_type and store tdata with 'token' attribute set + to the token. + This module uses the hash of random 512 bytes as a token. + + :param opts: Salt master config options + :param tdata: Token data to be stored with 'token' attirbute of this dict set to the token. + :returns: tdata with token if successful. Empty dict if failed. + ''' + redis_client = _redis_client(opts) + if not redis_client: + return {} + hash_type = getattr(hashlib, opts.get('hash_type', 'md5')) + tok = str(hash_type(os.urandom(512)).hexdigest()) + try: + while redis_client.get(tok) is not None: + tok = str(hash_type(os.urandom(512)).hexdigest()) + except Exception as err: + log.warning("Authentication failure: cannot get token {0} from redis: {1}".format(tok, err)) + return {} + tdata['token'] = tok + serial = salt.payload.Serial(opts) + try: + redis_client.set(tok, serial.dumps(tdata)) + except Exception as err: + log.warning("Authentication failure: cannot save token {0} to redis: {1}".format(tok, err)) + return {} + return tdata + + +def get_token(opts, tok): + ''' + Fetch the token data from the store. + + :param opts: Salt master config options + :param tok: Token value to get + :returns: Token data if successful. Empty dict if failed. + ''' + redis_client = _redis_client(opts) + if not redis_client: + return {} + serial = salt.payload.Serial(opts) + try: + tdata = serial.loads(redis_client.get(tok)) + return tdata + except Exception as err: + log.warning("Authentication failure: cannot get token {0} from redis: {1}".format(tok, err)) + return {} + + +def rm_token(opts, tok): + ''' + Remove token from the store. + + :param opts: Salt master config options + :param tok: Token to remove + :returns: Empty dict if successful. None if failed. + ''' + redis_client = _redis_client(opts) + if not redis_client: + return + try: + redis_client.delete(tok) + return {} + except Exception as err: + log.warning("Could not remove token {0}: {1}".format(tok, err)) + + +def list_tokens(opts): + ''' + List all tokens in the store. + + :param opts: Salt master config options + :returns: List of dicts (token_data) + ''' + ret = [] + redis_client = _redis_client(opts) + if not redis_client: + return [] + serial = salt.payload.Serial(opts) + try: + return [k.decode('utf8') for k in redis_client.keys()] + except Exception as err: + log.warning("Failed to list keys: {0}".format(err)) + return [] diff --git a/salt/transport/__init__.py b/salt/transport/__init__.py index 0072076dfe6..95290252e4a 100644 --- a/salt/transport/__init__.py +++ b/salt/transport/__init__.py @@ -3,9 +3,13 @@ Encapsulate the different transports available to Salt. ''' from __future__ import absolute_import +import logging # Import third party libs -import salt.ext.six as six +from salt.ext import six +from salt.ext.six.moves import range + +log = logging.getLogger(__name__) def iter_transport_opts(opts): @@ -47,3 +51,19 @@ class Channel(object): # salt.transport.channel.Channel.factory() from salt.transport.client import ReqChannel return ReqChannel.factory(opts, **kwargs) + + +class MessageClientPool(object): + def __init__(self, tgt, opts, args=None, kwargs=None): + sock_pool_size = opts['sock_pool_size'] if 'sock_pool_size' in opts else 1 + if sock_pool_size < 1: + log.warn('sock_pool_size is not correctly set, \ + the option should be greater than 0 but, {0}'.format(sock_pool_size)) + sock_pool_size = 1 + + if args is None: + args = () + if kwargs is None: + kwargs = {} + + self.message_clients = [tgt(*args, **kwargs) for _ in range(sock_pool_size)] diff --git a/salt/transport/frame.py b/salt/transport/frame.py index a4919f4e542..c26eeb68daf 100644 --- a/salt/transport/frame.py +++ b/salt/transport/frame.py @@ -5,7 +5,7 @@ Helper functions for transport components to handle message framing # Import python libs from __future__ import absolute_import import msgpack -import salt.ext.six as six +from salt.ext import six def frame_msg(body, header=None, raw_body=False): # pylint: disable=unused-argument diff --git a/salt/transport/ipc.py b/salt/transport/ipc.py index 46789c3dd57..488acaa0f27 100644 --- a/salt/transport/ipc.py +++ b/salt/transport/ipc.py @@ -24,7 +24,7 @@ from tornado.iostream import IOStream # Import Salt libs import salt.transport.client import salt.transport.frame -import salt.ext.six as six +from salt.ext import six log = logging.getLogger(__name__) @@ -250,15 +250,16 @@ class IPCClient(object): # FIXME key = str(socket_path) - if key not in loop_instance_map: + client = loop_instance_map.get(key) + if client is None: log.debug('Initializing new IPCClient for path: {0}'.format(key)) - new_client = object.__new__(cls) + client = object.__new__(cls) # FIXME - new_client.__singleton_init__(io_loop=io_loop, socket_path=socket_path) - loop_instance_map[key] = new_client + client.__singleton_init__(io_loop=io_loop, socket_path=socket_path) + loop_instance_map[key] = client else: log.debug('Re-using IPCClient for {0}'.format(key)) - return loop_instance_map[key] + return client def __singleton_init__(self, socket_path, io_loop=None): ''' diff --git a/salt/transport/local.py b/salt/transport/local.py index a3c3aff7e0c..7870a9e21a8 100644 --- a/salt/transport/local.py +++ b/salt/transport/local.py @@ -5,7 +5,7 @@ from __future__ import absolute_import, print_function import logging # Import Salt Libs -import salt.utils +import salt.utils.files from salt.transport.client import ReqChannel log = logging.getLogger(__name__) @@ -27,7 +27,7 @@ class LocalChannel(ReqChannel): #data = json.loads(load) #{'path': 'apt-cacher-ng/map.jinja', 'saltenv': 'base', 'cmd': '_serve_file', 'loc': 0} #f = open(data['path']) - with salt.utils.fopen(load['path']) as f: + with salt.utils.files.fopen(load['path']) as f: ret = { 'data': ''.join(f.readlines()), 'dest': load['path'], diff --git a/salt/transport/mixins/auth.py b/salt/transport/mixins/auth.py index 9e0f7ecbdc8..609899725f7 100644 --- a/salt/transport/mixins/auth.py +++ b/salt/transport/mixins/auth.py @@ -16,10 +16,14 @@ import salt.payload import salt.master import salt.transport.frame import salt.utils.event -import salt.ext.six as six +import salt.utils.files +import salt.utils.minions +import salt.utils.stringutils +import salt.utils.verify from salt.utils.cache import CacheCli # Import Third Party Libs +from salt.ext import six import tornado.gen try: from Cryptodome.Cipher import PKCS1_OAEP @@ -111,7 +115,7 @@ class AESReqServerMixin(object): self.opts, key) try: - with salt.utils.fopen(pubfn) as f: + with salt.utils.files.fopen(pubfn) as f: pub = RSA.importKey(f.read()) except (ValueError, IndexError, TypeError): return self.crypticle.dumps({}) @@ -124,7 +128,7 @@ class AESReqServerMixin(object): if six.PY2: pret['key'] = cipher.encrypt(key) else: - pret['key'] = cipher.encrypt(salt.utils.to_bytes(key)) + pret['key'] = cipher.encrypt(salt.utils.stringutils.to_bytes(key)) pret[dictkey] = pcrypt.dumps( ret if ret is not False else {} ) @@ -238,7 +242,7 @@ class AESReqServerMixin(object): elif os.path.isfile(pubfn): # The key has been accepted, check it - with salt.utils.fopen(pubfn, 'r') as pubfn_handle: + with salt.utils.files.fopen(pubfn, 'r') as pubfn_handle: if pubfn_handle.read().strip() != load['pub'].strip(): log.error( 'Authentication attempt from {id} failed, the public ' @@ -246,7 +250,7 @@ class AESReqServerMixin(object): 'the Salt cluster.'.format(**load) ) # put denied minion key into minions_denied - with salt.utils.fopen(pubfn_denied, 'w+') as fp_: + with salt.utils.files.fopen(pubfn_denied, 'w+') as fp_: fp_.write(load['pub']) eload = {'result': False, 'id': load['id'], @@ -289,7 +293,7 @@ class AESReqServerMixin(object): if key_path is not None: # Write the key to the appropriate location - with salt.utils.fopen(key_path, 'w+') as fp_: + with salt.utils.files.fopen(key_path, 'w+') as fp_: fp_.write(load['pub']) ret = {'enc': 'clear', 'load': {'ret': key_result}} @@ -326,7 +330,7 @@ class AESReqServerMixin(object): # Check if the keys are the same and error out if this is the # case. Otherwise log the fact that the minion is still # pending. - with salt.utils.fopen(pubfn_pend, 'r') as pubfn_handle: + with salt.utils.files.fopen(pubfn_pend, 'r') as pubfn_handle: if pubfn_handle.read() != load['pub']: log.error( 'Authentication attempt from {id} failed, the public ' @@ -335,7 +339,7 @@ class AESReqServerMixin(object): .format(**load) ) # put denied minion key into minions_denied - with salt.utils.fopen(pubfn_denied, 'w+') as fp_: + with salt.utils.files.fopen(pubfn_denied, 'w+') as fp_: fp_.write(load['pub']) eload = {'result': False, 'id': load['id'], @@ -362,7 +366,7 @@ class AESReqServerMixin(object): # auto-signed. Check to see if it is the same key, and if # so, pass on doing anything here, and let it get automatically # accepted below. - with salt.utils.fopen(pubfn_pend, 'r') as pubfn_handle: + with salt.utils.files.fopen(pubfn_pend, 'r') as pubfn_handle: if pubfn_handle.read() != load['pub']: log.error( 'Authentication attempt from {id} failed, the public ' @@ -371,7 +375,7 @@ class AESReqServerMixin(object): .format(**load) ) # put denied minion key into minions_denied - with salt.utils.fopen(pubfn_denied, 'w+') as fp_: + with salt.utils.files.fopen(pubfn_denied, 'w+') as fp_: fp_.write(load['pub']) eload = {'result': False, 'id': load['id'], @@ -396,16 +400,16 @@ class AESReqServerMixin(object): # only write to disk if you are adding the file, and in open mode, # which implies we accept any key from a minion. if not os.path.isfile(pubfn) and not self.opts['open_mode']: - with salt.utils.fopen(pubfn, 'w+') as fp_: + with salt.utils.files.fopen(pubfn, 'w+') as fp_: fp_.write(load['pub']) elif self.opts['open_mode']: disk_key = '' if os.path.isfile(pubfn): - with salt.utils.fopen(pubfn, 'r') as fp_: + with salt.utils.files.fopen(pubfn, 'r') as fp_: disk_key = fp_.read() if load['pub'] and load['pub'] != disk_key: log.debug('Host key change detected in open mode.') - with salt.utils.fopen(pubfn, 'w+') as fp_: + with salt.utils.files.fopen(pubfn, 'w+') as fp_: fp_.write(load['pub']) pub = None @@ -417,7 +421,7 @@ class AESReqServerMixin(object): # The key payload may sometimes be corrupt when using auto-accept # and an empty request comes in try: - with salt.utils.fopen(pubfn) as f: + with salt.utils.files.fopen(pubfn) as f: pub = RSA.importKey(f.read()) except (ValueError, IndexError, TypeError) as err: log.error('Corrupt public key "{0}": {1}'.format(pubfn, err)) @@ -440,9 +444,13 @@ class AESReqServerMixin(object): else: # the master has its own signing-keypair, compute the master.pub's # signature and append that to the auth-reply + + # get the key_pass for the signing key + key_pass = salt.utils.sdb.sdb_get(self.opts['signing_key_pass'], self.opts) + log.debug("Signing master public key before sending") pub_sign = salt.crypt.sign_message(self.master_key.get_sign_paths()[1], - ret['pub_key']) + ret['pub_key'], key_pass) ret.update({'pub_sig': binascii.b2a_base64(pub_sign)}) mcipher = PKCS1_OAEP.new(self.master_key.key) diff --git a/salt/transport/raet.py b/salt/transport/raet.py index 1384d61d5a0..4b4930d465d 100644 --- a/salt/transport/raet.py +++ b/salt/transport/raet.py @@ -9,7 +9,7 @@ import time # Import Salt Libs import logging -from salt.utils import kinds +import salt.utils.kinds as kinds from salt.transport.client import ReqChannel log = logging.getLogger(__name__) diff --git a/salt/transport/tcp.py b/salt/transport/tcp.py index 3c381471262..dcfd8ef6858 100644 --- a/salt/transport/tcp.py +++ b/salt/transport/tcp.py @@ -20,9 +20,10 @@ import errno # Import Salt Libs import salt.crypt import salt.utils -import salt.utils.verify -import salt.utils.event import salt.utils.async +import salt.utils.event +import salt.utils.platform +import salt.utils.verify import salt.payload import salt.exceptions import salt.transport.frame @@ -30,7 +31,7 @@ import salt.transport.ipc import salt.transport.client import salt.transport.server import salt.transport.mixins.auth -import salt.ext.six as six +from salt.ext import six from salt.exceptions import SaltReqTimeoutError, SaltClientError from salt.transport import iter_transport_opts @@ -55,7 +56,7 @@ try: except ImportError: from Crypto.Cipher import PKCS1_OAEP -if six.PY3 and salt.utils.is_windows(): +if six.PY3 and salt.utils.platform.is_windows(): USE_LOAD_BALANCER = True else: USE_LOAD_BALANCER = False @@ -221,17 +222,18 @@ class AsyncTCPReqChannel(salt.transport.client.ReqChannel): loop_instance_map = cls.instance_map[io_loop] key = cls.__key(opts, **kwargs) - if key not in loop_instance_map: + obj = loop_instance_map.get(key) + if obj is None: log.debug('Initializing new AsyncTCPReqChannel for {0}'.format(key)) # we need to make a local variable for this, as we are going to store # it in a WeakValueDictionary-- which will remove the item if no one # references it-- this forces a reference while we return to the caller - new_obj = object.__new__(cls) - new_obj.__singleton_init__(opts, **kwargs) - loop_instance_map[key] = new_obj + obj = object.__new__(cls) + obj.__singleton_init__(opts, **kwargs) + loop_instance_map[key] = obj else: log.debug('Re-using AsyncTCPReqChannel for {0}'.format(key)) - return loop_instance_map[key] + return obj @classmethod def __key(cls, opts, **kwargs): @@ -267,9 +269,9 @@ class AsyncTCPReqChannel(salt.transport.client.ReqChannel): host, port = parse.netloc.rsplit(':', 1) self.master_addr = (host, int(port)) self._closing = False - self.message_client = SaltMessageClient( - self.opts, host, int(port), io_loop=self.io_loop, - resolver=resolver) + self.message_client = SaltMessageClientPool(self.opts, + args=(self.opts, host, int(port),), + kwargs={'io_loop': self.io_loop, 'resolver': resolver}) def close(self): if self._closing: @@ -404,7 +406,7 @@ class AsyncTCPPubChannel(salt.transport.mixins.auth.AESPubClientMixin, salt.tran def _do_transfer(): msg = self._package_load(self.auth.crypticle.dumps(load)) package = salt.transport.frame.frame_msg(msg, header=None) - yield self.message_client._stream.write(package) + yield self.message_client.write_to_stream(package) raise tornado.gen.Return(True) if force_auth or not self.auth.authenticated: @@ -494,13 +496,12 @@ class AsyncTCPPubChannel(salt.transport.mixins.auth.AESPubClientMixin, salt.tran if not self.auth.authenticated: yield self.auth.authenticate() if self.auth.authenticated: - self.message_client = SaltMessageClient( + self.message_client = SaltMessageClientPool( self.opts, - self.opts['master_ip'], - int(self.auth.creds['publish_port']), - io_loop=self.io_loop, - connect_callback=self.connect_callback, - disconnect_callback=self.disconnect_callback) + args=(self.opts, self.opts['master_ip'], int(self.auth.creds['publish_port']),), + kwargs={'io_loop': self.io_loop, + 'connect_callback': self.connect_callback, + 'disconnect_callback': self.disconnect_callback}) yield self.message_client.connect() # wait for the client to be connected self.connected = True # TODO: better exception handling... @@ -569,7 +570,7 @@ class TCPReqServerChannel(salt.transport.mixins.auth.AESReqServerMixin, salt.tra process_manager.add_process( LoadBalancerServer, args=(self.opts, self.socket_queue) ) - elif not salt.utils.is_windows(): + elif not salt.utils.platform.is_windows(): self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) _set_tcp_keepalive(self._socket, self.opts) @@ -592,7 +593,7 @@ class TCPReqServerChannel(salt.transport.mixins.auth.AESReqServerMixin, salt.tra io_loop=self.io_loop, ssl_options=self.opts.get('ssl')) else: - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) _set_tcp_keepalive(self._socket, self.opts) @@ -611,49 +612,61 @@ class TCPReqServerChannel(salt.transport.mixins.auth.AESReqServerMixin, salt.tra Handle incoming messages from underylying tcp streams ''' try: - payload = self._decode_payload(payload) - except Exception: - stream.write(salt.transport.frame.frame_msg('bad load', header=header)) - raise tornado.gen.Return() + try: + payload = self._decode_payload(payload) + except Exception: + stream.write(salt.transport.frame.frame_msg('bad load', header=header)) + raise tornado.gen.Return() - # TODO helper functions to normalize payload? - if not isinstance(payload, dict) or not isinstance(payload.get('load'), dict): - yield stream.write(salt.transport.frame.frame_msg( - 'payload and load must be a dict', header=header)) - raise tornado.gen.Return() + # TODO helper functions to normalize payload? + if not isinstance(payload, dict) or not isinstance(payload.get('load'), dict): + yield stream.write(salt.transport.frame.frame_msg( + 'payload and load must be a dict', header=header)) + raise tornado.gen.Return() - # intercept the "_auth" commands, since the main daemon shouldn't know - # anything about our key auth - if payload['enc'] == 'clear' and payload.get('load', {}).get('cmd') == '_auth': - yield stream.write(salt.transport.frame.frame_msg( - self._auth(payload['load']), header=header)) - raise tornado.gen.Return() + # intercept the "_auth" commands, since the main daemon shouldn't know + # anything about our key auth + if payload['enc'] == 'clear' and payload.get('load', {}).get('cmd') == '_auth': + yield stream.write(salt.transport.frame.frame_msg( + self._auth(payload['load']), header=header)) + raise tornado.gen.Return() - # TODO: test - try: - ret, req_opts = yield self.payload_handler(payload) - except Exception as e: - # always attempt to return an error to the minion - stream.write('Some exception handling minion payload') - log.error('Some exception handling a payload from minion', exc_info=True) - stream.close() - raise tornado.gen.Return() + # TODO: test + try: + ret, req_opts = yield self.payload_handler(payload) + except Exception as e: + # always attempt to return an error to the minion + stream.write('Some exception handling minion payload') + log.error('Some exception handling a payload from minion', exc_info=True) + stream.close() + raise tornado.gen.Return() + + req_fun = req_opts.get('fun', 'send') + if req_fun == 'send_clear': + stream.write(salt.transport.frame.frame_msg(ret, header=header)) + elif req_fun == 'send': + stream.write(salt.transport.frame.frame_msg(self.crypticle.dumps(ret), header=header)) + elif req_fun == 'send_private': + stream.write(salt.transport.frame.frame_msg(self._encrypt_private(ret, + req_opts['key'], + req_opts['tgt'], + ), header=header)) + else: + log.error('Unknown req_fun {0}'.format(req_fun)) + # always attempt to return an error to the minion + stream.write('Server-side exception handling payload') + stream.close() + except tornado.gen.Return: + raise + except tornado.iostream.StreamClosedError: + # Stream was closed. This could happen if the remote side + # closed the connection on its end (eg in a timeout or shutdown + # situation). + log.error('Connection was unexpectedly closed', exc_info=True) + except Exception as exc: # pylint: disable=broad-except + # Absorb any other exceptions + log.error('Unexpected exception occurred: {0}'.format(exc), exc_info=True) - req_fun = req_opts.get('fun', 'send') - if req_fun == 'send_clear': - stream.write(salt.transport.frame.frame_msg(ret, header=header)) - elif req_fun == 'send': - stream.write(salt.transport.frame.frame_msg(self.crypticle.dumps(ret), header=header)) - elif req_fun == 'send_private': - stream.write(salt.transport.frame.frame_msg(self._encrypt_private(ret, - req_opts['key'], - req_opts['tgt'], - ), header=header)) - else: - log.error('Unknown req_fun {0}'.format(req_fun)) - # always attempt to return an error to the minion - stream.write('Server-side exception handling payload') - stream.close() raise tornado.gen.Return() @@ -764,6 +777,43 @@ class TCPClientKeepAlive(tornado.tcpclient.TCPClient): return stream.connect(addr) +class SaltMessageClientPool(salt.transport.MessageClientPool): + ''' + Wrapper class of SaltMessageClient to avoid blocking waiting while writing data to socket. + ''' + def __init__(self, opts, args=None, kwargs=None): + super(SaltMessageClientPool, self).__init__(SaltMessageClient, opts, args=args, kwargs=kwargs) + + def __del__(self): + self.close() + + def close(self): + for message_client in self.message_clients: + message_client.close() + self.message_clients = [] + + @tornado.gen.coroutine + def connect(self): + futures = [] + for message_client in self.message_clients: + futures.append(message_client.connect()) + for future in futures: + yield future + raise tornado.gen.Return(None) + + def on_recv(self, *args, **kwargs): + for message_client in self.message_clients: + message_client.on_recv(*args, **kwargs) + + def send(self, *args, **kwargs): + message_clients = sorted(self.message_clients, key=lambda x: len(x.send_queue)) + return message_clients[0].send(*args, **kwargs) + + def write_to_stream(self, *args, **kwargs): + message_clients = sorted(self.message_clients, key=lambda x: len(x.send_queue)) + return message_clients[0]._stream.write(*args, **kwargs) + + # TODO consolidate with IPCClient # TODO: limit in-flight messages. # TODO: singleton? Something to not re-create the tcp connection so much @@ -1318,7 +1368,7 @@ class TCPPubServerChannel(salt.transport.server.PubServerChannel): do the actual publishing ''' kwargs = {} - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): kwargs['log_queue'] = ( salt.log.setup.get_multiprocessing_logging_queue() ) diff --git a/salt/transport/zeromq.py b/salt/transport/zeromq.py index e6110e0523c..7bb5409baf0 100644 --- a/salt/transport/zeromq.py +++ b/salt/transport/zeromq.py @@ -21,6 +21,7 @@ import salt.crypt import salt.utils import salt.utils.verify import salt.utils.event +import salt.utils.stringutils import salt.payload import salt.transport.client import salt.transport.server @@ -46,7 +47,7 @@ import tornado.gen import tornado.concurrent # Import third party libs -import salt.ext.six as six +from salt.ext import six try: from Cryptodome.Cipher import PKCS1_OAEP except ImportError: @@ -80,28 +81,19 @@ class AsyncZeroMQReqChannel(salt.transport.client.ReqChannel): loop_instance_map = cls.instance_map[io_loop] key = cls.__key(opts, **kwargs) - if key not in loop_instance_map: + obj = loop_instance_map.get(key) + if obj is None: log.debug('Initializing new AsyncZeroMQReqChannel for {0}'.format(key)) # we need to make a local variable for this, as we are going to store # it in a WeakValueDictionary-- which will remove the item if no one # references it-- this forces a reference while we return to the caller - new_obj = object.__new__(cls) - new_obj.__singleton_init__(opts, **kwargs) - loop_instance_map[key] = new_obj + obj = object.__new__(cls) + obj.__singleton_init__(opts, **kwargs) + loop_instance_map[key] = obj log.trace('Inserted key into loop_instance_map id {0} for key {1} and process {2}'.format(id(loop_instance_map), key, os.getpid())) else: log.debug('Re-using AsyncZeroMQReqChannel for {0}'.format(key)) - try: - return loop_instance_map[key] - except KeyError: - # In iterating over the loop_instance_map, we may have triggered - # garbage collection. Therefore, the key is no longer present in - # the map. Re-gen and add to map. - log.debug('Initializing new AsyncZeroMQReqChannel due to GC for {0}'.format(key)) - new_obj = object.__new__(cls) - new_obj.__singleton_init__(opts, **kwargs) - loop_instance_map[key] = new_obj - return loop_instance_map[key] + return obj def __deepcopy__(self, memo): cls = self.__class__ @@ -118,8 +110,9 @@ class AsyncZeroMQReqChannel(salt.transport.client.ReqChannel): # copied. The reason is the same as the io_loop skip above. setattr(result, key, AsyncReqMessageClientPool(result.opts, - self.master_uri, - io_loop=result._io_loop)) + args=(result.opts, self.master_uri,), + kwargs={'io_loop': self._io_loop})) + continue setattr(result, key, copy.deepcopy(self.__dict__[key], memo)) return result @@ -156,9 +149,8 @@ class AsyncZeroMQReqChannel(salt.transport.client.ReqChannel): # we don't need to worry about auth as a kwarg, since its a singleton self.auth = salt.crypt.AsyncAuth(self.opts, io_loop=self._io_loop) self.message_client = AsyncReqMessageClientPool(self.opts, - self.master_uri, - io_loop=self._io_loop, - ) + args=(self.opts, self.master_uri,), + kwargs={'io_loop': self._io_loop}) def __del__(self): ''' @@ -313,7 +305,7 @@ class AsyncZeroMQPubChannel(salt.transport.mixins.auth.AESPubClientMixin, salt.t else: self._socket.setsockopt(zmq.SUBSCRIBE, b'') - self._socket.setsockopt(zmq.IDENTITY, salt.utils.to_bytes(self.opts['id'])) + self._socket.setsockopt(zmq.IDENTITY, salt.utils.stringutils.to_bytes(self.opts['id'])) # TODO: cleanup all the socket opts stuff if hasattr(zmq, 'TCP_KEEPALIVE'): @@ -835,8 +827,9 @@ class ZeroMQPubServerChannel(salt.transport.server.PubServerChannel): match_targets = ["pcre", "glob", "list"] if self.opts['zmq_filtering'] and load['tgt_type'] in match_targets: # Fetch a list of minions that match - match_ids = self.ckminions.check_minions(load['tgt'], - tgt_type=load['tgt_type']) + _res = self.ckminions.check_minions(load['tgt'], + tgt_type=load['tgt_type']) + match_ids = _res['minions'] log.debug("Publish Side Match: {0}".format(match_ids)) # Send list of miions thru so zmq can target them @@ -847,32 +840,24 @@ class ZeroMQPubServerChannel(salt.transport.server.PubServerChannel): context.term() -# TODO: unit tests! -class AsyncReqMessageClientPool(object): - def __init__(self, opts, addr, linger=0, io_loop=None, socket_pool=1): - self.opts = opts - self.addr = addr - self.linger = linger - self.io_loop = io_loop - self.socket_pool = socket_pool - self.message_clients = [] +class AsyncReqMessageClientPool(salt.transport.MessageClientPool): + ''' + Wrapper class of AsyncReqMessageClientPool to avoid blocking waiting while writing data to socket. + ''' + def __init__(self, opts, args=None, kwargs=None): + super(AsyncReqMessageClientPool, self).__init__(AsyncReqMessageClient, opts, args=args, kwargs=kwargs) + + def __del__(self): + self.destroy() def destroy(self): for message_client in self.message_clients: message_client.destroy() self.message_clients = [] - def __del__(self): - self.destroy() - - def send(self, message, timeout=None, tries=3, future=None, callback=None, raw=False): - if len(self.message_clients) < self.socket_pool: - message_client = AsyncReqMessageClient(self.opts, self.addr, self.linger, self.io_loop) - self.message_clients.append(message_client) - return message_client.send(message, timeout, tries, future, callback, raw) - else: - available_clients = sorted(self.message_clients, key=lambda x: len(x.send_queue)) - return available_clients[0].send(message, timeout, tries, future, callback, raw) + def send(self, *args, **kwargs): + message_clients = sorted(self.message_clients, key=lambda x: len(x.send_queue)) + return message_clients[0].send(*args, **kwargs) # TODO: unit tests! diff --git a/salt/utils/__init__.py b/salt/utils/__init__.py index b39fda41656..c83bccdf6fa 100644 --- a/salt/utils/__init__.py +++ b/salt/utils/__init__.py @@ -1,6 +1,11 @@ # -*- coding: utf-8 -*- ''' Some of the utils used by salt + +NOTE: The dev team is working on splitting up this file for the Oxygen release. +Please do not add any new functions to this file. New functions should be +organized in other files under salt/utils/. Please consult the dev team if you +are unsure where a new function should go. ''' # Import python libs @@ -12,24 +17,18 @@ import datetime import errno import fnmatch import hashlib -import imp import json import logging -import numbers import os -import posixpath import random import re import shlex import shutil import socket -import stat import sys import pstats -import tempfile import time import types -import warnings import string import subprocess import getpass @@ -37,13 +36,14 @@ import getpass # Import 3rd-party libs from salt.ext import six # pylint: disable=import-error -from salt.ext.six.moves.urllib.parse import urlparse # pylint: disable=no-name-in-module # pylint: disable=redefined-builtin from salt.ext.six.moves import range -from salt.ext.six.moves import zip -from stat import S_IMODE # pylint: enable=import-error,redefined-builtin +if six.PY3: + import importlib.util # pylint: disable=no-name-in-module,import-error +else: + import imp try: import cProfile @@ -71,13 +71,6 @@ try: except ImportError: HAS_PARSEDATETIME = False -try: - import fcntl - HAS_FCNTL = True -except ImportError: - # fcntl is not available on windows - HAS_FCNTL = False - try: import win32api HAS_WIN32API = True @@ -124,11 +117,9 @@ from salt.defaults import DEFAULT_TARGET_DELIM import salt.defaults.exitcodes import salt.log import salt.utils.dictupdate +import salt.utils.versions import salt.version -from salt.utils.decorators import jinja_filter -from salt.utils.decorators import memoize as real_memoize -from salt.utils.versions import LooseVersion as _LooseVersion -from salt.textformat import TextFormat +from salt.utils.decorators.jinja import jinja_filter from salt.exceptions import ( CommandExecutionError, SaltClientError, CommandNotFoundError, SaltSystemExit, @@ -137,115 +128,6 @@ from salt.exceptions import ( log = logging.getLogger(__name__) -_empty = object() - - -def safe_rm(tgt): - ''' - Safely remove a file - ''' - try: - os.remove(tgt) - except (IOError, OSError): - pass - - -@jinja_filter('is_empty') -def is_empty(filename): - ''' - Is a file empty? - ''' - try: - return os.stat(filename).st_size == 0 - except OSError: - # Non-existent file or permission denied to the parent dir - return False - - -@jinja_filter('is_hex') -def is_hex(value): - ''' - Returns True if value is a hexidecimal string, otherwise returns False - ''' - try: - int(value, 16) - return True - except (TypeError, ValueError): - return False - - -def get_color_theme(theme): - ''' - Return the color theme to use - ''' - # Keep the heavy lifting out of the module space - import yaml - if not os.path.isfile(theme): - log.warning('The named theme {0} if not available'.format(theme)) - try: - with fopen(theme, 'rb') as fp_: - colors = yaml.safe_load(fp_.read()) - ret = {} - for color in colors: - ret[color] = '\033[{0}m'.format(colors[color]) - if not isinstance(colors, dict): - log.warning('The theme file {0} is not a dict'.format(theme)) - return {} - return ret - except Exception: - log.warning('Failed to read the color theme {0}'.format(theme)) - return {} - - -def get_colors(use=True, theme=None): - ''' - Return the colors as an easy to use dict. Pass `False` to deactivate all - colors by setting them to empty strings. Pass a string containing only the - name of a single color to be used in place of all colors. Examples: - - .. code-block:: python - - colors = get_colors() # enable all colors - no_colors = get_colors(False) # disable all colors - red_colors = get_colors('RED') # set all colors to red - ''' - - colors = { - 'BLACK': TextFormat('black'), - 'DARK_GRAY': TextFormat('bold', 'black'), - 'RED': TextFormat('red'), - 'LIGHT_RED': TextFormat('bold', 'red'), - 'GREEN': TextFormat('green'), - 'LIGHT_GREEN': TextFormat('bold', 'green'), - 'YELLOW': TextFormat('yellow'), - 'LIGHT_YELLOW': TextFormat('bold', 'yellow'), - 'BLUE': TextFormat('blue'), - 'LIGHT_BLUE': TextFormat('bold', 'blue'), - 'MAGENTA': TextFormat('magenta'), - 'LIGHT_MAGENTA': TextFormat('bold', 'magenta'), - 'CYAN': TextFormat('cyan'), - 'LIGHT_CYAN': TextFormat('bold', 'cyan'), - 'LIGHT_GRAY': TextFormat('white'), - 'WHITE': TextFormat('bold', 'white'), - 'DEFAULT_COLOR': TextFormat('default'), - 'ENDC': TextFormat('reset'), - } - if theme: - colors.update(get_color_theme(theme)) - - if not use: - for color in colors: - colors[color] = '' - if isinstance(use, str): - # Try to set all of the colors to the passed color - if use in colors: - for color in colors: - # except for color reset - if color == 'ENDC': - continue - colors[color] = colors[use] - - return colors def get_context(template, line, num_lines=5, marker=None): @@ -254,6 +136,7 @@ def get_context(template, line, num_lines=5, marker=None): Returns:: string ''' + import salt.utils.stringutils template_lines = template.splitlines() num_template_lines = len(template_lines) @@ -280,11 +163,7 @@ def get_context(template, line, num_lines=5, marker=None): if marker: buf[error_line_in_context] += marker - # warning: jinja content may contain unicode strings - # instead of utf-8. - buf = [to_str(i) if isinstance(i, six.text_type) else i for i in buf] - - return '---\n{0}\n---'.format('\n'.join(buf)) + return u'---\n{0}\n---'.format(u'\n'.join(buf)) def get_user(): @@ -408,8 +287,9 @@ def get_specific_user(): Get a user name for publishing. If you find the user is "root" attempt to be more specific ''' + import salt.utils.platform user = get_user() - if is_windows(): + if salt.utils.platform.is_windows(): if _win_current_user_is_admin(): return 'sudo_{0}'.format(user) else: @@ -422,12 +302,17 @@ def get_specific_user(): def get_master_key(key_user, opts, skip_perm_errors=False): + # Late import to avoid circular import. + import salt.utils.files + import salt.utils.verify + import salt.utils.platform + if key_user == 'root': if opts.get('user', 'root') != 'root': key_user = opts.get('user', 'root') if key_user.startswith('sudo_'): key_user = opts.get('user', 'root') - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): # The username may contain '\' if it is in Windows # 'DOMAIN\username' format. Fix this for the keyfile path. key_user = key_user.replace('\\', '_') @@ -439,7 +324,7 @@ def get_master_key(key_user, opts, skip_perm_errors=False): skip_perm_errors) try: - with salt.utils.fopen(keyfile, 'r') as key: + with salt.utils.files.fopen(keyfile, 'r') as key: return key.read() except (OSError, IOError): # Fall back to eauth @@ -448,7 +333,7 @@ def get_master_key(key_user, opts, skip_perm_errors=False): def reinit_crypto(): ''' - When a fork arrises, pycrypto needs to reinit + When a fork arises, pycrypto needs to reinit From its doc:: Caveat: For the random number generator to work correctly, @@ -464,6 +349,9 @@ def daemonize(redirect_out=True): ''' Daemonize a process ''' + # Late import to avoid circular import. + import salt.utils.files + try: pid = os.fork() if pid > 0: @@ -503,7 +391,7 @@ def daemonize(redirect_out=True): # not cleanly redirected and the parent process dies when the # multiprocessing process attempts to access stdout or err. if redirect_out: - with fopen('/dev/null', 'r+') as dev_null: + with salt.utils.files.fopen('/dev/null', 'r+') as dev_null: # Redirect python stdin/out/err # and the os stdin/out/err which can be different os.dup2(dev_null.fileno(), sys.stdin.fileno()) @@ -550,88 +438,6 @@ def profile_func(filename=None): return proffunc -def rand_str(size=9999999999, hash_type=None): - ''' - Return a random string - ''' - if not hash_type: - hash_type = 'md5' - hasher = getattr(hashlib, hash_type) - return hasher(to_bytes(str(random.SystemRandom().randint(0, size)))).hexdigest() - - -@jinja_filter('which') -def which(exe=None): - ''' - Python clone of /usr/bin/which - ''' - def _is_executable_file_or_link(exe): - # check for os.X_OK doesn't suffice because directory may executable - return (os.access(exe, os.X_OK) and - (os.path.isfile(exe) or os.path.islink(exe))) - - if exe: - if _is_executable_file_or_link(exe): - # executable in cwd or fullpath - return exe - - ext_list = os.environ.get('PATHEXT', '.EXE').split(';') - - @real_memoize - def _exe_has_ext(): - ''' - Do a case insensitive test if exe has a file extension match in - PATHEXT - ''' - for ext in ext_list: - try: - pattern = r'.*\.' + ext.lstrip('.') + r'$' - re.match(pattern, exe, re.I).groups() - return True - except AttributeError: - continue - return False - - # Enhance POSIX path for the reliability at some environments, when $PATH is changing - # This also keeps order, where 'first came, first win' for cases to find optional alternatives - search_path = os.environ.get('PATH') and os.environ['PATH'].split(os.pathsep) or list() - for default_path in ['/bin', '/sbin', '/usr/bin', '/usr/sbin', '/usr/local/bin']: - if default_path not in search_path: - search_path.append(default_path) - os.environ['PATH'] = os.pathsep.join(search_path) - for path in search_path: - full_path = os.path.join(path, exe) - if _is_executable_file_or_link(full_path): - return full_path - elif is_windows() and not _exe_has_ext(): - # On Windows, check for any extensions in PATHEXT. - # Allows both 'cmd' and 'cmd.exe' to be matched. - for ext in ext_list: - # Windows filesystem is case insensitive so we - # safely rely on that behavior - if _is_executable_file_or_link(full_path + ext): - return full_path + ext - log.trace('\'{0}\' could not be found in the following search path: \'{1}\''.format(exe, search_path)) - else: - log.error('No executable was passed to be searched by salt.utils.which()') - - return None - - -def which_bin(exes): - ''' - Scan over some possible executables and return the first one that is found - ''' - if not isinstance(exes, collections.Iterable): - return None - for exe in exes: - path = which(exe) - if not path: - continue - return path - return None - - def activate_profile(test=True): pr = None if test: @@ -644,6 +450,12 @@ def activate_profile(test=True): def output_profile(pr, stats_path='/tmp/stats', stop=False, id_=None): + # Late import to avoid circular import. + import salt.utils.files + import salt.utils.hashutils + import salt.utils.path + import salt.utils.stringutils + if pr is not None and HAS_CPROFILE: try: pr.disable() @@ -651,17 +463,17 @@ def output_profile(pr, stats_path='/tmp/stats', stop=False, id_=None): os.makedirs(stats_path) date = datetime.datetime.now().isoformat() if id_ is None: - id_ = rand_str(size=32) + id_ = salt.utils.hashutils.random_hash(size=32) ficp = os.path.join(stats_path, '{0}.{1}.pstats'.format(id_, date)) fico = os.path.join(stats_path, '{0}.{1}.dot'.format(id_, date)) ficn = os.path.join(stats_path, '{0}.{1}.stats'.format(id_, date)) if not os.path.exists(ficp): pr.dump_stats(ficp) - with fopen(ficn, 'w') as fic: + with salt.utils.files.fopen(ficn, 'w') as fic: pstats.Stats(pr, stream=fic).sort_stats('cumulative') log.info('PROFILING: {0} generated'.format(ficp)) log.info('PROFILING (cumulative): {0} generated'.format(ficn)) - pyprof = which('pyprof2calltree') + pyprof = salt.utils.path.which('pyprof2calltree') cmd = [pyprof, '-i', ficp, '-o', fico] if pyprof: failed = False @@ -678,8 +490,8 @@ def output_profile(pr, stats_path='/tmp/stats', stop=False, id_=None): else: log.info('PROFILING (dot): {0} generated'.format(fico)) log.trace('pyprof2calltree output:') - log.trace(to_str(pro.stdout.read()).strip() + - to_str(pro.stderr.read()).strip()) + log.trace(salt.utils.stringutils.to_str(pro.stdout.read()).strip() + + salt.utils.stringutils.to_str(pro.stderr.read()).strip()) else: log.info('You can run {0} for additional stats.'.format(cmd)) finally: @@ -763,6 +575,14 @@ def ip_bracket(addr): return addr +def refresh_dns(): + ''' + issue #21397: force glibc to re-read resolv.conf + ''' + if HAS_RESINIT: + res_init() + + @jinja_filter('dns_check') def dns_check(addr, port, safe=False, ipv6=None): ''' @@ -775,9 +595,7 @@ def dns_check(addr, port, safe=False, ipv6=None): lookup = addr seen_ipv6 = False try: - # issue #21397: force glibc to re-read resolv.conf - if HAS_RESINIT: - res_init() + refresh_dns() hostnames = socket.getaddrinfo( addr, None, socket.AF_UNSPEC, socket.SOCK_STREAM ) @@ -802,8 +620,8 @@ def dns_check(addr, port, safe=False, ipv6=None): if h[0] != socket.AF_INET6 or ipv6 is not None: candidates.append(candidate_addr) - s = socket.socket(h[0], socket.SOCK_STREAM) try: + s = socket.socket(h[0], socket.SOCK_STREAM) s.connect((candidate_addr.strip('[]'), port)) s.close() @@ -840,13 +658,20 @@ def required_module_list(docstring=None): Return a list of python modules required by a salt module that aren't in stdlib and don't exist on the current pythonpath. ''' + # Late import to avoid circular import. + import salt.utils.doc + if not docstring: return [] ret = [] - modules = parse_docstring(docstring).get('deps', []) + modules = salt.utils.doc.parse_docstring(docstring).get('deps', []) for mod in modules: try: - imp.find_module(mod) + if six.PY3: + if importlib.util.find_spec(mod) is None: # pylint: disable=no-member + ret.append(mod) + else: + imp.find_module(mod) except ImportError: ret.append(mod) return ret @@ -885,10 +710,11 @@ def check_or_die(command): Lazily import `salt.modules.cmdmod` to avoid any sort of circular dependencies. ''' + import salt.utils.path if command is None: raise CommandNotFoundError('\'None\' is not a valid command.') - if not which(command): + if not salt.utils.path.which(command): raise CommandNotFoundError('\'{0}\' is not in the path'.format(command)) @@ -896,15 +722,16 @@ def backup_minion(path, bkroot): ''' Backup a file on the minion ''' + import salt.utils.platform dname, bname = os.path.split(path) - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): src_dir = dname.replace(':', '_') else: src_dir = dname[1:] - if not salt.utils.is_windows(): + if not salt.utils.platform.is_windows(): fstat = os.stat(path) msecs = str(int(time.time() * 1000000))[-6:] - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): # ':' is an illegal filesystem path character on Windows stamp = time.strftime('%a_%b_%d_%H-%M-%S_%Y') else: @@ -916,69 +743,11 @@ def backup_minion(path, bkroot): if not os.path.isdir(os.path.dirname(bkpath)): os.makedirs(os.path.dirname(bkpath)) shutil.copyfile(path, bkpath) - if not salt.utils.is_windows(): + if not salt.utils.platform.is_windows(): os.chown(bkpath, fstat.st_uid, fstat.st_gid) os.chmod(bkpath, fstat.st_mode) -@jinja_filter('path_join') -def path_join(*parts, **kwargs): - ''' - This functions tries to solve some issues when joining multiple absolute - paths on both *nix and windows platforms. - - See tests/unit/utils/path_join_test.py for some examples on what's being - talked about here. - - The "use_posixpath" kwarg can be be used to force joining using poxixpath, - which is useful for Salt fileserver paths on Windows masters. - ''' - if six.PY3: - new_parts = [] - for part in parts: - new_parts.append(to_str(part)) - parts = new_parts - - kwargs = salt.utils.clean_kwargs(**kwargs) - use_posixpath = kwargs.pop('use_posixpath', False) - if kwargs: - invalid_kwargs(kwargs) - - pathlib = posixpath if use_posixpath else os.path - - # Normalize path converting any os.sep as needed - parts = [pathlib.normpath(p) for p in parts] - - try: - root = parts.pop(0) - except IndexError: - # No args passed to func - return '' - - if not parts: - ret = root - else: - stripped = [p.lstrip(os.sep) for p in parts] - try: - ret = pathlib.join(root, *stripped) - except UnicodeDecodeError: - # This is probably Python 2 and one of the parts contains unicode - # characters in a bytestring. First try to decode to the system - # encoding. - try: - enc = __salt_system_encoding__ - except NameError: - enc = sys.stdin.encoding or sys.getdefaultencoding() - try: - ret = pathlib.join(root.decode(enc), - *[x.decode(enc) for x in stripped]) - except UnicodeDecodeError: - # Last resort, try UTF-8 - ret = pathlib.join(root.decode('UTF-8'), - *[x.decode('UTF-8') for x in stripped]) - return pathlib.normpath(ret) - - def pem_finger(path=None, key=None, sum_type='sha256'): ''' Pass in either a raw pem string, or the path on disk to the location of a @@ -987,11 +756,14 @@ def pem_finger(path=None, key=None, sum_type='sha256'): If neither a key nor a path are passed in, a blank string will be returned. ''' + # Late import to avoid circular import. + import salt.utils.files + if not key: if not os.path.isfile(path): return '' - with fopen(path, 'rb') as fp_: + with salt.utils.files.fopen(path, 'rb') as fp_: key = b''.join([x for x in fp_.readlines() if x.strip()][1:-1]) pre = getattr(hashlib, sum_type)(key).hexdigest() @@ -1012,26 +784,26 @@ def build_whitespace_split_regex(text): Example: - .. code-block:: yaml + .. code-block:: python - >>> import re - >>> from salt.utils import * - >>> regex = build_whitespace_split_regex( - ... """if [ -z "$debian_chroot" ] && [ -r /etc/debian_chroot ]; then""" - ... ) + >>> import re + >>> import salt.utils + >>> regex = salt.utils.build_whitespace_split_regex( + ... """if [ -z "$debian_chroot" ] && [ -r /etc/debian_chroot ]; then""" + ... ) - >>> regex - '(?:[\\s]+)?if(?:[\\s]+)?\\[(?:[\\s]+)?\\-z(?:[\\s]+)?\\"\\$debian' - '\\_chroot\\"(?:[\\s]+)?\\](?:[\\s]+)?\\&\\&(?:[\\s]+)?\\[(?:[\\s]+)?' - '\\-r(?:[\\s]+)?\\/etc\\/debian\\_chroot(?:[\\s]+)?\\]\\;(?:[\\s]+)?' - 'then(?:[\\s]+)?' - >>> re.search( - ... regex, - ... """if [ -z "$debian_chroot" ] && [ -r /etc/debian_chroot ]; then""" - ... ) + >>> regex + '(?:[\\s]+)?if(?:[\\s]+)?\\[(?:[\\s]+)?\\-z(?:[\\s]+)?\\"\\$debian' + '\\_chroot\\"(?:[\\s]+)?\\](?:[\\s]+)?\\&\\&(?:[\\s]+)?\\[(?:[\\s]+)?' + '\\-r(?:[\\s]+)?\\/etc\\/debian\\_chroot(?:[\\s]+)?\\]\\;(?:[\\s]+)?' + 'then(?:[\\s]+)?' + >>> re.search( + ... regex, + ... """if [ -z "$debian_chroot" ] && [ -r /etc/debian_chroot ]; then""" + ... ) - <_sre.SRE_Match object at 0xb70639c0> - >>> + <_sre.SRE_Match object at 0xb70639c0> + >>> ''' def __build_parts(text): @@ -1077,6 +849,9 @@ def format_call(fun, :returns: A dictionary with the function required arguments and keyword arguments. ''' + # Late import to avoid circular import + import salt.utils.versions + import salt.utils.args ret = initial_ret is not None and initial_ret or {} ret['args'] = [] @@ -1084,7 +859,7 @@ def format_call(fun, aspec = salt.utils.args.get_function_argspec(fun, is_class_method=is_class_method) - arg_data = arg_lookup(fun, aspec) + arg_data = salt.utils.args.arg_lookup(fun, aspec) args = arg_data['args'] kwargs = arg_data['kwargs'] @@ -1143,7 +918,7 @@ def format_call(fun, # We'll be showing errors to the users until Salt Oxygen comes out, after # which, errors will be raised instead. - warn_until( + salt.utils.versions.warn_until( 'Oxygen', 'It\'s time to start raising `SaltInvocationError` instead of ' 'returning warnings', @@ -1192,60 +967,7 @@ def format_call(fun, return ret -def arg_lookup(fun, aspec=None): - ''' - Return a dict containing the arguments and default arguments to the - function. - ''' - ret = {'kwargs': {}} - if aspec is None: - aspec = salt.utils.args.get_function_argspec(fun) - if aspec.defaults: - ret['kwargs'] = dict(zip(aspec.args[::-1], aspec.defaults[::-1])) - ret['args'] = [arg for arg in aspec.args if arg not in ret['kwargs']] - return ret - - -@jinja_filter('is_text_file') -def istextfile(fp_, blocksize=512): - ''' - Uses heuristics to guess whether the given file is text or binary, - by reading a single block of bytes from the file. - If more than 30% of the chars in the block are non-text, or there - are NUL ('\x00') bytes in the block, assume this is a binary file. - ''' - int2byte = (lambda x: bytes((x,))) if six.PY3 else chr - text_characters = ( - b''.join(int2byte(i) for i in range(32, 127)) + - b'\n\r\t\f\b') - try: - block = fp_.read(blocksize) - except AttributeError: - # This wasn't an open filehandle, so treat it as a file path and try to - # open the file - try: - with fopen(fp_, 'rb') as fp2_: - block = fp2_.read(blocksize) - except IOError: - # Unable to open file, bail out and return false - return False - if b'\x00' in block: - # Files with null bytes are binary - return False - elif not block: - # An empty file is considered a valid text file - return True - try: - block.decode('utf-8') - return True - except UnicodeDecodeError: - pass - - nontext = block.translate(None, text_characters) - return float(len(nontext)) / len(block) <= 0.30 - - -@jinja_filter('is_sorted') +@jinja_filter('sorted_ignorecase') def isorted(to_sort): ''' Sort a list of strings ignoring case. @@ -1265,6 +987,7 @@ def mysql_to_dict(data, key): ''' Convert MySQL-style output to a python dictionary ''' + import salt.utils.stringutils ret = {} headers = [''] for line in data: @@ -1282,154 +1005,13 @@ def mysql_to_dict(data, key): if field < 1: continue else: - row[headers[field]] = str_to_num(comps[field]) + row[headers[field]] = salt.utils.stringutils.to_num(comps[field]) ret[row[key]] = row else: headers = comps return ret -@jinja_filter('contains_whitespace') -def contains_whitespace(text): - ''' - Returns True if there are any whitespace characters in the string - ''' - return any(x.isspace() for x in text) - - -@jinja_filter('str_to_num') -def str_to_num(text): - ''' - Convert a string to a number. - Returns an integer if the string represents an integer, a floating - point number if the string is a real number, or the string unchanged - otherwise. - ''' - try: - return int(text) - except ValueError: - try: - return float(text) - except ValueError: - return text - - -def fopen(*args, **kwargs): - ''' - Wrapper around open() built-in to set CLOEXEC on the fd. - - This flag specifies that the file descriptor should be closed when an exec - function is invoked; - When a file descriptor is allocated (as with open or dup), this bit is - initially cleared on the new file descriptor, meaning that descriptor will - survive into the new program after exec. - - NB! We still have small race condition between open and fcntl. - - ''' - # ensure 'binary' mode is always used on Windows in Python 2 - if ((six.PY2 and is_windows() and 'binary' not in kwargs) or - kwargs.pop('binary', False)): - if len(args) > 1: - args = list(args) - if 'b' not in args[1]: - args[1] += 'b' - elif kwargs.get('mode', None): - if 'b' not in kwargs['mode']: - kwargs['mode'] += 'b' - else: - # the default is to read - kwargs['mode'] = 'rb' - elif six.PY3 and 'encoding' not in kwargs: - # In Python 3, if text mode is used and the encoding - # is not specified, set the encoding to 'utf-8'. - binary = False - if len(args) > 1: - args = list(args) - if 'b' in args[1]: - binary = True - if kwargs.get('mode', None): - if 'b' in kwargs['mode']: - binary = True - if not binary: - kwargs['encoding'] = __salt_system_encoding__ - - if six.PY3 and not binary and not kwargs.get('newline', None): - kwargs['newline'] = '' - - fhandle = open(*args, **kwargs) # pylint: disable=resource-leakage - - if is_fcntl_available(): - # modify the file descriptor on systems with fcntl - # unix and unix-like systems only - try: - FD_CLOEXEC = fcntl.FD_CLOEXEC # pylint: disable=C0103 - except AttributeError: - FD_CLOEXEC = 1 # pylint: disable=C0103 - old_flags = fcntl.fcntl(fhandle.fileno(), fcntl.F_GETFD) - fcntl.fcntl(fhandle.fileno(), fcntl.F_SETFD, old_flags | FD_CLOEXEC) - - return fhandle - - -@contextlib.contextmanager -def flopen(*args, **kwargs): - ''' - Shortcut for fopen with lock and context manager - ''' - with fopen(*args, **kwargs) as fhandle: - try: - if is_fcntl_available(check_sunos=True): - fcntl.flock(fhandle.fileno(), fcntl.LOCK_SH) - yield fhandle - finally: - if is_fcntl_available(check_sunos=True): - fcntl.flock(fhandle.fileno(), fcntl.LOCK_UN) - - -@contextlib.contextmanager -def fpopen(*args, **kwargs): - ''' - Shortcut for fopen with extra uid, gid and mode options. - - Supported optional Keyword Arguments: - - mode: explicit mode to set. Mode is anything os.chmod - would accept as input for mode. Works only on unix/unix - like systems. - - uid: the uid to set, if not set, or it is None or -1 no changes are - made. Same applies if the path is already owned by this - uid. Must be int. Works only on unix/unix like systems. - - gid: the gid to set, if not set, or it is None or -1 no changes are - made. Same applies if the path is already owned by this - gid. Must be int. Works only on unix/unix like systems. - - ''' - # Remove uid, gid and mode from kwargs if present - uid = kwargs.pop('uid', -1) # -1 means no change to current uid - gid = kwargs.pop('gid', -1) # -1 means no change to current gid - mode = kwargs.pop('mode', None) - with fopen(*args, **kwargs) as fhandle: - path = args[0] - d_stat = os.stat(path) - - if hasattr(os, 'chown'): - # if uid and gid are both -1 then go ahead with - # no changes at all - if (d_stat.st_uid != uid or d_stat.st_gid != gid) and \ - [i for i in (uid, gid) if i != -1]: - os.chown(path, uid, gid) - - if mode is not None: - mode_part = S_IMODE(d_stat.st_mode) - if mode_part != mode: - os.chmod(path, (d_stat.st_mode ^ mode_part) | mode) - - yield fhandle - - def expr_match(line, expr): ''' Evaluate a line of text against an expression. First try a full-string @@ -1453,6 +1035,24 @@ def expr_match(line, expr): def check_whitelist_blacklist(value, whitelist=None, blacklist=None): ''' Check a whitelist and/or blacklist to see if the value matches it. + + value + The item to check the whitelist and/or blacklist against. + + whitelist + The list of items that are white-listed. If ``value`` is found + in the whitelist, then the function returns ``True``. Otherwise, + it returns ``False``. + + blacklist + The list of items that are black-listed. If ``value`` is found + in the blacklist, then the function returns ``False``. Otherwise, + it returns ``True``. + + If both a whitelist and a blacklist are provided, value membership + in the blacklist will be examined first. If the value is not found + in the blacklist, then the whitelist is checked. If the value isn't + found in the whitelist, the function returns ``False``. ''' if blacklist is not None: if not hasattr(blacklist, '__iter__'): @@ -1648,29 +1248,6 @@ def traverse_dict_and_list(data, key, default=None, delimiter=DEFAULT_TARGET_DEL return data -def clean_kwargs(**kwargs): - ''' - Return a dict without any of the __pub* keys (or any other keys starting - with a dunder) from the kwargs dict passed into the execution module - functions. These keys are useful for tracking what was used to invoke - the function call, but they may not be desirable to have if passing the - kwargs forward wholesale. - ''' - ret = {} - for key, val in six.iteritems(kwargs): - if not key.startswith('__'): - ret[key] = val - return ret - - -@real_memoize -def is_windows(): - ''' - Simple function to return if a host is Windows or not - ''' - return sys.platform.startswith('win') - - def sanitize_win_path_string(winpath): ''' Remove illegal path characters for windows @@ -1678,163 +1255,13 @@ def sanitize_win_path_string(winpath): intab = '<>:|?*' outtab = '_' * len(intab) trantab = ''.maketrans(intab, outtab) if six.PY3 else string.maketrans(intab, outtab) # pylint: disable=no-member - if isinstance(winpath, str): + if isinstance(winpath, six.string_types): winpath = winpath.translate(trantab) elif isinstance(winpath, six.text_type): winpath = winpath.translate(dict((ord(c), u'_') for c in intab)) return winpath -@real_memoize -def is_proxy(): - ''' - Return True if this minion is a proxy minion. - Leverages the fact that is_linux() and is_windows - both return False for proxies. - TODO: Need to extend this for proxies that might run on - other Unices - ''' - import __main__ as main - # This is a hack. If a proxy minion is started by other - # means, e.g. a custom script that creates the minion objects - # then this will fail. - is_proxy = False - try: - # Changed this from 'salt-proxy in main...' to 'proxy in main...' - # to support the testsuite's temp script that is called 'cli_salt_proxy' - if 'proxy' in main.__file__: - is_proxy = True - except AttributeError: - pass - return is_proxy - - -@real_memoize -def is_linux(): - ''' - Simple function to return if a host is Linux or not. - Note for a proxy minion, we need to return something else - ''' - return sys.platform.startswith('linux') - - -@real_memoize -def is_darwin(): - ''' - Simple function to return if a host is Darwin (macOS) or not - ''' - return sys.platform.startswith('darwin') - - -@real_memoize -def is_sunos(): - ''' - Simple function to return if host is SunOS or not - ''' - return sys.platform.startswith('sunos') - - -@real_memoize -def is_smartos(): - ''' - Simple function to return if host is SmartOS (Illumos) or not - ''' - if not is_sunos(): - return False - else: - return os.uname()[3].startswith('joyent_') - - -@real_memoize -def is_smartos_globalzone(): - ''' - Function to return if host is SmartOS (Illumos) global zone or not - ''' - if not is_smartos(): - return False - else: - cmd = ['zonename'] - try: - zonename = subprocess.Popen( - cmd, shell=False, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - except OSError: - return False - if zonename.returncode: - return False - if zonename.stdout.read().strip() == 'global': - return True - - return False - - -@real_memoize -def is_smartos_zone(): - ''' - Function to return if host is SmartOS (Illumos) and not the gz - ''' - if not is_smartos(): - return False - else: - cmd = ['zonename'] - try: - zonename = subprocess.Popen( - cmd, shell=False, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - except OSError: - return False - if zonename.returncode: - return False - if zonename.stdout.read().strip() == 'global': - return False - - return True - - -@real_memoize -def is_freebsd(): - ''' - Simple function to return if host is FreeBSD or not - ''' - return sys.platform.startswith('freebsd') - - -@real_memoize -def is_netbsd(): - ''' - Simple function to return if host is NetBSD or not - ''' - return sys.platform.startswith('netbsd') - - -@real_memoize -def is_openbsd(): - ''' - Simple function to return if host is OpenBSD or not - ''' - return sys.platform.startswith('openbsd') - - -@real_memoize -def is_aix(): - ''' - Simple function to return if host is AIX or not - ''' - return sys.platform.startswith('aix') - - -def is_fcntl_available(check_sunos=False): - ''' - Simple function to check if the `fcntl` module is available or not. - - If `check_sunos` is passed as `True` an additional check to see if host is - SunOS is also made. For additional information see: http://goo.gl/159FF8 - ''' - if check_sunos and is_sunos(): - return False - return HAS_FCNTL - - def check_include_exclude(path_str, include_pat=None, exclude_pat=None): ''' Check for glob or regexp patterns for include_pat and exclude_pat in the @@ -1885,148 +1312,6 @@ def check_include_exclude(path_str, include_pat=None, exclude_pat=None): return ret -def gen_state_tag(low): - ''' - Generate the running dict tag string from the low data structure - ''' - return '{0[state]}_|-{0[__id__]}_|-{0[name]}_|-{0[fun]}'.format(low) - - -def search_onfail_requisites(sid, highstate): - """ - For a particular low chunk, search relevant onfail related - states - """ - onfails = [] - if '_|-' in sid: - st = salt.state.split_low_tag(sid) - else: - st = {'__id__': sid} - for fstate, fchunks in six.iteritems(highstate): - if fstate == st['__id__']: - continue - else: - for mod_, fchunk in six.iteritems(fchunks): - if ( - not isinstance(mod_, six.string_types) or - mod_.startswith('__') - ): - continue - else: - if not isinstance(fchunk, list): - continue - else: - # bydefault onfail will fail, but you can - # set onfail_stop: False to prevent the highstate - # to stop if you handle it - onfail_handled = False - for fdata in fchunk: - if not isinstance(fdata, dict): - continue - onfail_handled = (fdata.get('onfail_stop', True) - is False) - if onfail_handled: - break - if not onfail_handled: - continue - for fdata in fchunk: - if not isinstance(fdata, dict): - continue - for knob, fvalue in six.iteritems(fdata): - if knob != 'onfail': - continue - for freqs in fvalue: - for fmod, fid in six.iteritems(freqs): - if not ( - fid == st['__id__'] and - fmod == st.get('state', fmod) - ): - continue - onfails.append((fstate, mod_, fchunk)) - return onfails - - -def check_onfail_requisites(state_id, state_result, running, highstate): - ''' - When a state fail and is part of a highstate, check - if there is onfail requisites. - When we find onfail requisites, we will consider the state failed - only if at least one of those onfail requisites also failed - - Returns: - - True: if onfail handlers suceeded - False: if one on those handler failed - None: if the state does not have onfail requisites - - ''' - nret = None - if ( - state_id and state_result and - highstate and isinstance(highstate, dict) - ): - onfails = search_onfail_requisites(state_id, highstate) - if onfails: - for handler in onfails: - fstate, mod_, fchunk = handler - ofresult = True - for rstateid, rstate in six.iteritems(running): - if '_|-' in rstateid: - st = salt.state.split_low_tag(rstateid) - # in case of simple state, try to guess - else: - id_ = rstate.get('__id__', rstateid) - if not id_: - raise ValueError('no state id') - st = {'__id__': id_, 'state': mod_} - if mod_ == st['state'] and fstate == st['__id__']: - ofresult = rstate.get('result', _empty) - if ofresult in [False, True]: - nret = ofresult - if ofresult is False: - # as soon as we find an errored onfail, we stop - break - # consider that if we parsed onfailes without changing - # the ret, that we have failed - if nret is None: - nret = False - return nret - - -def check_state_result(running, recurse=False, highstate=None): - ''' - Check the total return value of the run and determine if the running - dict has any issues - ''' - if not isinstance(running, dict): - return False - - if not running: - return False - - ret = True - for state_id, state_result in six.iteritems(running): - if not recurse and not isinstance(state_result, dict): - ret = False - if ret and isinstance(state_result, dict): - result = state_result.get('result', _empty) - if result is False: - ret = False - # only override return value if we are not already failed - elif result is _empty and isinstance(state_result, dict) and ret: - ret = check_state_result( - state_result, recurse=True, highstate=highstate) - # if we detect a fail, check for onfail requisites - if not ret: - # ret can be None in case of no onfail reqs, recast it to bool - ret = bool(check_onfail_requisites(state_id, state_result, - running, highstate)) - # return as soon as we got a failure - if not ret: - break - return ret - - def st_mode_to_octal(mode): ''' Convert the st_mode value from a stat(2) call (as returned from os.stat()) @@ -2092,7 +1377,7 @@ def is_true(value=None): pass # Now check for truthiness - if isinstance(value, (int, float)): + if isinstance(value, (six.integer_types, float)): return value > 0 elif isinstance(value, six.string_types): return str(value).lower() == 'true' @@ -2118,34 +1403,6 @@ def exactly_one(l): return exactly_n(l) -def rm_rf(path): - ''' - Platform-independent recursive delete. Includes code from - http://stackoverflow.com/a/2656405 - ''' - def _onerror(func, path, exc_info): - ''' - Error handler for `shutil.rmtree`. - - If the error is due to an access error (read only file) - it attempts to add write permission and then retries. - - If the error is for another reason it re-raises the error. - - Usage : `shutil.rmtree(path, onerror=onerror)` - ''' - if is_windows() and not os.access(path, os.W_OK): - # Is the error an access error ? - os.chmod(path, stat.S_IWUSR) - func(path) - else: - raise # pylint: disable=E0704 - if os.path.islink(path) or not os.path.isdir(path): - os.remove(path) - else: - shutil.rmtree(path, onerror=_onerror) - - def option(value, default='', opts=None, pillar=None): ''' Pass in a generic option and receive the value that will be assigned @@ -2166,44 +1423,6 @@ def option(value, default='', opts=None, pillar=None): return default -def parse_docstring(docstring): - ''' - Parse a docstring into its parts. - - Currently only parses dependencies, can be extended to parse whatever is - needed. - - Parses into a dictionary: - { - 'full': full docstring, - 'deps': list of dependencies (empty list if none) - } - ''' - # First try with regex search for :depends: - ret = {} - ret['full'] = docstring - regex = r'([ \t]*):depends:[ \t]+- (\w+)[^\n]*\n(\1[ \t]+- (\w+)[^\n]*\n)*' - match = re.search(regex, docstring, re.M) - if match: - deps = [] - regex = r'- (\w+)' - for line in match.group(0).strip().splitlines(): - deps.append(re.search(regex, line).group(1)) - ret['deps'] = deps - return ret - # Try searching for a one-liner instead - else: - txt = 'Required python modules: ' - data = docstring.splitlines() - dep_list = list(x for x in data if x.strip().startswith(txt)) - if not dep_list: - ret['deps'] = [] - return ret - deps = dep_list[0].replace(txt, '').strip().split(', ') - ret['deps'] = deps - return ret - - def print_cli(msg, retries=10, step=0.01): ''' Wrapper around print() that suppresses tracebacks on broken pipes (i.e. @@ -2293,11 +1512,14 @@ def get_hash(path, form='sha256', chunk_size=65536): ``get_sum`` cannot really be trusted since it is vulnerable to collisions: ``get_sum(..., 'xyz') == 'Hash xyz not supported'`` ''' + # Late import to avoid circular import. + import salt.utils.files + hash_type = hasattr(hashlib, form) and getattr(hashlib, form) or None if hash_type is None: raise ValueError('Invalid hash type: {0}'.format(form)) - with salt.utils.fopen(path, 'rb') as ifile: + with salt.utils.files.fopen(path, 'rb') as ifile: hash_obj = hash_type() # read the file in in chunks, not the entire file for chunk in iter(lambda: ifile.read(chunk_size), b''): @@ -2431,228 +1653,6 @@ def date_format(date=None, format="%Y-%m-%d"): return date_cast(date).strftime(format) -def warn_until(version, - message, - category=DeprecationWarning, - stacklevel=None, - _version_info_=None, - _dont_call_warnings=False): - ''' - Helper function to raise a warning, by default, a ``DeprecationWarning``, - until the provided ``version``, after which, a ``RuntimeError`` will - be raised to remind the developers to remove the warning because the - target version has been reached. - - :param version: The version info or name after which the warning becomes a - ``RuntimeError``. For example ``(0, 17)`` or ``Hydrogen`` - or an instance of :class:`salt.version.SaltStackVersion`. - :param message: The warning message to be displayed. - :param category: The warning class to be thrown, by default - ``DeprecationWarning`` - :param stacklevel: There should be no need to set the value of - ``stacklevel``. Salt should be able to do the right thing. - :param _version_info_: In order to reuse this function for other SaltStack - projects, they need to be able to provide the - version info to compare to. - :param _dont_call_warnings: This parameter is used just to get the - functionality until the actual error is to be - issued. When we're only after the salt version - checks to raise a ``RuntimeError``. - ''' - if not isinstance(version, (tuple, - six.string_types, - salt.version.SaltStackVersion)): - raise RuntimeError( - 'The \'version\' argument should be passed as a tuple, string or ' - 'an instance of \'salt.version.SaltStackVersion\'.' - ) - elif isinstance(version, tuple): - version = salt.version.SaltStackVersion(*version) - elif isinstance(version, six.string_types): - version = salt.version.SaltStackVersion.from_name(version) - - if stacklevel is None: - # Attribute the warning to the calling function, not to warn_until() - stacklevel = 2 - - if _version_info_ is None: - _version_info_ = salt.version.__version_info__ - - _version_ = salt.version.SaltStackVersion(*_version_info_) - - if _version_ >= version: - import inspect - caller = inspect.getframeinfo(sys._getframe(stacklevel - 1)) - raise RuntimeError( - 'The warning triggered on filename \'{filename}\', line number ' - '{lineno}, is supposed to be shown until version ' - '{until_version} is released. Current version is now ' - '{salt_version}. Please remove the warning.'.format( - filename=caller.filename, - lineno=caller.lineno, - until_version=version.formatted_version, - salt_version=_version_.formatted_version - ), - ) - - if _dont_call_warnings is False: - def _formatwarning(message, - category, - filename, - lineno, - line=None): # pylint: disable=W0613 - ''' - Replacement for warnings.formatwarning that disables the echoing of - the 'line' parameter. - ''' - return '{0}:{1}: {2}: {3}\n'.format( - filename, lineno, category.__name__, message - ) - saved = warnings.formatwarning - warnings.formatwarning = _formatwarning - warnings.warn( - message.format(version=version.formatted_version), - category, - stacklevel=stacklevel - ) - warnings.formatwarning = saved - - -def kwargs_warn_until(kwargs, - version, - category=DeprecationWarning, - stacklevel=None, - _version_info_=None, - _dont_call_warnings=False): - ''' - Helper function to raise a warning (by default, a ``DeprecationWarning``) - when unhandled keyword arguments are passed to function, until the - provided ``version_info``, after which, a ``RuntimeError`` will be raised - to remind the developers to remove the ``**kwargs`` because the target - version has been reached. - This function is used to help deprecate unused legacy ``**kwargs`` that - were added to function parameters lists to preserve backwards compatibility - when removing a parameter. See - :ref:`the deprecation development docs ` - for the modern strategy for deprecating a function parameter. - - :param kwargs: The caller's ``**kwargs`` argument value (a ``dict``). - :param version: The version info or name after which the warning becomes a - ``RuntimeError``. For example ``(0, 17)`` or ``Hydrogen`` - or an instance of :class:`salt.version.SaltStackVersion`. - :param category: The warning class to be thrown, by default - ``DeprecationWarning`` - :param stacklevel: There should be no need to set the value of - ``stacklevel``. Salt should be able to do the right thing. - :param _version_info_: In order to reuse this function for other SaltStack - projects, they need to be able to provide the - version info to compare to. - :param _dont_call_warnings: This parameter is used just to get the - functionality until the actual error is to be - issued. When we're only after the salt version - checks to raise a ``RuntimeError``. - ''' - if not isinstance(version, (tuple, - six.string_types, - salt.version.SaltStackVersion)): - raise RuntimeError( - 'The \'version\' argument should be passed as a tuple, string or ' - 'an instance of \'salt.version.SaltStackVersion\'.' - ) - elif isinstance(version, tuple): - version = salt.version.SaltStackVersion(*version) - elif isinstance(version, six.string_types): - version = salt.version.SaltStackVersion.from_name(version) - - if stacklevel is None: - # Attribute the warning to the calling function, - # not to kwargs_warn_until() or warn_until() - stacklevel = 3 - - if _version_info_ is None: - _version_info_ = salt.version.__version_info__ - - _version_ = salt.version.SaltStackVersion(*_version_info_) - - if kwargs or _version_.info >= version.info: - arg_names = ', '.join('\'{0}\''.format(key) for key in kwargs) - warn_until( - version, - message='The following parameter(s) have been deprecated and ' - 'will be removed in \'{0}\': {1}.'.format(version.string, - arg_names), - category=category, - stacklevel=stacklevel, - _version_info_=_version_.info, - _dont_call_warnings=_dont_call_warnings - ) - - -def version_cmp(pkg1, pkg2, ignore_epoch=False): - ''' - Compares two version strings using salt.utils.versions.LooseVersion. This is - a fallback for providers which don't have a version comparison utility - built into them. Return -1 if version1 < version2, 0 if version1 == - version2, and 1 if version1 > version2. Return None if there was a problem - making the comparison. - ''' - normalize = lambda x: str(x).split(':', 1)[-1] if ignore_epoch else str(x) - pkg1 = normalize(pkg1) - pkg2 = normalize(pkg2) - - try: - # pylint: disable=no-member - if _LooseVersion(pkg1) < _LooseVersion(pkg2): - return -1 - elif _LooseVersion(pkg1) == _LooseVersion(pkg2): - return 0 - elif _LooseVersion(pkg1) > _LooseVersion(pkg2): - return 1 - except Exception as exc: - log.exception(exc) - return None - - -def compare_versions(ver1='', - oper='==', - ver2='', - cmp_func=None, - ignore_epoch=False): - ''' - Compares two version numbers. Accepts a custom function to perform the - cmp-style version comparison, otherwise uses version_cmp(). - ''' - cmp_map = {'<': (-1,), '<=': (-1, 0), '==': (0,), - '>=': (0, 1), '>': (1,)} - if oper not in ('!=',) and oper not in cmp_map: - log.error('Invalid operator \'%s\' for version comparison', oper) - return False - - if cmp_func is None: - cmp_func = version_cmp - - cmp_result = cmp_func(ver1, ver2, ignore_epoch=ignore_epoch) - if cmp_result is None: - return False - - # Check if integer/long - if not isinstance(cmp_result, numbers.Integral): - log.error('The version comparison function did not return an ' - 'integer/long.') - return False - - if oper == '!=': - return cmp_result not in cmp_map['=='] - else: - # Gracefully handle cmp_result not in (-1, 0, 1). - if cmp_result < -1: - cmp_result = -1 - elif cmp_result > 1: - cmp_result = 1 - - return cmp_result in cmp_map[oper] - - @jinja_filter('compare_dicts') def compare_dicts(old=None, new=None): ''' @@ -2692,51 +1692,6 @@ def compare_lists(old=None, new=None): return ret -def argspec_report(functions, module=''): - ''' - Pass in a functions dict as it is returned from the loader and return the - argspec function signatures - ''' - ret = {} - if '*' in module or '.' in module: - for fun in fnmatch.filter(functions, module): - try: - aspec = salt.utils.args.get_function_argspec(functions[fun]) - except TypeError: - # this happens if not callable - continue - - args, varargs, kwargs, defaults = aspec - - ret[fun] = {} - ret[fun]['args'] = args if args else None - ret[fun]['defaults'] = defaults if defaults else None - ret[fun]['varargs'] = True if varargs else None - ret[fun]['kwargs'] = True if kwargs else None - - else: - # "sys" should just match sys without also matching sysctl - moduledot = module + '.' - - for fun in functions: - if fun.startswith(moduledot): - try: - aspec = salt.utils.args.get_function_argspec(functions[fun]) - except TypeError: - # this happens if not callable - continue - - args, varargs, kwargs, defaults = aspec - - ret[fun] = {} - ret[fun]['args'] = args if args else None - ret[fun]['defaults'] = defaults if defaults else None - ret[fun]['varargs'] = True if varargs else None - ret[fun]['kwargs'] = True if kwargs else None - - return ret - - @jinja_filter('json_decode_list') def decode_list(data): ''' @@ -2798,47 +1753,25 @@ def is_bin_file(path): Detects if the file is a binary, returns bool. Returns True if the file is a bin, False if the file is not and None if the file is not available. ''' + # Late import to avoid circular import. + import salt.utils.files + import salt.utils.stringutils + if not os.path.isfile(path): return False try: - with fopen(path, 'rb') as fp_: + with salt.utils.files.fopen(path, 'rb') as fp_: try: data = fp_.read(2048) if six.PY3: data = data.decode(__salt_system_encoding__) - return is_bin_str(data) + return salt.utils.stringutils.is_binary(data) except UnicodeDecodeError: return True except os.error: return False -def is_bin_str(data): - ''' - Detects if the passed string of data is bin or text - ''' - if '\0' in data: - return True - if not data: - return False - - text_characters = ''.join([chr(x) for x in range(32, 127)] + list('\n\r\t\b')) - # Get the non-text characters (map each character to itself then use the - # 'remove' option to get rid of the text characters.) - if six.PY3: - trans = ''.maketrans('', '', text_characters) - nontext = data.translate(trans) - else: - trans = string.maketrans('', '') # pylint: disable=no-member - nontext = data.translate(trans, text_characters) - - # If more than 30% non-text characters, then - # this is considered a binary file - if len(nontext) / len(data) > 0.30: - return True - return False - - def is_dictlist(data): ''' Returns True if data is a list of one-element dicts (as found in many SLS @@ -2877,7 +1810,7 @@ def repack_dictlist(data, if val_cb is None: val_cb = lambda x, y: y - valid_non_dict = (six.string_types, int, float) + valid_non_dict = (six.string_types, six.integer_types, float) if isinstance(data, list): for element in data: if isinstance(element, valid_non_dict): @@ -3128,41 +2061,6 @@ def chugid_and_umask(runas, umask): os.umask(umask) -@jinja_filter('random_str') -def rand_string(size=32): - key = os.urandom(size) - return key.encode('base64').replace('\n', '') - - -def relpath(path, start='.'): - ''' - Work around Python bug #5117, which is not (and will not be) patched in - Python 2.6 (http://bugs.python.org/issue5117) - ''' - if sys.version_info < (2, 7) and 'posix' in sys.builtin_module_names: - # The below code block is based on posixpath.relpath from Python 2.7, - # which has the fix for this bug. - if not path: - raise ValueError('no path specified') - - start_list = [ - x for x in os.path.abspath(start).split(os.path.sep) if x - ] - path_list = [ - x for x in os.path.abspath(path).split(os.path.sep) if x - ] - - # work out how much of the filepath is shared by start and path. - i = len(os.path.commonprefix([start_list, path_list])) - - rel_list = [os.path.pardir] * (len(start_list)-i) + path_list[i:] - if not rel_list: - return os.path.curdir - return os.path.join(*rel_list) - - return os.path.relpath(path, start=start) - - def human_size_to_bytes(human_size): ''' Convert human-readable units to bytes @@ -3180,59 +2078,6 @@ def human_size_to_bytes(human_size): return size_num * unit_multiplier -def to_str(s, encoding=None): - ''' - Given str, bytes, bytearray, or unicode (py2), return str - ''' - if isinstance(s, str): - return s - if six.PY3: - if isinstance(s, (bytes, bytearray)): - # https://docs.python.org/3/howto/unicode.html#the-unicode-type - # replace error with U+FFFD, REPLACEMENT CHARACTER - return s.decode(encoding or __salt_system_encoding__, "replace") - raise TypeError('expected str, bytes, or bytearray not {}'.format(type(s))) - else: - if isinstance(s, bytearray): - return str(s) - if isinstance(s, unicode): # pylint: disable=incompatible-py3-code,undefined-variable - return s.encode(encoding or __salt_system_encoding__) - raise TypeError('expected str, bytearray, or unicode') - - -@jinja_filter('to_bytes') -def to_bytes(s, encoding=None): - ''' - Given bytes, bytearray, str, or unicode (python 2), return bytes (str for - python 2) - ''' - if six.PY3: - if isinstance(s, bytes): - return s - if isinstance(s, bytearray): - return bytes(s) - if isinstance(s, str): - return s.encode(encoding or __salt_system_encoding__) - raise TypeError('expected bytes, bytearray, or str') - else: - return to_str(s, encoding) - - -def to_unicode(s, encoding=None): - ''' - Given str or unicode, return unicode (str for python 3) - ''' - if not isinstance(s, (bytes, bytearray, six.string_types)): - return s - if six.PY3: - if isinstance(s, (bytes, bytearray)): - return to_str(s, encoding) - else: - if isinstance(s, str): - return s.decode(encoding or __salt_system_encoding__) - return s - - @jinja_filter('is_list') def is_list(value): ''' @@ -3265,37 +2110,6 @@ def is_iter(y, ignore=six.string_types): return False -def invalid_kwargs(invalid_kwargs, raise_exc=True): - ''' - Raise a SaltInvocationError if invalid_kwargs is non-empty - ''' - if invalid_kwargs: - if isinstance(invalid_kwargs, dict): - new_invalid = [ - '{0}={1}'.format(x, y) - for x, y in six.iteritems(invalid_kwargs) - ] - invalid_kwargs = new_invalid - msg = ( - 'The following keyword arguments are not valid: {0}' - .format(', '.join(invalid_kwargs)) - ) - if raise_exc: - raise SaltInvocationError(msg) - else: - return msg - - -def shlex_split(s, **kwargs): - ''' - Only split if variable is a string - ''' - if isinstance(s, six.string_types): - return shlex.split(s, **kwargs) - else: - return s - - def split_input(val): ''' Take an input value and split it into a list, returning the resulting list @@ -3308,41 +2122,6 @@ def split_input(val): return [x.strip() for x in str(val).split(',')] -def str_version_to_evr(verstring): - ''' - Split the package version string into epoch, version and release. - Return this as tuple. - - The epoch is always not empty. The version and the release can be an empty - string if such a component could not be found in the version string. - - "2:1.0-1.2" => ('2', '1.0', '1.2) - "1.0" => ('0', '1.0', '') - "" => ('0', '', '') - ''' - if verstring in [None, '']: - return '0', '', '' - - idx_e = verstring.find(':') - if idx_e != -1: - try: - epoch = str(int(verstring[:idx_e])) - except ValueError: - # look, garbage in the epoch field, how fun, kill it - epoch = '0' # this is our fallback, deal - else: - epoch = '0' - idx_r = verstring.find('-') - if idx_r != -1: - version = verstring[idx_e + 1:idx_r] - release = verstring[idx_r + 1:] - else: - version = verstring[idx_e + 1:] - release = '' - - return epoch, version, release - - def simple_types_filter(data): ''' Convert the data list, dictionary into simple types, i.e., int, float, string, @@ -3463,24 +2242,1058 @@ def fnmatch_multiple(candidates, pattern): return None -def is_quoted(val): +# +# MOVED FUNCTIONS +# +# These are deprecated and will be removed in Neon. +def to_bytes(s, encoding=None): + ''' + Given bytes, bytearray, str, or unicode (python 2), return bytes (str for + python 2) + + .. deprecated:: Oxygen + ''' + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.stringutils + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.to_bytes\' detected. This function has been ' + 'moved to \'salt.utils.stringutils.to_bytes\' as of Salt Oxygen. ' + 'This warning will be removed in Salt Neon.' + ) + return salt.utils.stringutils.to_bytes(s, encoding) + + +def to_str(s, encoding=None): + ''' + Given str, bytes, bytearray, or unicode (py2), return str + + .. deprecated:: Oxygen + ''' + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.stringutils + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.to_str\' detected. This function has been moved ' + 'to \'salt.utils.stringutils.to_str\' as of Salt Oxygen. This ' + 'warning will be removed in Salt Neon.' + ) + return salt.utils.stringutils.to_str(s, encoding) + + +def to_unicode(s, encoding=None): + ''' + Given str or unicode, return unicode (str for python 3) + + .. deprecated:: Oxygen + ''' + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.stringutils + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.to_unicode\' detected. This function has been ' + 'moved to \'salt.utils.stringutils.to_unicode\' as of Salt Oxygen. ' + 'This warning will be removed in Salt Neon.' + ) + return salt.utils.stringutils.to_unicode(s, encoding) + + +def str_to_num(text): + ''' + Convert a string to a number. + Returns an integer if the string represents an integer, a floating + point number if the string is a real number, or the string unchanged + otherwise. + + .. deprecated:: Oxygen + ''' + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.stringutils + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.str_to_num\' detected. This function has been ' + 'moved to \'salt.utils.stringutils.to_num\' as of Salt Oxygen. This ' + 'warning will be removed in Salt Neon.' + ) + return salt.utils.stringutils.to_num(text) + + +def is_quoted(value): ''' Return a single or double quote, if a string is wrapped in extra quotes. Otherwise return an empty string. + + .. deprecated:: Oxygen ''' - ret = '' - if ( - isinstance(val, six.string_types) and val[0] == val[-1] and - val.startswith(('\'', '"')) - ): - ret = val[0] - return ret + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.stringutils + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.is_quoted\' detected. This function has been ' + 'moved to \'salt.utils.stringutils.is_quoted\' as of Salt Oxygen. ' + 'This warning will be removed in Salt Neon.' + ) + return salt.utils.stringutils.is_quoted(value) -def dequote(val): +def dequote(value): ''' Remove extra quotes around a string. + + .. deprecated:: Oxygen ''' - if is_quoted(val): - return val[1:-1] - return val + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.stringutils + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.dequote\' detected. This function has been moved ' + 'to \'salt.utils.stringutils.dequote\' as of Salt Oxygen. This ' + 'warning will be removed in Salt Neon.' + ) + return salt.utils.stringutils.dequote(value) + + +def is_hex(value): + ''' + Returns True if value is a hexidecimal string, otherwise returns False + + .. deprecated:: Oxygen + ''' + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.stringutils + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.is_hex\' detected. This function has been moved ' + 'to \'salt.utils.stringutils.is_hex\' as of Salt Oxygen. This warning ' + 'will be removed in Salt Neon.' + ) + return salt.utils.stringutils.is_hex(value) + + +def is_bin_str(data): + ''' + Detects if the passed string of data is binary or text + + .. deprecated:: Oxygen + ''' + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.stringutils + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.is_bin_str\' detected. This function has been ' + 'moved to \'salt.utils.stringutils.is_binary\' as of Salt Oxygen. ' + 'This warning will be removed in Salt Neon.' + ) + return salt.utils.stringutils.is_binary(data) + + +def rand_string(size=32): + ''' + .. deprecated:: Oxygen + ''' + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.stringutils + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.rand_string\' detected. This function has been ' + 'moved to \'salt.utils.stringutils.random\' as of Salt Oxygen. This ' + 'warning will be removed in Salt Neon.' + ) + return salt.utils.stringutils.random(size) + + +def contains_whitespace(text): + ''' + Returns True if there are any whitespace characters in the string + + .. deprecated:: Oxygen + ''' + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.stringutils + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.contains_whitespace\' detected. This function ' + 'has been moved to \'salt.utils.stringutils.contains_whitespace\' as ' + 'of Salt Oxygen. This warning will be removed in Salt Neon.' + ) + return salt.utils.stringutils.contains_whitespace(text) + + +def clean_kwargs(**kwargs): + ''' + Return a dict without any of the __pub* keys (or any other keys starting + with a dunder) from the kwargs dict passed into the execution module + functions. These keys are useful for tracking what was used to invoke + the function call, but they may not be desirable to have if passing the + kwargs forward wholesale. + + .. deprecated:: Oxygen + ''' + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.args + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.clean_kwargs\' detected. This function has been ' + 'moved to \'salt.utils.args.clean_kwargs\' as of Salt Oxygen. This ' + 'warning will be removed in Salt Neon.' + ) + return salt.utils.args.clean_kwargs(**kwargs) + + +def invalid_kwargs(invalid_kwargs, raise_exc=True): + ''' + Raise a SaltInvocationError if invalid_kwargs is non-empty + + .. deprecated:: Oxygen + ''' + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.args + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.invalid_kwargs\' detected. This function has ' + 'been moved to \'salt.utils.args.invalid_kwargs\' as of Salt Oxygen. ' + 'This warning will be removed in Salt Neon.' + ) + return salt.utils.args.invalid_kwargs(invalid_kwargs, raise_exc) + + +def shlex_split(s, **kwargs): + ''' + Only split if variable is a string + + .. deprecated:: Oxygen + ''' + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.args + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.shlex_split\' detected. This function has been ' + 'moved to \'salt.utils.args.shlex_split\' as of Salt Oxygen. This ' + 'warning will be removed in Salt Neon.' + ) + return salt.utils.args.shlex_split(s, **kwargs) + + +def arg_lookup(fun, aspec=None): + ''' + Return a dict containing the arguments and default arguments to the + function. + + .. deprecated:: Oxygen + ''' + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.args + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.arg_lookup\' detected. This function has been ' + 'moved to \'salt.utils.args.arg_lookup\' as of Salt Oxygen. This ' + 'warning will be removed in Salt Neon.' + ) + return salt.utils.args.arg_lookup(fun, aspec=aspec) + + +def argspec_report(functions, module=''): + ''' + Pass in a functions dict as it is returned from the loader and return the + argspec function signatures + + .. deprecated:: Oxygen + ''' + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.args + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.argspec_report\' detected. This function has been ' + 'moved to \'salt.utils.args.argspec_report\' as of Salt Oxygen. This ' + 'warning will be removed in Salt Neon.' + ) + return salt.utils.args.argspec_report(functions, module=module) + + +def which(exe=None): + ''' + Python clone of /usr/bin/which + + .. deprecated:: Oxygen + ''' + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.path + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.which\' detected. This function has been moved to ' + '\'salt.utils.path.which\' as of Salt Oxygen. This warning will be ' + 'removed in Salt Neon.' + ) + return salt.utils.path.which(exe) + + +def which_bin(exes): + ''' + Scan over some possible executables and return the first one that is found + + .. deprecated:: Oxygen + ''' + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.path + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.which_bin\' detected. This function has been ' + 'moved to \'salt.utils.path.which_bin\' as of Salt Oxygen. This ' + 'warning will be removed in Salt Neon.' + ) + return salt.utils.path.which_bin(exes) + + +def path_join(*parts, **kwargs): + ''' + This functions tries to solve some issues when joining multiple absolute + paths on both *nix and windows platforms. + + See tests/unit/utils/test_path.py for some examples on what's being + talked about here. + + The "use_posixpath" kwarg can be be used to force joining using poxixpath, + which is useful for Salt fileserver paths on Windows masters. + + .. deprecated:: Oxygen + ''' + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.path + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.path_join\' detected. This function has been ' + 'moved to \'salt.utils.path.join\' as of Salt Oxygen. This warning ' + 'will be removed in Salt Neon.' + ) + return salt.utils.path.join(*parts, **kwargs) + + +def rand_str(size=9999999999, hash_type=None): + ''' + Return a hash of a randomized data from random.SystemRandom() + + .. deprecated:: Oxygen + ''' + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.hashutils + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.rand_str\' detected. This function has been ' + 'moved to \'salt.utils.hashutils.random_hash\' as of Salt Oxygen. ' + 'This warning will be removed in Salt Neon.' + ) + return salt.utils.hashutils.random_hash(size, hash_type) + + +def is_windows(): + ''' + Simple function to return if a host is Windows or not + + .. deprecated:: Oxygen + ''' + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.platform + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.is_windows\' detected. This function has been ' + 'moved to \'salt.utils.platform.is_windows\' as of Salt Oxygen. This ' + 'warning will be removed in Salt Neon.' + ) + return salt.utils.platform.is_windows() + + +def is_proxy(): + ''' + Return True if this minion is a proxy minion. + Leverages the fact that is_linux() and is_windows + both return False for proxies. + TODO: Need to extend this for proxies that might run on + other Unices + + .. deprecated:: Oxygen + ''' + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.platform + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.is_proxy\' detected. This function has been ' + 'moved to \'salt.utils.platform.is_proxy\' as of Salt Oxygen. This ' + 'warning will be removed in Salt Neon.' + ) + return salt.utils.platform.is_proxy() + + +def is_linux(): + ''' + Simple function to return if a host is Linux or not. + Note for a proxy minion, we need to return something else + + .. deprecated:: Oxygen + ''' + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.platform + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.is_linux\' detected. This function has been ' + 'moved to \'salt.utils.platform.is_linux\' as of Salt Oxygen. This ' + 'warning will be removed in Salt Neon.' + ) + return salt.utils.platform.is_linux() + + +def is_darwin(): + ''' + Simple function to return if a host is Darwin (macOS) or not + + .. deprecated:: Oxygen + ''' + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.platform + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.is_darwin\' detected. This function has been ' + 'moved to \'salt.utils.platform.is_darwin\' as of Salt Oxygen. This ' + 'warning will be removed in Salt Neon.' + ) + return salt.utils.platform.is_darwin() + + +def is_sunos(): + ''' + Simple function to return if host is SunOS or not + + .. deprecated:: Oxygen + ''' + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.platform + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.is_sunos\' detected. This function has been ' + 'moved to \'salt.utils.platform.is_sunos\' as of Salt Oxygen. This ' + 'warning will be removed in Salt Neon.' + ) + return salt.utils.platform.is_sunos() + + +def is_smartos(): + ''' + Simple function to return if host is SmartOS (Illumos) or not + + .. deprecated:: Oxygen + ''' + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.platform + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.is_smartos\' detected. This function has been ' + 'moved to \'salt.utils.platform.is_smartos\' as of Salt Oxygen. This ' + 'warning will be removed in Salt Neon.' + ) + return salt.utils.platform.is_smartos() + + +def is_smartos_globalzone(): + ''' + Function to return if host is SmartOS (Illumos) global zone or not + + .. deprecated:: Oxygen + ''' + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.platform + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.is_smartos_globalzone\' detected. This function ' + 'has been moved to \'salt.utils.platform.is_smartos_globalzone\' as ' + 'of Salt Oxygen. This warning will be removed in Salt Neon.' + ) + return salt.utils.platform.is_smartos_globalzone() + + +def is_smartos_zone(): + ''' + Function to return if host is SmartOS (Illumos) and not the gz + + .. deprecated:: Oxygen + ''' + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.platform + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.is_smartos_zone\' detected. This function has ' + 'been moved to \'salt.utils.platform.is_smartos_zone\' as of Salt ' + 'Oxygen. This warning will be removed in Salt Neon.' + ) + return salt.utils.platform.is_smartos_zone() + + +def is_freebsd(): + ''' + Simple function to return if host is FreeBSD or not + + .. deprecated:: Oxygen + ''' + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.platform + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.is_freebsd\' detected. This function has been ' + 'moved to \'salt.utils.platform.is_freebsd\' as of Salt Oxygen. This ' + 'warning will be removed in Salt Neon.' + ) + return salt.utils.platform.is_freebsd() + + +def is_netbsd(): + ''' + Simple function to return if host is NetBSD or not + + .. deprecated:: Oxygen + ''' + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.platform + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.is_netbsd\' detected. This function has been ' + 'moved to \'salt.utils.platform.is_netbsd\' as of Salt Oxygen. This ' + 'warning will be removed in Salt Neon.' + ) + return salt.utils.platform.is_netbsd() + + +def is_openbsd(): + ''' + Simple function to return if host is OpenBSD or not + + .. deprecated:: Oxygen + ''' + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.platform + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.is_openbsd\' detected. This function has been ' + 'moved to \'salt.utils.platform.is_openbsd\' as of Salt Oxygen. This ' + 'warning will be removed in Salt Neon.' + ) + return salt.utils.platform.is_openbsd() + + +def is_aix(): + ''' + Simple function to return if host is AIX or not + + .. deprecated:: Oxygen + ''' + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.platform + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.is_aix\' detected. This function has been moved to ' + '\'salt.utils.platform.is_aix\' as of Salt Oxygen. This warning will be ' + 'removed in Salt Neon.' + ) + return salt.utils.platform.is_aix() + + +def safe_rm(tgt): + ''' + Safely remove a file + + .. deprecated:: Oxygen + ''' + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.files + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.safe_rm\' detected. This function has been moved to ' + '\'salt.utils.files.safe_rm\' as of Salt Oxygen. This warning will be ' + 'removed in Salt Neon.' + ) + return salt.utils.files.safe_rm(tgt) + + +@jinja_filter('is_empty') +def is_empty(filename): + ''' + Is a file empty? + + .. deprecated:: Oxygen + ''' + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.files + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.is_empty\' detected. This function has been moved to ' + '\'salt.utils.files.is_empty\' as of Salt Oxygen. This warning will be ' + 'removed in Salt Neon.' + ) + return salt.utils.files.is_empty(filename) + + +def fopen(*args, **kwargs): + ''' + Wrapper around open() built-in to set CLOEXEC on the fd. + + This flag specifies that the file descriptor should be closed when an exec + function is invoked; + When a file descriptor is allocated (as with open or dup), this bit is + initially cleared on the new file descriptor, meaning that descriptor will + survive into the new program after exec. + + NB! We still have small race condition between open and fcntl. + + .. deprecated:: Oxygen + ''' + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.files + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.fopen\' detected. This function has been moved to ' + '\'salt.utils.files.fopen\' as of Salt Oxygen. This warning will be ' + 'removed in Salt Neon.' + ) + return salt.utils.files.fopen(*args, **kwargs) # pylint: disable=W8470 + + +@contextlib.contextmanager +def flopen(*args, **kwargs): + ''' + Shortcut for fopen with lock and context manager + + .. deprecated:: Oxygen + ''' + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.files + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.flopen\' detected. This function has been moved to ' + '\'salt.utils.files.flopen\' as of Salt Oxygen. This warning will be ' + 'removed in Salt Neon.' + ) + return salt.utils.files.flopen(*args, **kwargs) + + +@contextlib.contextmanager +def fpopen(*args, **kwargs): + ''' + Shortcut for fopen with extra uid, gid and mode options. + + Supported optional Keyword Arguments: + + mode: explicit mode to set. Mode is anything os.chmod + would accept as input for mode. Works only on unix/unix + like systems. + + uid: the uid to set, if not set, or it is None or -1 no changes are + made. Same applies if the path is already owned by this + uid. Must be int. Works only on unix/unix like systems. + + gid: the gid to set, if not set, or it is None or -1 no changes are + made. Same applies if the path is already owned by this + gid. Must be int. Works only on unix/unix like systems. + + .. deprecated:: Oxygen + ''' + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.files + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.fpopen\' detected. This function has been moved to ' + '\'salt.utils.files.fpopen\' as of Salt Oxygen. This warning will be ' + 'removed in Salt Neon.' + ) + return salt.utils.files.fpopen(*args, **kwargs) + + +def rm_rf(path): + ''' + Platform-independent recursive delete. Includes code from + http://stackoverflow.com/a/2656405 + + .. deprecated:: Oxygen + ''' + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.files + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.rm_rf\' detected. This function has been moved to ' + '\'salt.utils.files.rm_rf\' as of Salt Oxygen. This warning will be ' + 'removed in Salt Neon.' + ) + return salt.utils.files.rm_rf(path) + + +def mkstemp(*args, **kwargs): + ''' + Helper function which does exactly what `tempfile.mkstemp()` does but + accepts another argument, `close_fd`, which, by default, is true and closes + the fd before returning the file path. Something commonly done throughout + Salt's code. + + .. deprecated:: Oxygen + ''' + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.files + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.mkstemp\' detected. This function has been moved to ' + '\'salt.utils.files.mkstemp\' as of Salt Oxygen. This warning will be ' + 'removed in Salt Neon.' + ) + return salt.utils.files.mkstemp(*args, **kwargs) + + +@jinja_filter('is_text_file') +def istextfile(fp_, blocksize=512): + ''' + Uses heuristics to guess whether the given file is text or binary, + by reading a single block of bytes from the file. + If more than 30% of the chars in the block are non-text, or there + are NUL ('\x00') bytes in the block, assume this is a binary file. + + .. deprecated:: Oxygen + ''' + # Late import to avoid circular import. + import salt.utils.files + + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.istextfile\' detected. This function has been moved ' + 'to \'salt.utils.files.is_text_file\' as of Salt Oxygen. This warning will ' + 'be removed in Salt Neon.' + ) + return salt.utils.files.is_text_file(fp_, blocksize=blocksize) + + +def str_version_to_evr(verstring): + ''' + Split the package version string into epoch, version and release. + Return this as tuple. + + The epoch is always not empty. The version and the release can be an empty + string if such a component could not be found in the version string. + + "2:1.0-1.2" => ('2', '1.0', '1.2) + "1.0" => ('0', '1.0', '') + "" => ('0', '', '') + + .. deprecated:: Oxygen + ''' + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.pkg.rpm + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.str_version_to_evr\' detected. This function has ' + 'been moved to \'salt.utils.pkg.rpm.version_to_evr\' as of Salt ' + 'Oxygen. This warning will be removed in Salt Neon.' + ) + return salt.utils.pkg.rpm.version_to_evr(verstring) + + +def parse_docstring(docstring): + ''' + Parse a docstring into its parts. + + Currently only parses dependencies, can be extended to parse whatever is + needed. + + Parses into a dictionary: + { + 'full': full docstring, + 'deps': list of dependencies (empty list if none) + } + + .. deprecated:: Oxygen + ''' + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.doc + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.parse_docstring\' detected. This function has ' + 'been moved to \'salt.utils.doc.parse_docstring\' as of Salt Oxygen. ' + 'This warning will be removed in Salt Neon.' + ) + return salt.utils.doc.parse_docstring(docstring) + + +def compare_versions(ver1='', oper='==', ver2='', cmp_func=None, ignore_epoch=False): + ''' + Compares two version numbers. Accepts a custom function to perform the + cmp-style version comparison, otherwise uses version_cmp(). + + .. deprecated:: Oxygen + ''' + # Late import to avoid circular import. + import salt.utils.versions + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.compare_versions\' detected. This function has ' + 'been moved to \'salt.utils.versions.compare\' as of Salt Oxygen. ' + 'This warning will be removed in Salt Neon.' + ) + return salt.utils.versions.compare(ver1=ver1, + oper=oper, + ver2=ver2, + cmp_func=cmp_func, + ignore_epoch=ignore_epoch) + + +def version_cmp(pkg1, pkg2, ignore_epoch=False): + ''' + Compares two version strings using salt.utils.versions.LooseVersion. This + is a fallback for providers which don't have a version comparison utility + built into them. Return -1 if version1 < version2, 0 if version1 == + version2, and 1 if version1 > version2. Return None if there was a problem + making the comparison. + + .. deprecated:: Oxygen + ''' + # Late import to avoid circular import. + import salt.utils.versions + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.version_cmp\' detected. This function has ' + 'been moved to \'salt.utils.versions.version_cmp\' as of Salt Oxygen. ' + 'This warning will be removed in Salt Neon.' + ) + return salt.utils.versions.version_cmp(pkg1, + pkg2, + ignore_epoch=ignore_epoch) + + +def warn_until(version, + message, + category=DeprecationWarning, + stacklevel=None, + _version_info_=None, + _dont_call_warnings=False): + ''' + Helper function to raise a warning, by default, a ``DeprecationWarning``, + until the provided ``version``, after which, a ``RuntimeError`` will + be raised to remind the developers to remove the warning because the + target version has been reached. + + .. deprecated:: Oxygen + ''' + # Late import to avoid circular import. + import salt.utils.versions + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.warn_until\' detected. This function has ' + 'been moved to \'salt.utils.versions.warn_until\' as of Salt ' + 'Oxygen. This warning will be removed in Salt Neon.' + ) + return salt.utils.versions.warn_until(version, + message, + category=category, + stacklevel=stacklevel, + _version_info_=_version_info_, + _dont_call_warnings=_dont_call_warnings) + + +def kwargs_warn_until(kwargs, + version, + category=DeprecationWarning, + stacklevel=None, + _version_info_=None, + _dont_call_warnings=False): + ''' + Helper function to raise a warning (by default, a ``DeprecationWarning``) + when unhandled keyword arguments are passed to function, until the + provided ``version_info``, after which, a ``RuntimeError`` will be raised + to remind the developers to remove the ``**kwargs`` because the target + version has been reached. + This function is used to help deprecate unused legacy ``**kwargs`` that + were added to function parameters lists to preserve backwards compatibility + when removing a parameter. See + :ref:`the deprecation development docs ` + for the modern strategy for deprecating a function parameter. + + .. deprecated:: Oxygen + ''' + # Late import to avoid circular import. + import salt.utils.versions + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.kwargs_warn_until\' detected. This function has ' + 'been moved to \'salt.utils.versions.kwargs_warn_until\' as of Salt ' + 'Oxygen. This warning will be removed in Salt Neon.' + ) + return salt.utils.versions.kwargs_warn_until( + kwargs, + version, + category=category, + stacklevel=stacklevel, + _version_info_=_version_info_, + _dont_call_warnings=_dont_call_warnings) + + +def get_color_theme(theme): + ''' + Return the color theme to use + + .. deprecated:: Oxygen + ''' + # Late import to avoid circular import. + import salt.utils.color + import salt.utils.versions + + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.get_color_theme\' detected. This function has ' + 'been moved to \'salt.utils.color.get_color_theme\' as of Salt ' + 'Oxygen. This warning will be removed in Salt Neon.' + ) + return salt.utils.color.get_color_theme(theme) + + +def get_colors(use=True, theme=None): + ''' + Return the colors as an easy to use dict. Pass `False` to deactivate all + colors by setting them to empty strings. Pass a string containing only the + name of a single color to be used in place of all colors. Examples: + + .. code-block:: python + + colors = get_colors() # enable all colors + no_colors = get_colors(False) # disable all colors + red_colors = get_colors('RED') # set all colors to red + + .. deprecated:: Oxygen + ''' + # Late import to avoid circular import. + import salt.utils.color + import salt.utils.versions + + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.get_colors\' detected. This function has ' + 'been moved to \'salt.utils.color.get_colors\' as of Salt ' + 'Oxygen. This warning will be removed in Salt Neon.' + ) + return salt.utils.color.get_colors(use=use, theme=theme) + + +def gen_state_tag(low): + ''' + Generate the running dict tag string from the low data structure + + .. deprecated:: Oxygen + ''' + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.state + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.gen_state_tag\' detected. This function has been ' + 'moved to \'salt.utils.state.gen_tag\' as of Salt Oxygen. This warning ' + 'will be removed in Salt Neon.' + ) + return salt.utils.state.gen_tag(low) + + +def search_onfail_requisites(sid, highstate): + ''' + For a particular low chunk, search relevant onfail related states + + .. deprecated:: Oxygen + ''' + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.state + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.search_onfail_requisites\' detected. This function ' + 'has been moved to \'salt.utils.state.search_onfail_requisites\' as of ' + 'Salt Oxygen. This warning will be removed in Salt Neon.' + ) + return salt.utils.state.search_onfail_requisites(sid, highstate) + + +def check_onfail_requisites(state_id, state_result, running, highstate): + ''' + When a state fail and is part of a highstate, check + if there is onfail requisites. + When we find onfail requisites, we will consider the state failed + only if at least one of those onfail requisites also failed + + Returns: + + True: if onfail handlers suceeded + False: if one on those handler failed + None: if the state does not have onfail requisites + + .. deprecated:: Oxygen + ''' + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.state + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.check_onfail_requisites\' detected. This function ' + 'has been moved to \'salt.utils.state.check_onfail_requisites\' as of ' + 'Salt Oxygen. This warning will be removed in Salt Neon.' + ) + return salt.utils.state.check_onfail_requisites( + state_id, state_result, running, highstate + ) + + +def check_state_result(running, recurse=False, highstate=None): + ''' + Check the total return value of the run and determine if the running + dict has any issues + + .. deprecated:: Oxygen + ''' + # Late import to avoid circular import. + import salt.utils.versions + import salt.utils.state + salt.utils.versions.warn_until( + 'Neon', + 'Use of \'salt.utils.check_state_result\' detected. This function ' + 'has been moved to \'salt.utils.state.check_result\' as of ' + 'Salt Oxygen. This warning will be removed in Salt Neon.' + ) + return salt.utils.state.check_result( + running, recurse=recurse, highstate=highstate + ) diff --git a/salt/utils/aggregation.py b/salt/utils/aggregation.py index bfd968bef8d..b65eebe4ead 100644 --- a/salt/utils/aggregation.py +++ b/salt/utils/aggregation.py @@ -112,7 +112,7 @@ from copy import copy from salt.utils.odict import OrderedDict # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six __all__ = ['aggregate', 'Aggregate', 'Map', 'Scalar', 'Sequence'] diff --git a/salt/utils/args.py b/salt/utils/args.py index c7a49b3a29c..7d47c528624 100644 --- a/salt/utils/args.py +++ b/salt/utils/args.py @@ -2,18 +2,20 @@ ''' Functions used for CLI argument handling ''' -from __future__ import absolute_import # Import python libs -import re +from __future__ import absolute_import +import fnmatch import inspect +import re +import shlex # Import salt libs +from salt.exceptions import SaltInvocationError +from salt.ext import six +from salt.ext.six.moves import zip # pylint: disable=import-error,redefined-builtin import salt.utils.jid -# Import 3rd-party libs -import salt.ext.six as six - if six.PY3: KWARG_REGEX = re.compile(r'^([^\d\W][\w.-]*)=(?!=)(.*)$', re.UNICODE) @@ -21,6 +23,42 @@ else: KWARG_REGEX = re.compile(r'^([^\d\W][\w.-]*)=(?!=)(.*)$') +def clean_kwargs(**kwargs): + ''' + Return a dict without any of the __pub* keys (or any other keys starting + with a dunder) from the kwargs dict passed into the execution module + functions. These keys are useful for tracking what was used to invoke + the function call, but they may not be desirable to have if passing the + kwargs forward wholesale. + ''' + ret = {} + for key, val in six.iteritems(kwargs): + if not key.startswith('__'): + ret[key] = val + return ret + + +def invalid_kwargs(invalid_kwargs, raise_exc=True): + ''' + Raise a SaltInvocationError if invalid_kwargs is non-empty + ''' + if invalid_kwargs: + if isinstance(invalid_kwargs, dict): + new_invalid = [ + '{0}={1}'.format(x, y) + for x, y in six.iteritems(invalid_kwargs) + ] + invalid_kwargs = new_invalid + msg = ( + 'The following keyword arguments are not valid: {0}' + .format(', '.join(invalid_kwargs)) + ) + if raise_exc: + raise SaltInvocationError(msg) + else: + return msg + + def condition_input(args, kwargs): ''' Return a single arg structure for the publisher to safely use @@ -220,3 +258,72 @@ def get_function_argspec(func, is_class_method=None): 'Cannot inspect argument list for \'{0}\''.format(func) ) return aspec + + +def shlex_split(s, **kwargs): + ''' + Only split if variable is a string + ''' + if isinstance(s, six.string_types): + return shlex.split(s, **kwargs) + else: + return s + + +def arg_lookup(fun, aspec=None): + ''' + Return a dict containing the arguments and default arguments to the + function. + ''' + ret = {'kwargs': {}} + if aspec is None: + aspec = get_function_argspec(fun) + if aspec.defaults: + ret['kwargs'] = dict(zip(aspec.args[::-1], aspec.defaults[::-1])) + ret['args'] = [arg for arg in aspec.args if arg not in ret['kwargs']] + return ret + + +def argspec_report(functions, module=''): + ''' + Pass in a functions dict as it is returned from the loader and return the + argspec function signatures + ''' + ret = {} + if '*' in module or '.' in module: + for fun in fnmatch.filter(functions, module): + try: + aspec = get_function_argspec(functions[fun]) + except TypeError: + # this happens if not callable + continue + + args, varargs, kwargs, defaults = aspec + + ret[fun] = {} + ret[fun]['args'] = args if args else None + ret[fun]['defaults'] = defaults if defaults else None + ret[fun]['varargs'] = True if varargs else None + ret[fun]['kwargs'] = True if kwargs else None + + else: + # "sys" should just match sys without also matching sysctl + module_dot = module + '.' + + for fun in functions: + if fun.startswith(module_dot): + try: + aspec = get_function_argspec(functions[fun]) + except TypeError: + # this happens if not callable + continue + + args, varargs, kwargs, defaults = aspec + + ret[fun] = {} + ret[fun]['args'] = args if args else None + ret[fun]['defaults'] = defaults if defaults else None + ret[fun]['varargs'] = True if varargs else None + ret[fun]['kwargs'] = True if kwargs else None + + return ret diff --git a/salt/utils/async.py b/salt/utils/async.py index a66ba97f9ff..6b24733253f 100644 --- a/salt/utils/async.py +++ b/salt/utils/async.py @@ -98,5 +98,7 @@ class SyncWrapper(object): # Other things should be deallocated after the io_loop closes. # See Issue #26889. del self.async - else: + del self.io_loop + elif hasattr(self, 'io_loop'): self.io_loop.close() + del self.io_loop diff --git a/salt/utils/atomicfile.py b/salt/utils/atomicfile.py index 2aab01c31a3..0d88773d46c 100644 --- a/salt/utils/atomicfile.py +++ b/salt/utils/atomicfile.py @@ -13,7 +13,7 @@ import errno import time import random import shutil -import salt.ext.six as six +from salt.ext import six CAN_RENAME_OPEN_FILE = False diff --git a/salt/utils/aws.py b/salt/utils/aws.py index 9e1212e92e6..011dd346d7e 100644 --- a/salt/utils/aws.py +++ b/salt/utils/aws.py @@ -392,7 +392,7 @@ def query(params=None, setname=None, requesturl=None, location=None, service_url = prov_dict.get('service_url', 'amazonaws.com') if not location: - location = get_location(opts, provider) + location = get_location(opts, prov_dict) if endpoint is None: if not requesturl: diff --git a/salt/utils/boto.py b/salt/utils/boto.py index 5f816715dc4..86b09a8e7d6 100644 --- a/salt/utils/boto.py +++ b/salt/utils/boto.py @@ -43,11 +43,12 @@ from functools import partial from salt.loader import minion_mods # Import salt libs -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import range # pylint: disable=import-error,redefined-builtin from salt.exceptions import SaltInvocationError from salt.utils.versions import LooseVersion as _LooseVersion import salt.utils +import salt.utils.stringutils # Import third party libs # pylint: disable=import-error @@ -109,7 +110,7 @@ def _get_profile(service, region, key, keyid, profile): if keyid: hash_string = region + keyid + key if six.PY3: - hash_string = salt.utils.to_bytes(hash_string) + hash_string = salt.utils.stringutils.to_bytes(hash_string) cxkey = label + hashlib.md5(hash_string).hexdigest() else: cxkey = label + region @@ -159,7 +160,7 @@ def cache_id_func(service): ''' Returns a partial `cache_id` function for the provided service. - ... code-block:: python + .. code-block:: python cache_id = __utils__['boto.cache_id_func']('ec2') cache_id('myinstance', 'i-a1b2c3') @@ -179,8 +180,9 @@ def get_connection(service, module=None, region=None, key=None, keyid=None, ''' module = module or service + module, submodule = ('boto.' + module).rsplit('.', 1) - svc_mod = __import__('boto.' + module, fromlist=[module]) + svc_mod = getattr(__import__(module, fromlist=[submodule]), submodule) cxkey, region, key, keyid = _get_profile(service, region, key, keyid, profile) @@ -207,7 +209,7 @@ def get_connection_func(service, module=None): ''' Returns a partial `get_connection` function for the provided service. - ... code-block:: python + .. code-block:: python get_conn = __utils__['boto.get_connection_func']('ec2') conn = get_conn() diff --git a/salt/utils/boto3.py b/salt/utils/boto3.py index 1861c4d372c..866ab7ed17a 100644 --- a/salt/utils/boto3.py +++ b/salt/utils/boto3.py @@ -47,6 +47,7 @@ from salt.exceptions import SaltInvocationError from salt.utils.versions import LooseVersion as _LooseVersion from salt.ext import six import salt.utils +import salt.utils.stringutils # Import third party libs # pylint: disable=import-error @@ -131,7 +132,7 @@ def _get_profile(service, region, key, keyid, profile): if keyid: hash_string = region + keyid + key if six.PY3: - hash_string = salt.utils.to_bytes(hash_string) + hash_string = salt.utils.stringutils.to_bytes(hash_string) cxkey = label + hashlib.md5(hash_string).hexdigest() else: cxkey = label + region @@ -181,7 +182,7 @@ def cache_id_func(service): ''' Returns a partial `cache_id` function for the provided service. - ... code-block:: python + .. code-block:: python cache_id = __utils__['boto.cache_id_func']('ec2') cache_id('myinstance', 'i-a1b2c3') @@ -232,7 +233,7 @@ def get_connection_func(service, module=None): ''' Returns a partial `get_connection` function for the provided service. - ... code-block:: python + .. code-block:: python get_conn = __utils__['boto.get_connection_func']('ec2') conn = get_conn() @@ -328,16 +329,6 @@ def paged_call(function, *args, **kwargs): kwargs[marker_arg] = marker -def get_role_arn(name, region=None, key=None, keyid=None, profile=None): - if name.startswith('arn:aws:iam:'): - return name - - account_id = __salt__['boto_iam.get_account_id']( - region=region, key=key, keyid=keyid, profile=profile - ) - return 'arn:aws:iam::{0}:role/{1}'.format(account_id, name) - - def ordered(obj): if isinstance(obj, (list, tuple)): return sorted(ordered(x) for x in obj) diff --git a/salt/utils/cache.py b/salt/utils/cache.py index 647fe02c769..fad2eeb10eb 100644 --- a/salt/utils/cache.py +++ b/salt/utils/cache.py @@ -15,6 +15,7 @@ except ImportError: import salt.config import salt.payload import salt.utils.dictupdate +import salt.utils.files # Import third party libs from salt.ext.six.moves import range # pylint: disable=import-error,redefined-builtin @@ -137,7 +138,7 @@ class CacheDisk(CacheDict): ''' if not HAS_MSGPACK or not os.path.exists(self._path): return - with salt.utils.fopen(self._path, 'rb') as fp_: + with salt.utils.files.fopen(self._path, 'rb') as fp_: cache = msgpack.load(fp_, encoding=__salt_system_encoding__) if "CacheDisk_cachetime" in cache: # new format self._dict = cache["CacheDisk_data"] @@ -158,7 +159,7 @@ class CacheDisk(CacheDict): return # TODO Add check into preflight to ensure dir exists # TODO Dir hashing? - with salt.utils.fopen(self._path, 'wb+') as fp_: + with salt.utils.files.fopen(self._path, 'wb+') as fp_: cache = { "CacheDisk_data": self._dict, "CacheDisk_cachetime": self._key_cache_time @@ -283,14 +284,14 @@ class ContextCache(object): ''' if not os.path.isdir(os.path.dirname(self.cache_path)): os.mkdir(os.path.dirname(self.cache_path)) - with salt.utils.fopen(self.cache_path, 'w+b') as cache: + with salt.utils.files.fopen(self.cache_path, 'w+b') as cache: self.serial.dump(context, cache) def get_cache_context(self): ''' Retrieve a context cache from disk ''' - with salt.utils.fopen(self.cache_path, 'rb') as cache: + with salt.utils.files.fopen(self.cache_path, 'rb') as cache: return self.serial.load(cache) diff --git a/salt/utils/cloud.py b/salt/utils/cloud.py index cd6d59e9b5c..b013c48f307 100644 --- a/salt/utils/cloud.py +++ b/salt/utils/cloud.py @@ -7,7 +7,6 @@ Utility functions for salt.cloud from __future__ import absolute_import import errno import os -import sys import stat import codecs import shutil @@ -26,16 +25,6 @@ import re import uuid -# Let's import pwd and catch the ImportError. We'll raise it if this is not -# Windows -try: - import pwd -except ImportError: - if not sys.platform.lower().startswith('win'): - # We can't use salt.utils.is_windows() from the import a little down - # because that will cause issues under windows at install time. - raise - try: import salt.utils.smb @@ -57,9 +46,12 @@ import salt.client import salt.config import salt.loader import salt.template -import salt.utils +import salt.utils # Can be removed when pem_finger is moved +import salt.utils.compat import salt.utils.event -from salt.utils import vt +import salt.utils.files +import salt.utils.platform +import salt.utils.vt from salt.utils.nb_popen import NonBlockingPopen from salt.utils.yamldumper import SafeOrderedDumper from salt.utils.validate.path import is_writeable @@ -75,12 +67,20 @@ from salt.exceptions import ( SaltCloudPasswordError ) -# Import third party libs -import salt.ext.six as six +# Import 3rd-party libs +from salt.ext import six from salt.ext.six.moves import range # pylint: disable=import-error,redefined-builtin,W0611 from jinja2 import Template import yaml +# Let's import pwd and catch the ImportError. We'll raise it if this is not +# Windows. This import has to be below where we import salt.utils.platform! +try: + import pwd +except ImportError: + if not salt.utils.platform.is_windows(): + raise + try: import getpass @@ -109,12 +109,12 @@ def __render_script(path, vm_=None, opts=None, minion=''): ''' log.info('Rendering deploy script: {0}'.format(path)) try: - with salt.utils.fopen(path, 'r') as fp_: + with salt.utils.files.fopen(path, 'r') as fp_: template = Template(fp_.read()) return str(template.render(opts=opts, vm=vm_, minion=minion)) except AttributeError: # Specified renderer was not found - with salt.utils.fopen(path, 'r') as fp_: + with salt.utils.files.fopen(path, 'r') as fp_: return fp_.read() @@ -161,9 +161,9 @@ def gen_keys(keysize=2048): salt.crypt.gen_keys(tdir, 'minion', keysize) priv_path = os.path.join(tdir, 'minion.pem') pub_path = os.path.join(tdir, 'minion.pub') - with salt.utils.fopen(priv_path) as fp_: + with salt.utils.files.fopen(priv_path) as fp_: priv = fp_.read() - with salt.utils.fopen(pub_path) as fp_: + with salt.utils.files.fopen(pub_path) as fp_: pub = fp_.read() shutil.rmtree(tdir) return priv, pub @@ -181,12 +181,12 @@ def accept_key(pki_dir, pub, id_): os.makedirs(key_path) key = os.path.join(pki_dir, 'minions', id_) - with salt.utils.fopen(key, 'w+') as fp_: + with salt.utils.files.fopen(key, 'w+') as fp_: fp_.write(pub) oldkey = os.path.join(pki_dir, 'minions_pre', id_) if os.path.isfile(oldkey): - with salt.utils.fopen(oldkey) as fp_: + with salt.utils.files.fopen(oldkey) as fp_: if fp_.read() == pub: os.remove(oldkey) @@ -292,12 +292,14 @@ def salt_config_to_yaml(configuration, line_break='\n'): Dumper=SafeOrderedDumper) -def bootstrap(vm_, opts): +def bootstrap(vm_, opts=None): ''' This is the primary entry point for logging into any system (POSIX or Windows) to install Salt. It will make the decision on its own as to which deploy function to call. ''' + if opts is None: + opts = __opts__ deploy_config = salt.config.get_cloud_config_value( 'deploy', vm_, opts, default=False) @@ -311,6 +313,11 @@ def bootstrap(vm_, opts): } } + if vm_.get('driver') == 'saltify': + saltify_driver = True + else: + saltify_driver = False + key_filename = salt.config.get_cloud_config_value( 'key_filename', vm_, opts, search_global=False, default=salt.config.get_cloud_config_value( @@ -346,7 +353,7 @@ def bootstrap(vm_, opts): ret = {} - minion_conf = salt.utils.cloud.minion_config(opts, vm_) + minion_conf = minion_config(opts, vm_) deploy_script_code = os_script( salt.config.get_cloud_config_value( 'os', vm_, opts, default='bootstrap-salt' @@ -405,10 +412,7 @@ def bootstrap(vm_, opts): 'tmp_dir': salt.config.get_cloud_config_value( 'tmp_dir', vm_, opts, default='/tmp/.saltcloud' ), - 'deploy_command': salt.config.get_cloud_config_value( - 'deploy_command', vm_, opts, - default='/tmp/.saltcloud/deploy.sh', - ), + 'vm_': vm_, 'start_action': opts['start_action'], 'parallel': opts['parallel'], 'sock_dir': opts['sock_dir'], @@ -438,6 +442,9 @@ def bootstrap(vm_, opts): 'script_env', vm_, opts ), 'minion_conf': minion_conf, + 'force_minion_config': salt.config.get_cloud_config_value( + 'force_minion_config', vm_, opts, default=False + ), 'preseed_minion_keys': vm_.get('preseed_minion_keys', None), 'display_ssh_output': salt.config.get_cloud_config_value( 'display_ssh_output', vm_, opts, default=True @@ -451,9 +458,16 @@ def bootstrap(vm_, opts): 'maxtries': salt.config.get_cloud_config_value( 'wait_for_passwd_maxtries', vm_, opts, default=15 ), + 'preflight_cmds': salt.config.get_cloud_config_value( + 'preflight_cmds', vm_, opts, default=[] + ), + 'cloud_grains': {'driver': vm_['driver'], + 'provider': vm_['provider'], + 'profile': vm_['profile'] + } } - inline_script_kwargs = deploy_kwargs + inline_script_kwargs = deploy_kwargs.copy() # make a copy at this point # forward any info about possible ssh gateway to deploy script # as some providers need also a 'gateway' configuration @@ -465,7 +479,7 @@ def bootstrap(vm_, opts): deploy_kwargs['make_master'] = True deploy_kwargs['master_pub'] = vm_['master_pub'] deploy_kwargs['master_pem'] = vm_['master_pem'] - master_conf = salt.utils.cloud.master_config(opts, vm_) + master_conf = master_config(opts, vm_) deploy_kwargs['master_conf'] = master_conf if master_conf.get('syndic_master', None): @@ -475,6 +489,9 @@ def bootstrap(vm_, opts): 'make_minion', vm_, opts, default=True ) + if saltify_driver: + deploy_kwargs['wait_for_passwd_maxtries'] = 0 # No need to wait/retry with Saltify + win_installer = salt.config.get_cloud_config_value( 'win_installer', vm_, opts ) @@ -483,7 +500,7 @@ def bootstrap(vm_, opts): 'smb_port', vm_, opts, default=445 ) deploy_kwargs['win_installer'] = win_installer - minion = salt.utils.cloud.minion_config(opts, vm_) + minion = minion_config(opts, vm_) deploy_kwargs['master'] = minion['master'] deploy_kwargs['username'] = salt.config.get_cloud_config_value( 'win_username', vm_, opts, default='Administrator' @@ -500,8 +517,13 @@ def bootstrap(vm_, opts): 'winrm_port', vm_, opts, default=5986 ) deploy_kwargs['winrm_use_ssl'] = salt.config.get_cloud_config_value( - 'winrm_use_ssl', vm_, opts, default=True + 'winrm_use_ssl', vm_, opts, default=True ) + deploy_kwargs['winrm_verify_ssl'] = salt.config.get_cloud_config_value( + 'winrm_verify_ssl', vm_, opts, default=True + ) + if saltify_driver: + deploy_kwargs['port_timeout'] = 1 # No need to wait/retry with Saltify # Store what was used to the deploy the VM event_kwargs = copy.deepcopy(deploy_kwargs) @@ -809,24 +831,24 @@ def wait_for_winexesvc(host, port, username, password, timeout=900): log.debug('winexe connected...') return True log.debug('Return code was {0}'.format(ret_code)) - time.sleep(1) except socket.error as exc: log.debug('Caught exception in wait_for_winexesvc: {0}'.format(exc)) - time.sleep(1) - if time.time() - start > timeout: - log.error('winexe connection timed out: {0}'.format(timeout)) - return False - log.debug( - 'Retrying winexe connection to host {0} on port {1} ' - '(try {2})'.format( - host, - port, - try_count - ) + + if time.time() - start > timeout: + log.error('winexe connection timed out: {0}'.format(timeout)) + return False + log.debug( + 'Retrying winexe connection to host {0} on port {1} ' + '(try {2})'.format( + host, + port, + try_count ) + ) + time.sleep(1) -def wait_for_winrm(host, port, username, password, timeout=900, use_ssl=True): +def wait_for_winrm(host, port, username, password, timeout=900, use_ssl=True, verify=True): ''' Wait until WinRM connection can be established. ''' @@ -836,14 +858,20 @@ def wait_for_winrm(host, port, username, password, timeout=900, use_ssl=True): host, port ) ) + transport = 'ssl' + if not use_ssl: + transport = 'plaintext' trycount = 0 while True: trycount += 1 try: - transport = 'ssl' - if not use_ssl: - transport = 'plaintext' - s = winrm.Session(host, auth=(username, password), transport=transport) + winrm_kwargs = {'target': host, + 'auth': (username, password), + 'transport': transport} + if not verify: + log.debug("SSL validation for WinRM disabled.") + winrm_kwargs['server_cert_validation'] = 'ignore' + s = winrm.Session(**winrm_kwargs) if hasattr(s.protocol, 'set_timeout'): s.protocol.set_timeout(15) log.trace('WinRM endpoint url: {0}'.format(s.url)) @@ -852,19 +880,19 @@ def wait_for_winrm(host, port, username, password, timeout=900, use_ssl=True): log.debug('WinRM session connected...') return s log.debug('Return code was {0}'.format(r.status_code)) - time.sleep(1) except WinRMTransportError as exc: log.debug('Caught exception in wait_for_winrm: {0}'.format(exc)) - if time.time() - start > timeout: - log.error('WinRM connection timed out: {0}'.format(timeout)) - return None - log.debug( - 'Retrying WinRM connection to host {0} on port {1} ' - '(try {2})'.format( - host, port, trycount - ) + + if time.time() - start > timeout: + log.error('WinRM connection timed out: {0}'.format(timeout)) + return None + log.debug( + 'Retrying WinRM connection to host {0} on port {1} ' + '(try {2})'.format( + host, port, trycount ) - time.sleep(1) + ) + time.sleep(1) def validate_windows_cred(host, @@ -885,7 +913,7 @@ def validate_windows_cred(host, host ) - for i in xrange(retries): + for i in range(retries): ret_code = win_cmd( cmd, logging_command=logging_cmd @@ -991,6 +1019,7 @@ def deploy_windows(host, use_winrm=False, winrm_port=5986, winrm_use_ssl=True, + winrm_verify_ssl=True, **kwargs): ''' Copy the install files to a remote Windows box, and execute them @@ -1017,7 +1046,8 @@ def deploy_windows(host, if HAS_WINRM and use_winrm: winrm_session = wait_for_winrm(host=host, port=winrm_port, username=username, password=password, - timeout=port_timeout * 60, use_ssl=winrm_use_ssl) + timeout=port_timeout * 60, use_ssl=winrm_use_ssl, + verify=winrm_verify_ssl) if winrm_session is not None: service_available = True else: @@ -1065,7 +1095,7 @@ def deploy_windows(host, # Read master-sign.pub file log.debug("Copying master_sign.pub file from {0} to minion".format(master_sign_pub_file)) try: - with salt.utils.fopen(master_sign_pub_file, 'rb') as master_sign_fh: + with salt.utils.files.fopen(master_sign_pub_file, 'rb') as master_sign_fh: smb_conn.putFile('C$', 'salt\\conf\\pki\\minion\\master_sign.pub', master_sign_fh.read) except Exception as e: log.debug("Exception copying master_sign.pub file {0} to minion".format(master_sign_pub_file)) @@ -1077,7 +1107,7 @@ def deploy_windows(host, comps = win_installer.split('/') local_path = '/'.join(comps[:-1]) installer = comps[-1] - with salt.utils.fopen(win_installer, 'rb') as inst_fh: + with salt.utils.files.fopen(win_installer, 'rb') as inst_fh: smb_conn.putFile('C$', 'salttemp/{0}'.format(installer), inst_fh.read) if use_winrm: @@ -1211,20 +1241,26 @@ def deploy_script(host, sudo_password=None, sudo=False, tty=None, - deploy_command='/tmp/.saltcloud/deploy.sh', + vm_=None, opts=None, tmp_dir='/tmp/.saltcloud', file_map=None, master_sign_pub_file=None, + cloud_grains=None, + force_minion_config=False, **kwargs): ''' Copy a deploy script to a remote server, execute it, and remove it ''' if not isinstance(opts, dict): opts = {} + vm_ = vm_ or {} # if None, default to empty dict + cloud_grains = cloud_grains or {} tmp_dir = '{0}-{1}'.format(tmp_dir.rstrip('/'), uuid.uuid4()) - deploy_command = os.path.join(tmp_dir, 'deploy.sh') + deploy_command = salt.config.get_cloud_config_value( + 'deploy_command', vm_, opts, + default=os.path.join(tmp_dir, 'deploy.sh')) if key_filename is not None and not os.path.isfile(key_filename): raise SaltCloudConfigError( 'The defined key_filename \'{0}\' does not exist'.format( @@ -1236,9 +1272,11 @@ def deploy_script(host, if 'gateway' in kwargs: gateway = kwargs['gateway'] - starttime = time.mktime(time.localtime()) - log.debug('Deploying {0} at {1}'.format(host, starttime)) - + starttime = time.localtime() + log.debug('Deploying {0} at {1}'.format( + host, + time.strftime('%Y-%m-%d %H:%M:%S', starttime)) + ) known_hosts_file = kwargs.get('known_hosts_file', '/dev/null') hard_timeout = opts.get('hard_timeout', None) @@ -1370,6 +1408,8 @@ def deploy_script(host, salt_config_to_yaml(minion_grains), ssh_kwargs ) + if cloud_grains and opts.get('enable_cloud_grains', True): + minion_conf['grains'] = {'salt-cloud': cloud_grains} ssh_file( opts, '{0}/minion'.format(tmp_dir), @@ -1453,6 +1493,15 @@ def deploy_script(host, 'Can\'t set ownership for {0}'.format( preseed_minion_keys_tempdir)) + # Run any pre-flight commands before running deploy scripts + preflight_cmds = kwargs.get('preflight_cmds', []) + for command in preflight_cmds: + cmd_ret = root_cmd(command, tty, sudo, **ssh_kwargs) + if cmd_ret: + raise SaltCloudSystemExit( + 'Pre-flight command failed: \'{0}\''.format(command) + ) + # The actual deploy script if script: # got strange escaping issues with sudoer, going onto a @@ -1466,7 +1515,8 @@ def deploy_script(host, raise SaltCloudSystemExit( 'Can\'t set perms on {0}/deploy.sh'.format(tmp_dir)) - newtimeout = timeout - (time.mktime(time.localtime()) - starttime) + time_used = time.mktime(time.localtime()) - time.mktime(starttime) + newtimeout = timeout - time_used queue = None process = None # Consider this code experimental. It causes Salt Cloud to wait @@ -1487,6 +1537,8 @@ def deploy_script(host, if script: if 'bootstrap-salt' in script: deploy_command += ' -c \'{0}\''.format(tmp_dir) + if force_minion_config: + deploy_command += ' -F' if make_syndic is True: deploy_command += ' -S' if make_master is True: @@ -1771,18 +1823,10 @@ def filter_event(tag, data, defaults): return ret -def fire_event(key, msg, tag, args=None, sock_dir=None, transport='zeromq'): +def fire_event(key, msg, tag, sock_dir, args=None, transport='zeromq'): ''' Fire deploy action ''' - if sock_dir is None: - salt.utils.warn_until( - 'Oxygen', - '`salt.utils.cloud.fire_event` requires that the `sock_dir`' - 'parameter be passed in when calling the function.' - ) - sock_dir = __opts__['sock_dir'] - event = salt.utils.event.get_event( 'master', sock_dir, @@ -1810,7 +1854,7 @@ def _exec_ssh_cmd(cmd, error_msg=None, allow_failure=False, **kwargs): password_retries = kwargs.get('password_retries', 3) try: stdout, stderr = None, None - proc = vt.Terminal( + proc = salt.utils.vt.Terminal( cmd, shell=True, log_stdout=True, @@ -1850,7 +1894,7 @@ def _exec_ssh_cmd(cmd, error_msg=None, allow_failure=False, **kwargs): ) ) return proc.exitstatus - except vt.TerminalException as err: + except salt.utils.vt.TerminalException as err: trace = traceback.format_exc() log.error(error_msg.format(cmd, err, trace)) finally: @@ -2001,7 +2045,7 @@ def sftp_file(dest_path, contents=None, kwargs=None, local_file=None): if contents is not None: try: tmpfd, file_to_upload = tempfile.mkstemp() - if isinstance(contents, str): + if isinstance(contents, six.string_types): os.write(tmpfd, contents.encode(__salt_system_encoding__)) else: os.write(tmpfd, contents) @@ -2503,7 +2547,7 @@ def lock_file(filename, interval=.5, timeout=15): else: break - with salt.utils.fopen(lock, 'a'): + with salt.utils.files.fopen(lock, 'a'): pass @@ -2543,7 +2587,7 @@ def cachedir_index_add(minion_id, profile, driver, provider, base=None): if os.path.exists(index_file): mode = 'rb' if six.PY3 else 'r' - with salt.utils.fopen(index_file, mode) as fh_: + with salt.utils.files.fopen(index_file, mode) as fh_: index = msgpack.load(fh_) else: index = {} @@ -2560,7 +2604,7 @@ def cachedir_index_add(minion_id, profile, driver, provider, base=None): }) mode = 'wb' if six.PY3 else 'w' - with salt.utils.fopen(index_file, mode) as fh_: + with salt.utils.files.fopen(index_file, mode) as fh_: msgpack.dump(index, fh_) unlock_file(index_file) @@ -2577,7 +2621,7 @@ def cachedir_index_del(minion_id, base=None): if os.path.exists(index_file): mode = 'rb' if six.PY3 else 'r' - with salt.utils.fopen(index_file, mode) as fh_: + with salt.utils.files.fopen(index_file, mode) as fh_: index = msgpack.load(fh_) else: return @@ -2586,7 +2630,7 @@ def cachedir_index_del(minion_id, base=None): del index[minion_id] mode = 'wb' if six.PY3 else 'w' - with salt.utils.fopen(index_file, mode) as fh_: + with salt.utils.files.fopen(index_file, mode) as fh_: msgpack.dump(index, fh_) unlock_file(index_file) @@ -2643,7 +2687,7 @@ def request_minion_cachedir( fname = '{0}.p'.format(minion_id) path = os.path.join(base, 'requested', fname) - with salt.utils.fopen(path, 'w') as fh_: + with salt.utils.files.fopen(path, 'w') as fh_: msgpack.dump(data, fh_) @@ -2675,12 +2719,12 @@ def change_minion_cachedir( fname = '{0}.p'.format(minion_id) path = os.path.join(base, cachedir, fname) - with salt.utils.fopen(path, 'r') as fh_: + with salt.utils.files.fopen(path, 'r') as fh_: cache_data = msgpack.load(fh_) cache_data.update(data) - with salt.utils.fopen(path, 'w') as fh_: + with salt.utils.files.fopen(path, 'w') as fh_: msgpack.dump(cache_data, fh_) @@ -2753,7 +2797,7 @@ def list_cache_nodes_full(opts=None, provider=None, base=None): # Finally, get a list of full minion data fpath = os.path.join(min_dir, fname) minion_id = fname[:-2] # strip '.p' from end of msgpack filename - with salt.utils.fopen(fpath, 'r') as fh_: + with salt.utils.files.fopen(fpath, 'r') as fh_: minions[driver][prov][minion_id] = msgpack.load(fh_) return minions @@ -2764,6 +2808,11 @@ def cache_nodes_ip(opts, base=None): Retrieve a list of all nodes from Salt Cloud cache, and any associated IP addresses. Returns a dict. ''' + salt.utils.warn_until( + 'Flourine', + 'This function is incomplete and non-functional ' + 'and will be removed in Salt Flourine.' + ) if base is None: base = opts['cachedir'] @@ -2810,7 +2859,7 @@ def update_bootstrap(config, url=None): else: script_name = os.path.basename(url) elif os.path.exists(url): - with salt.utils.fopen(url) as fic: + with salt.utils.files.fopen(url) as fic: script_content = fic.read() script_name = os.path.basename(url) # in last case, assuming we got a script content @@ -2899,7 +2948,7 @@ def update_bootstrap(config, url=None): deploy_path = os.path.join(entry, script_name) try: finished_full.append(deploy_path) - with salt.utils.fopen(deploy_path, 'w') as fp_: + with salt.utils.files.fopen(deploy_path, 'w') as fp_: fp_.write(script_content) except (OSError, IOError) as err: log.debug( @@ -2932,7 +2981,7 @@ def cache_node_list(nodes, provider, opts): for node in nodes: diff_node_cache(prov_dir, node, nodes[node], opts) path = os.path.join(prov_dir, '{0}.p'.format(node)) - with salt.utils.fopen(path, 'w') as fh_: + with salt.utils.files.fopen(path, 'w') as fh_: msgpack.dump(nodes[node], fh_) @@ -2957,7 +3006,7 @@ def cache_node(node, provider, opts): if not os.path.exists(prov_dir): os.makedirs(prov_dir) path = os.path.join(prov_dir, '{0}.p'.format(node['name'])) - with salt.utils.fopen(path, 'w') as fh_: + with salt.utils.files.fopen(path, 'w') as fh_: msgpack.dump(node, fh_) @@ -3031,7 +3080,7 @@ def diff_node_cache(prov_dir, node, new_data, opts): ) return - with salt.utils.fopen(path, 'r') as fh_: + with salt.utils.files.fopen(path, 'r') as fh_: try: cache_data = msgpack.load(fh_) except ValueError: @@ -3041,7 +3090,7 @@ def diff_node_cache(prov_dir, node, new_data, opts): # Perform a simple diff between the old and the new data, and if it differs, # return both dicts. # TODO: Return an actual diff - diff = cmp(new_data, cache_data) + diff = salt.utils.compat.cmp(new_data, cache_data) if diff != 0: fire_event( 'event', diff --git a/salt/utils/color.py b/salt/utils/color.py new file mode 100644 index 00000000000..21efb315dc2 --- /dev/null +++ b/salt/utils/color.py @@ -0,0 +1,92 @@ +# -*- coding: utf-8 -*- +''' +Functions used for CLI color themes. +''' + +# Import Python libs +from __future__ import absolute_import +import logging +import os + +# Import Salt libs +from salt.ext import six +from salt.textformat import TextFormat +import salt.utils.files + +log = logging.getLogger(__name__) + + +def get_color_theme(theme): + ''' + Return the color theme to use + ''' + # Keep the heavy lifting out of the module space + import yaml + if not os.path.isfile(theme): + log.warning('The named theme {0} if not available'.format(theme)) + + try: + with salt.utils.files.fopen(theme, 'rb') as fp_: + colors = yaml.safe_load(fp_.read()) + ret = {} + for color in colors: + ret[color] = '\033[{0}m'.format(colors[color]) + if not isinstance(colors, dict): + log.warning('The theme file {0} is not a dict'.format(theme)) + return {} + return ret + except Exception: + log.warning('Failed to read the color theme {0}'.format(theme)) + return {} + + +def get_colors(use=True, theme=None): + ''' + Return the colors as an easy to use dict. Pass `False` to deactivate all + colors by setting them to empty strings. Pass a string containing only the + name of a single color to be used in place of all colors. Examples: + + .. code-block:: python + + colors = get_colors() # enable all colors + no_colors = get_colors(False) # disable all colors + red_colors = get_colors('RED') # set all colors to red + + ''' + + colors = { + 'BLACK': TextFormat('black'), + 'DARK_GRAY': TextFormat('bold', 'black'), + 'RED': TextFormat('red'), + 'LIGHT_RED': TextFormat('bold', 'red'), + 'GREEN': TextFormat('green'), + 'LIGHT_GREEN': TextFormat('bold', 'green'), + 'YELLOW': TextFormat('yellow'), + 'LIGHT_YELLOW': TextFormat('bold', 'yellow'), + 'BLUE': TextFormat('blue'), + 'LIGHT_BLUE': TextFormat('bold', 'blue'), + 'MAGENTA': TextFormat('magenta'), + 'LIGHT_MAGENTA': TextFormat('bold', 'magenta'), + 'CYAN': TextFormat('cyan'), + 'LIGHT_CYAN': TextFormat('bold', 'cyan'), + 'LIGHT_GRAY': TextFormat('white'), + 'WHITE': TextFormat('bold', 'white'), + 'DEFAULT_COLOR': TextFormat('default'), + 'ENDC': TextFormat('reset'), + } + if theme: + colors.update(get_color_theme(theme)) + + if not use: + for color in colors: + colors[color] = '' + if isinstance(use, six.string_types): + # Try to set all of the colors to the passed color + if use in colors: + for color in colors: + # except for color reset + if color == 'ENDC': + continue + colors[color] = colors[use] + + return colors diff --git a/salt/utils/compat.py b/salt/utils/compat.py index b97820db928..09c5cc28283 100644 --- a/salt/utils/compat.py +++ b/salt/utils/compat.py @@ -46,3 +46,15 @@ def deepcopy_bound(name): finally: copy._deepcopy_dispatch = pre_dispatch return ret + + +def cmp(x, y): + ''' + Compatibility helper function to replace the ``cmp`` function from Python 2. The + ``cmp`` function is no longer available in Python 3. + + cmp(x, y) -> integer + + Return negative if xy. + ''' + return (x > y) - (x < y) diff --git a/salt/utils/configcomparer.py b/salt/utils/configcomparer.py index ffe1723bd56..b7a0f279ff6 100644 --- a/salt/utils/configcomparer.py +++ b/salt/utils/configcomparer.py @@ -8,7 +8,7 @@ changes in a way that can be easily reported in a state. from __future__ import absolute_import # Import Salt libs -import salt.ext.six as six +from salt.ext import six def compare_and_update_config(config, update_config, changes, namespace=''): diff --git a/salt/utils/configparser.py b/salt/utils/configparser.py new file mode 100644 index 00000000000..d060f9ddd24 --- /dev/null +++ b/salt/utils/configparser.py @@ -0,0 +1,269 @@ +# -*- coding: utf-8 -*- +# Import Python libs +from __future__ import absolute_import +import re + +# Import Salt libs +import salt.utils.stringutils + +# Import 3rd-party libs +from salt.ext import six +from salt.ext.six.moves.configparser import * # pylint: disable=no-name-in-module,wildcard-import + +try: + from collections import OrderedDict as _default_dict +except ImportError: + # fallback for setup.py which hasn't yet built _collections + _default_dict = dict + + +# pylint: disable=string-substitution-usage-error +class GitConfigParser(RawConfigParser, object): # pylint: disable=undefined-variable + ''' + Custom ConfigParser which reads and writes git config files. + + READ A GIT CONFIG FILE INTO THE PARSER OBJECT + + >>> import salt.utils.configparser + >>> conf = salt.utils.configparser.GitConfigParser() + >>> conf.read('/home/user/.git/config') + + MAKE SOME CHANGES + + >>> # Change user.email + >>> conf.set('user', 'email', 'myaddress@mydomain.tld') + >>> # Add another refspec to the "origin" remote's "fetch" multivar + >>> conf.set_multivar('remote "origin"', 'fetch', '+refs/tags/*:refs/tags/*') + + WRITE THE CONFIG TO A FILEHANDLE + + >>> import salt.utils.files + >>> with salt.utils.files.fopen('/home/user/.git/config', 'w') as fh: + ... conf.write(fh) + >>> + ''' + DEFAULTSECT = u'DEFAULT' + SPACEINDENT = u' ' * 8 + + def __init__(self, defaults=None, dict_type=_default_dict, + allow_no_value=True): + ''' + Changes default value for allow_no_value from False to True + ''' + super(GitConfigParser, self).__init__( + defaults, dict_type, allow_no_value) + + def _read(self, fp, fpname): + ''' + Makes the following changes from the RawConfigParser: + + 1. Strip leading tabs from non-section-header lines. + 2. Treat 8 spaces at the beginning of a line as a tab. + 3. Treat lines beginning with a tab as options. + 4. Drops support for continuation lines. + 5. Multiple values for a given option are stored as a list. + 6. Keys and values are decoded to the system encoding. + ''' + cursect = None # None, or a dictionary + optname = None + lineno = 0 + e = None # None, or an exception + while True: + line = fp.readline() + if six.PY2: + line = line.decode(__salt_system_encoding__) + if not line: + break + lineno = lineno + 1 + # comment or blank line? + if line.strip() == u'' or line[0] in u'#;': + continue + if line.split(None, 1)[0].lower() == u'rem' and line[0] in u'rR': + # no leading whitespace + continue + # Replace space indentation with a tab. Allows parser to work + # properly in cases where someone has edited the git config by hand + # and indented using spaces instead of tabs. + if line.startswith(self.SPACEINDENT): + line = u'\t' + line[len(self.SPACEINDENT):] + # is it a section header? + mo = self.SECTCRE.match(line) + if mo: + sectname = mo.group(u'header') + if sectname in self._sections: + cursect = self._sections[sectname] + elif sectname == self.DEFAULTSECT: + cursect = self._defaults + else: + cursect = self._dict() + self._sections[sectname] = cursect + # So sections can't start with a continuation line + optname = None + # no section header in the file? + elif cursect is None: + raise MissingSectionHeaderError( # pylint: disable=undefined-variable + salt.utils.stringutils.to_str(fpname), + lineno, + salt.utils.stringutils.to_str(line)) + # an option line? + else: + mo = self._optcre.match(line.lstrip()) + if mo: + optname, vi, optval = mo.group(u'option', u'vi', u'value') + optname = self.optionxform(optname.rstrip()) + if optval is None: + optval = u'' + if optval: + if vi in (u'=', u':') and u';' in optval: + # ';' is a comment delimiter only if it follows + # a spacing character + pos = optval.find(u';') + if pos != -1 and optval[pos-1].isspace(): + optval = optval[:pos] + optval = optval.strip() + # Empty strings should be considered as blank strings + if optval in (u'""', u"''"): + optval = u'' + self._add_option(cursect, optname, optval) + else: + # a non-fatal parsing error occurred. set up the + # exception but keep going. the exception will be + # raised at the end of the file and will contain a + # list of all bogus lines + if not e: + e = ParsingError(fpname) # pylint: disable=undefined-variable + e.append(lineno, repr(line)) + # if any parsing errors occurred, raise an exception + if e: + raise e # pylint: disable=raising-bad-type + + def _string_check(self, value, allow_list=False): + ''' + Based on the string-checking code from the SafeConfigParser's set() + function, this enforces string values for config options. + ''' + if self._optcre is self.OPTCRE or value: + is_list = isinstance(value, list) + if is_list and not allow_list: + raise TypeError('option value cannot be a list unless allow_list is True') # future lint: disable=non-unicode-string + elif not is_list: + value = [value] + if not all(isinstance(x, six.string_types) for x in value): + raise TypeError('option values must be strings') # future lint: disable=non-unicode-string + + def get(self, section, option, as_list=False): + ''' + Adds an optional "as_list" argument to ensure a list is returned. This + is helpful when iterating over an option which may or may not be a + multivar. + ''' + ret = super(GitConfigParser, self).get(section, option) + if as_list and not isinstance(ret, list): + ret = [ret] + return ret + + def set(self, section, option, value=u''): + ''' + This is overridden from the RawConfigParser merely to change the + default value for the 'value' argument. + ''' + self._string_check(value) + super(GitConfigParser, self).set(section, option, value) + + def _add_option(self, sectdict, key, value): + if isinstance(value, list): + sectdict[key] = value + elif isinstance(value, six.string_types): + try: + sectdict[key].append(value) + except KeyError: + # Key not present, set it + sectdict[key] = value + except AttributeError: + # Key is present but the value is not a list. Make it into a list + # and then append to it. + sectdict[key] = [sectdict[key]] + sectdict[key].append(value) + else: + raise TypeError('Expected str or list for option value, got %s' % type(value).__name__) # future lint: disable=non-unicode-string + + def set_multivar(self, section, option, value=u''): + ''' + This function is unique to the GitConfigParser. It will add another + value for the option if it already exists, converting the option's + value to a list if applicable. + + If "value" is a list, then any existing values for the specified + section and option will be replaced with the list being passed. + ''' + self._string_check(value, allow_list=True) + if not section or section == self.DEFAULTSECT: + sectdict = self._defaults + else: + try: + sectdict = self._sections[section] + except KeyError: + raise NoSectionError( # pylint: disable=undefined-variable + salt.utils.stringutils.to_str(section)) + key = self.optionxform(option) + self._add_option(sectdict, key, value) + + def remove_option_regexp(self, section, option, expr): + ''' + Remove an option with a value matching the expression. Works on single + values and multivars. + ''' + if not section or section == self.DEFAULTSECT: + sectdict = self._defaults + else: + try: + sectdict = self._sections[section] + except KeyError: + raise NoSectionError( # pylint: disable=undefined-variable + salt.utils.stringutils.to_str(section)) + option = self.optionxform(option) + if option not in sectdict: + return False + regexp = re.compile(expr) + if isinstance(sectdict[option], list): + new_list = [x for x in sectdict[option] if not regexp.search(x)] + # Revert back to a list if we removed all but one item + if len(new_list) == 1: + new_list = new_list[0] + existed = new_list != sectdict[option] + if existed: + del sectdict[option] + sectdict[option] = new_list + del new_list + else: + existed = bool(regexp.search(sectdict[option])) + if existed: + del sectdict[option] + return existed + + def write(self, fp_): + ''' + Makes the following changes from the RawConfigParser: + + 1. Prepends options with a tab character. + 2. Does not write a blank line between sections. + 3. When an option's value is a list, a line for each option is written. + This allows us to support multivars like a remote's "fetch" option. + 4. Drops support for continuation lines. + ''' + convert = salt.utils.stringutils.to_bytes \ + if u'b' in fp_.mode \ + else salt.utils.stringutils.to_str + if self._defaults: + fp_.write(convert(u'[%s]\n' % self.DEFAULTSECT)) + for (key, value) in six.iteritems(self._defaults): + value = salt.utils.stringutils.to_unicode(value).replace(u'\n', u'\n\t') + fp_.write(convert(u'%s = %s\n' % (key, value))) + for section in self._sections: + fp_.write(convert(u'[%s]\n' % section)) + for (key, value) in six.iteritems(self._sections[section]): + if (value is not None) or (self._optcre == self.OPTCRE): + if not isinstance(value, list): + value = [value] + for item in value: + fp_.write(convert(u'\t%s\n' % u' = '.join((key, item)).rstrip())) diff --git a/salt/utils/context.py b/salt/utils/context.py index 62934cd07af..1073e2df4e9 100644 --- a/salt/utils/context.py +++ b/salt/utils/context.py @@ -17,7 +17,7 @@ import threading import collections from contextlib import contextmanager -import salt.ext.six as six +from salt.ext import six @contextmanager @@ -125,6 +125,22 @@ class ContextDict(collections.MutableMapping): else: return iter(self.global_data) + def __copy__(self): + new_obj = type(self)(threadsafe=self._threadsafe) + if self.active: + new_obj.global_data = copy.copy(self._state.data) + else: + new_obj.global_data = copy.copy(self.global_data) + return new_obj + + def __deepcopy__(self, memo): + new_obj = type(self)(threadsafe=self._threadsafe) + if self.active: + new_obj.global_data = copy.deepcopy(self._state.data, memo) + else: + new_obj.global_data = copy.deepcopy(self.global_data, memo) + return new_obj + class ChildContextDict(collections.MutableMapping): '''An overrideable child of ContextDict @@ -189,7 +205,8 @@ class NamespacedDictWrapper(collections.MutableMapping, dict): self.pre_keys = pre_keys if override_name: self.__class__.__module__ = 'salt' - self.__class__.__name__ = override_name + # __name__ can't be assigned a unicode + self.__class__.__name__ = str(override_name) # future lint: disable=non-unicode-string super(NamespacedDictWrapper, self).__init__(self._dict()) def _dict(self): diff --git a/salt/utils/debug.py b/salt/utils/debug.py index 57324628e0d..938aaa38fd8 100644 --- a/salt/utils/debug.py +++ b/salt/utils/debug.py @@ -14,7 +14,7 @@ import traceback import inspect # Import salt libs -import salt.utils +import salt.utils.files def _makepretty(printout, stack): @@ -39,7 +39,7 @@ def _handle_sigusr1(sig, stack): else: filename = 'salt-debug-{0}.log'.format(int(time.time())) destfile = os.path.join(tempfile.gettempdir(), filename) - with salt.utils.fopen(destfile, 'w') as output: + with salt.utils.files.fopen(destfile, 'w') as output: _makepretty(output, stack) diff --git a/salt/utils/decorators/__init__.py b/salt/utils/decorators/__init__.py index f1638d3daeb..06276b00201 100644 --- a/salt/utils/decorators/__init__.py +++ b/salt/utils/decorators/__init__.py @@ -12,14 +12,12 @@ from functools import wraps from collections import defaultdict # Import salt libs -import salt.utils import salt.utils.args -from salt.exceptions import CommandNotFoundError, CommandExecutionError, SaltConfigurationError -from salt.version import SaltStackVersion, __saltstack_version__ +from salt.exceptions import CommandExecutionError, SaltConfigurationError from salt.log import LOG_LEVELS # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six log = logging.getLogger(__name__) @@ -154,7 +152,7 @@ def timing(function): @wraps(function) def wrapped(*args, **kwargs): start_time = time.time() - ret = function(*args, **salt.utils.clean_kwargs(**kwargs)) + ret = function(*args, **salt.utils.args.clean_kwargs(**kwargs)) end_time = time.time() if function.__module__.startswith('salt.loaded.int.'): mod_name = function.__module__[16:] @@ -171,67 +169,6 @@ def timing(function): return wrapped -def which(exe): - ''' - Decorator wrapper for salt.utils.which - ''' - def wrapper(function): - def wrapped(*args, **kwargs): - if salt.utils.which(exe) is None: - raise CommandNotFoundError( - 'The \'{0}\' binary was not found in $PATH.'.format(exe) - ) - return function(*args, **kwargs) - return identical_signature_wrapper(function, wrapped) - return wrapper - - -def which_bin(exes): - ''' - Decorator wrapper for salt.utils.which_bin - ''' - def wrapper(function): - def wrapped(*args, **kwargs): - if salt.utils.which_bin(exes) is None: - raise CommandNotFoundError( - 'None of provided binaries({0}) was not found ' - 'in $PATH.'.format( - ['\'{0}\''.format(exe) for exe in exes] - ) - ) - return function(*args, **kwargs) - return identical_signature_wrapper(function, wrapped) - return wrapper - - -def identical_signature_wrapper(original_function, wrapped_function): - ''' - Return a function with identical signature as ``original_function``'s which - will call the ``wrapped_function``. - ''' - context = {'__wrapped__': wrapped_function} - function_def = compile( - 'def {0}({1}):\n' - ' return __wrapped__({2})'.format( - # Keep the original function name - original_function.__name__, - # The function signature including defaults, i.e., 'timeout=1' - inspect.formatargspec( - *salt.utils.args.get_function_argspec(original_function) - )[1:-1], - # The function signature without the defaults - inspect.formatargspec( - formatvalue=lambda val: '', - *salt.utils.args.get_function_argspec(original_function) - )[1:-1] - ), - '', - 'exec' - ) - six.exec_(function_def, context) - return wraps(original_function)(context[original_function.__name__]) - - def memoize(func): ''' Memoize aka cache the return output of a function @@ -245,7 +182,14 @@ def memoize(func): @wraps(func) def _memoize(*args, **kwargs): - args_ = ','.join(list(args) + ['{0}={1}'.format(k, kwargs[k]) for k in sorted(kwargs)]) + str_args = [] + for arg in args: + if not isinstance(arg, six.string_types): + str_args.append(str(arg)) + else: + str_args.append(arg) + + args_ = ','.join(list(str_args) + ['{0}={1}'.format(k, kwargs[k]) for k in sorted(kwargs)]) if args_ not in cache: cache[args_] = func(*args, **kwargs) return cache[args_] @@ -270,7 +214,7 @@ class _DeprecationDecorator(object): :param version: Expiration version :return: ''' - + from salt.version import SaltStackVersion, __saltstack_version__ self._globals = globals self._exp_version_name = version self._exp_version = SaltStackVersion.from_name(self._exp_version_name) @@ -534,8 +478,14 @@ class _WithDeprecated(_DeprecationDecorator): f_name=function.__name__)) opts = self._globals.get('__opts__', '{}') - use_deprecated = full_name in opts.get(self.CFG_USE_DEPRECATED, list()) - use_superseded = full_name in opts.get(self.CFG_USE_SUPERSEDED, list()) + pillar = self._globals.get('__pillar__', '{}') + + use_deprecated = (full_name in opts.get(self.CFG_USE_DEPRECATED, list()) or + full_name in pillar.get(self.CFG_USE_DEPRECATED, list())) + + use_superseded = (full_name in opts.get(self.CFG_USE_SUPERSEDED, list()) or + full_name in pillar.get(self.CFG_USE_SUPERSEDED, list())) + if use_deprecated and use_superseded: raise SaltConfigurationError("Function '{0}' is mentioned both in deprecated " "and superseded sections. Please remove any of that.".format(full_name)) @@ -557,8 +507,11 @@ class _WithDeprecated(_DeprecationDecorator): f_name=self._orig_f_name) return func_path in self._globals.get('__opts__').get( + self.CFG_USE_DEPRECATED, list()) or func_path in self._globals.get('__pillar__').get( self.CFG_USE_DEPRECATED, list()) or (self._policy == self.OPT_IN and not (func_path in self._globals.get('__opts__', {}).get( + self.CFG_USE_SUPERSEDED, list())) + and not (func_path in self._globals.get('__pillar__', {}).get( self.CFG_USE_SUPERSEDED, list()))), func_path def __call__(self, function): @@ -633,53 +586,3 @@ def ignores_kwargs(*kwarg_names): return fn(*args, **kwargs_filtered) return __ignores_kwargs return _ignores_kwargs - - -class JinjaFilter(object): - ''' - This decorator is used to specify that a function is to be loaded as a - Jinja filter. - ''' - salt_jinja_filters = {} - - def __init__(self, name=None): - ''' - ''' - self.name = name - - def __call__(self, function): - ''' - ''' - name = self.name or function.__name__ - if name not in self.salt_jinja_filters: - log.debug('Marking "{0}" as a jinja filter'.format(name)) - self.salt_jinja_filters[name] = function - return function - - -jinja_filter = JinjaFilter - - -class JinjaTest(object): - ''' - This decorator is used to specify that a function is to be loaded as a - Jinja test. - ''' - salt_jinja_tests = {} - - def __init__(self, name=None): - ''' - ''' - self.name = name - - def __call__(self, function): - ''' - ''' - name = self.name or function.__name__ - if name not in self.salt_jinja_tests: - log.debug('Marking "{0}" as a jinja test'.format(name)) - self.salt_jinja_tests[name] = function - return function - - -jinja_test = JinjaTest diff --git a/salt/utils/decorators/jinja.py b/salt/utils/decorators/jinja.py new file mode 100644 index 00000000000..512eb181a0d --- /dev/null +++ b/salt/utils/decorators/jinja.py @@ -0,0 +1,85 @@ +# -*- coding: utf-8 -*- +''' +Jinja-specific decorators +''' +from __future__ import absolute_import + +# Import Python libs +import logging + +log = logging.getLogger(__name__) + + +class JinjaFilter(object): + ''' + This decorator is used to specify that a function is to be loaded as a + Jinja filter. + ''' + salt_jinja_filters = {} + + def __init__(self, name=None): + ''' + ''' + self.name = name + + def __call__(self, function): + ''' + ''' + name = self.name or function.__name__ + if name not in self.salt_jinja_filters: + log.debug(u'Marking \'%s\' as a jinja filter', name) + self.salt_jinja_filters[name] = function + return function + + +jinja_filter = JinjaFilter + + +class JinjaTest(object): + ''' + This decorator is used to specify that a function is to be loaded as a + Jinja test. + ''' + salt_jinja_tests = {} + + def __init__(self, name=None): + ''' + ''' + self.name = name + + def __call__(self, function): + ''' + ''' + name = self.name or function.__name__ + if name not in self.salt_jinja_tests: + log.debug('Marking \'%s\' as a jinja test', name) + self.salt_jinja_tests[name] = function + return function + + +jinja_test = JinjaTest + + +class JinjaGlobal(object): + ''' + This decorator is used to specify that a function is to be loaded as a + Jinja global. + ''' + salt_jinja_globals = {} + + def __init__(self, name=None): + ''' + ''' + self.name = name + + def __call__(self, function): + ''' + ''' + name = self.name or function.__name__ + if name not in self.salt_jinja_globals: + log.debug('Marking "{0}" as a jinja global'.format(name)) + self.salt_jinja_globals[name] = function + return function + + +jinja_global = JinjaGlobal diff --git a/salt/utils/decorators/path.py b/salt/utils/decorators/path.py new file mode 100644 index 00000000000..816d75785b5 --- /dev/null +++ b/salt/utils/decorators/path.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +''' +Decorators for salt.utils.path +''' +from __future__ import absolute_import + +# Import Salt libs +import salt.utils.path +from salt.exceptions import CommandNotFoundError +from salt.utils.decorators.signature import identical_signature_wrapper + + +def which(exe): + ''' + Decorator wrapper for salt.utils.path.which + ''' + def wrapper(function): + def wrapped(*args, **kwargs): + if salt.utils.path.which(exe) is None: + raise CommandNotFoundError( + 'The \'{0}\' binary was not found in $PATH.'.format(exe) + ) + return function(*args, **kwargs) + return identical_signature_wrapper(function, wrapped) + return wrapper + + +def which_bin(exes): + ''' + Decorator wrapper for salt.utils.path.which_bin + ''' + def wrapper(function): + def wrapped(*args, **kwargs): + if salt.utils.path.which_bin(exes) is None: + raise CommandNotFoundError( + 'None of provided binaries({0}) was not found ' + 'in $PATH.'.format( + ['\'{0}\''.format(exe) for exe in exes] + ) + ) + return function(*args, **kwargs) + return identical_signature_wrapper(function, wrapped) + return wrapper diff --git a/salt/utils/decorators/signature.py b/salt/utils/decorators/signature.py new file mode 100644 index 00000000000..a9e7662618a --- /dev/null +++ b/salt/utils/decorators/signature.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- + +# Import Python libs +from __future__ import absolute_import +import inspect +from functools import wraps + +# Import Salt libs +import salt.utils.args + +# Import 3rd-party libs +from salt.ext import six + + +def identical_signature_wrapper(original_function, wrapped_function): + ''' + Return a function with identical signature as ``original_function``'s which + will call the ``wrapped_function``. + ''' + context = {'__wrapped__': wrapped_function} + function_def = compile( + 'def {0}({1}):\n' + ' return __wrapped__({2})'.format( + # Keep the original function name + original_function.__name__, + # The function signature including defaults, i.e., 'timeout=1' + inspect.formatargspec( + *salt.utils.args.get_function_argspec(original_function) + )[1:-1], + # The function signature without the defaults + inspect.formatargspec( + formatvalue=lambda val: '', + *salt.utils.args.get_function_argspec(original_function) + )[1:-1] + ), + '', + 'exec' + ) + six.exec_(function_def, context) + return wraps(original_function)(context[original_function.__name__]) diff --git a/salt/utils/dictupdate.py b/salt/utils/dictupdate.py index 1d744dcc859..6687a411ab2 100644 --- a/salt/utils/dictupdate.py +++ b/salt/utils/dictupdate.py @@ -11,7 +11,7 @@ import collections # Import 3rd-party libs import copy import logging -import salt.ext.six as six +from salt.ext import six from salt.serializers.yamlex import merge_recursive as _yamlex_merge_recursive log = logging.getLogger(__name__) diff --git a/salt/utils/dns.py b/salt/utils/dns.py index bca3f0b3454..e741b0e20e6 100644 --- a/salt/utils/dns.py +++ b/salt/utils/dns.py @@ -11,26 +11,30 @@ dns.srv_name('ldap/tcp', 'example.com') ''' from __future__ import absolute_import -# Python + +# Import Python libs import base64 import binascii import hashlib import itertools +import logging import random -import socket import shlex +import socket import ssl import string -from salt.ext.six.moves import map, zip # pylint: disable=redefined-builtin + +# Import Salt libs +import salt.utils.files +import salt.utils.network +import salt.utils.path +import salt.modules.cmdmod from salt._compat import ipaddress from salt.utils.odict import OrderedDict -# Salt -import salt.modules.cmdmod -import salt.utils +# Import 3rd-party libs +from salt.ext.six.moves import map, zip # pylint: disable=redefined-builtin -# Debug & Logging -import logging # Integrations try: @@ -38,10 +42,10 @@ try: HAS_DNSPYTHON = True except ImportError: HAS_DNSPYTHON = False -HAS_DIG = salt.utils.which('dig') is not None -HAS_DRILL = salt.utils.which('drill') is not None -HAS_HOST = salt.utils.which('host') is not None -HAS_NSLOOKUP = salt.utils.which('nslookup') is not None +HAS_DIG = salt.utils.path.which('dig') is not None +HAS_DRILL = salt.utils.path.which('drill') is not None +HAS_HOST = salt.utils.path.which('host') is not None +HAS_NSLOOKUP = salt.utils.path.which('nslookup') is not None __salt__ = { 'cmd.run_all': salt.modules.cmdmod.run_all @@ -948,7 +952,7 @@ def services(services_file='/etc/services'): } ''' res = {} - with salt.utils.fopen(services_file, 'r') as svc_defs: + with salt.utils.files.fopen(services_file, 'r') as svc_defs: for svc_def in svc_defs.readlines(): svc_def = svc_def.strip() if not len(svc_def) or svc_def.startswith('#'): @@ -1011,7 +1015,7 @@ def parse_resolv(src='/etc/resolv.conf'): options = [] try: - with salt.utils.fopen(src) as src_file: + with salt.utils.files.fopen(src) as src_file: # pylint: disable=too-many-nested-blocks for line in src_file: line = line.strip().split() diff --git a/salt/utils/doc.py b/salt/utils/doc.py index 0d82eaf8829..f28b2f3c7f4 100644 --- a/salt/utils/doc.py +++ b/salt/utils/doc.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import import re -import salt.ext.six as six +from salt.ext import six def strip_rst(docs): @@ -24,3 +24,40 @@ def strip_rst(docs): if docstring != docstring_new: docs[func] = docstring_new return docs + + +def parse_docstring(docstring): + ''' + Parse a docstring into its parts. + + Currently only parses dependencies, can be extended to parse whatever is + needed. + + Parses into a dictionary: + { + 'full': full docstring, + 'deps': list of dependencies (empty list if none) + } + ''' + # First try with regex search for :depends: + ret = {'full': docstring} + regex = r'([ \t]*):depends:[ \t]+- (\w+)[^\n]*\n(\1[ \t]+- (\w+)[^\n]*\n)*' + match = re.search(regex, docstring, re.M) + if match: + deps = [] + regex = r'- (\w+)' + for line in match.group(0).strip().splitlines(): + deps.append(re.search(regex, line).group(1)) + ret['deps'] = deps + return ret + # Try searching for a one-liner instead + else: + txt = 'Required python modules: ' + data = docstring.splitlines() + dep_list = list(x for x in data if x.strip().startswith(txt)) + if not dep_list: + ret['deps'] = [] + return ret + deps = dep_list[0].replace(txt, '').strip().split(', ') + ret['deps'] = deps + return ret diff --git a/salt/utils/docker/__init__.py b/salt/utils/docker/__init__.py index e186d639d7c..530066f6ede 100644 --- a/salt/utils/docker/__init__.py +++ b/salt/utils/docker/__init__.py @@ -13,12 +13,13 @@ import os # Import Salt libs import salt.utils +import salt.utils.args import salt.utils.docker.translate from salt.exceptions import CommandExecutionError, SaltInvocationError from salt.utils.args import get_function_argspec as _argspec # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six try: import docker @@ -174,7 +175,7 @@ def translate_input(**kwargs): have their translation skipped. Optionally, skip_translate can be set to True to skip *all* translation. ''' - kwargs = salt.utils.clean_kwargs(**kwargs) + kwargs = salt.utils.args.clean_kwargs(**kwargs) invalid = {} collisions = [] diff --git a/salt/utils/docker/translate.py b/salt/utils/docker/translate.py index b6d87173168..9436fa3599c 100644 --- a/salt/utils/docker/translate.py +++ b/salt/utils/docker/translate.py @@ -14,7 +14,7 @@ import salt.utils.network from salt.exceptions import SaltInvocationError # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import range, zip # pylint: disable=import-error,redefined-builtin log = logging.getLogger(__name__) @@ -387,7 +387,7 @@ def dns(val, **kwargs): def domainname(val, **kwargs): # pylint: disable=unused-argument - return _translate_stringlist(val) + return _translate_str(val) def entrypoint(val, **kwargs): # pylint: disable=unused-argument diff --git a/salt/utils/etcd_util.py b/salt/utils/etcd_util.py index 3067c34a5e1..a0431a178e4 100644 --- a/salt/utils/etcd_util.py +++ b/salt/utils/etcd_util.py @@ -56,7 +56,7 @@ from __future__ import absolute_import import logging # Import salt libs -import salt.ext.six as six +from salt.ext import six from salt.exceptions import CommandExecutionError # Import third party libs @@ -79,7 +79,8 @@ class EtcdUtilWatchTimeout(Exception): class EtcdClient(object): - def __init__(self, opts, profile=None): + def __init__(self, opts, profile=None, + host=None, port=None, username=None, password=None, ca=None, client_key=None, client_cert=None, **kwargs): opts_pillar = opts.get('pillar', {}) opts_master = opts_pillar.get('master', {}) @@ -93,13 +94,13 @@ class EtcdClient(object): else: self.conf = opts_merged - host = self.conf.get('etcd.host', '127.0.0.1') - port = self.conf.get('etcd.port', 4001) - username = self.conf.get('etcd.username') - password = self.conf.get('etcd.password') - ca_cert = self.conf.get('etcd.ca') - cli_key = self.conf.get('etcd.client_key') - cli_cert = self.conf.get('etcd.client_cert') + host = host or self.conf.get('etcd.host', '127.0.0.1') + port = port or self.conf.get('etcd.port', 4001) + username = username or self.conf.get('etcd.username') + password = password or self.conf.get('etcd.password') + ca_cert = ca or self.conf.get('etcd.ca') + cli_key = client_key or self.conf.get('etcd.client_key') + cli_cert = client_cert or self.conf.get('etcd.client_cert') auth = {} if username and password: @@ -164,6 +165,9 @@ class EtcdClient(object): except ValueError: return {} + if result is None: + return {} + if recurse: ret['key'] = getattr(result, 'key', None) ret['value'] = getattr(result, 'value', None) @@ -348,8 +352,8 @@ class EtcdClient(object): return ret -def get_conn(opts, profile=None): - client = EtcdClient(opts, profile) +def get_conn(opts, profile=None, **kwargs): + client = EtcdClient(opts, profile, **kwargs) return client diff --git a/salt/utils/event.py b/salt/utils/event.py index 0b64ee18338..c7e8b457a38 100644 --- a/salt/utils/event.py +++ b/salt/utils/event.py @@ -59,12 +59,13 @@ import fnmatch import hashlib import logging import datetime +import sys from collections import MutableMapping from multiprocessing.util import Finalize from salt.ext.six.moves import range # Import third party libs -import salt.ext.six as six +from salt.ext import six import tornado.ioloop import tornado.iostream @@ -75,7 +76,9 @@ import salt.utils import salt.utils.async import salt.utils.cache import salt.utils.dicttrim +import salt.utils.platform import salt.utils.process +import salt.utils.stringutils import salt.utils.zeromq import salt.log.setup import salt.defaults.exitcodes @@ -248,7 +251,7 @@ class SaltEvent(object): else: self.opts['sock_dir'] = sock_dir - if salt.utils.is_windows() and 'ipc_mode' not in opts: + if salt.utils.platform.is_windows() and 'ipc_mode' not in opts: self.opts['ipc_mode'] = 'tcp' self.puburi, self.pulluri = self.__load_uri(sock_dir, node) self.pending_tags = [] @@ -299,7 +302,7 @@ class SaltEvent(object): hash_type = getattr(hashlib, self.opts['hash_type']) # Only use the first 10 chars to keep longer hashes from exceeding the # max socket path length. - id_hash = hash_type(salt.utils.to_bytes(self.opts['id'])).hexdigest()[:10] + id_hash = hash_type(salt.utils.stringutils.to_bytes(self.opts['id'])).hexdigest()[:10] puburi = os.path.join( sock_dir, 'minion_event_{0}_pub.ipc'.format(id_hash) @@ -432,8 +435,8 @@ class SaltEvent(object): mtag, sep, mdata = raw.partition(TAGEND) # split tag from data data = serial.loads(mdata) else: - mtag, sep, mdata = raw.partition(salt.utils.to_bytes(TAGEND)) # split tag from data - mtag = salt.utils.to_str(mtag) + mtag, sep, mdata = raw.partition(salt.utils.stringutils.to_bytes(TAGEND)) # split tag from data + mtag = salt.utils.stringutils.to_str(mtag) data = serial.loads(mdata, encoding='utf-8') return mtag, data @@ -729,10 +732,10 @@ class SaltEvent(object): event = '{0}{1}{2}'.format(tag, tagend, serialized_data) else: event = b''.join([ - salt.utils.to_bytes(tag), - salt.utils.to_bytes(tagend), + salt.utils.stringutils.to_bytes(tag), + salt.utils.stringutils.to_bytes(tagend), serialized_data]) - msg = salt.utils.to_bytes(event, 'utf-8') + msg = salt.utils.stringutils.to_bytes(event, 'utf-8') if self._run_io_loop_sync: with salt.utils.async.current_ioloop(self.io_loop): try: @@ -949,7 +952,7 @@ class AsyncEventPublisher(object): hash_type = getattr(hashlib, self.opts['hash_type']) # Only use the first 10 chars to keep longer hashes from exceeding the # max socket path length. - id_hash = hash_type(salt.utils.to_bytes(self.opts['id'])).hexdigest()[:10] + id_hash = hash_type(salt.utils.stringutils.to_bytes(self.opts['id'])).hexdigest()[:10] epub_sock_path = os.path.join( self.opts['sock_dir'], 'minion_event_{0}_pub.ipc'.format(id_hash) @@ -1159,6 +1162,16 @@ class EventReturn(salt.utils.process.SignalHandlingMultiprocessingProcess): A dedicated process which listens to the master event bus and queues and forwards events to the specified returner. ''' + def __new__(cls, *args, **kwargs): + if sys.platform.startswith('win'): + # This is required for Windows. On Linux, when a process is + # forked, the module namespace is copied and the current process + # gets all of sys.modules from where the fork happens. This is not + # the case for Windows. + import salt.minion + instance = super(EventReturn, cls).__new__(cls, *args, **kwargs) + return instance + def __init__(self, opts, log_queue=None): ''' Initialize the EventReturn system diff --git a/salt/utils/extend.py b/salt/utils/extend.py index 9a5885e50f1..39913bdbb57 100644 --- a/salt/utils/extend.py +++ b/salt/utils/extend.py @@ -29,7 +29,7 @@ from jinja2 import Template from salt.serializers.yaml import deserialize from salt.ext.six.moves import zip from salt.utils.odict import OrderedDict -import salt.utils +import salt.utils.files import salt.version log = logging.getLogger(__name__) @@ -56,7 +56,7 @@ def _get_template(path, option_key): :returns: Details about the template :rtype: ``tuple`` ''' - with salt.utils.fopen(path, "r") as template_f: + with salt.utils.files.fopen(path, 'r') as template_f: template = deserialize(template_f) info = (option_key, template.get('description', ''), template) return info @@ -138,10 +138,10 @@ def _mergetreejinja(src, dst, context): if item != TEMPLATE_FILE_NAME: d = Template(d).render(context) log.info("Copying file {0} to {1}".format(s, d)) - with salt.utils.fopen(s, 'r') as source_file: + with salt.utils.files.fopen(s, 'r') as source_file: src_contents = source_file.read() dest_contents = Template(src_contents).render(context) - with salt.utils.fopen(d, 'w') as dest_file: + with salt.utils.files.fopen(d, 'w') as dest_file: dest_file.write(dest_contents) diff --git a/salt/utils/filebuffer.py b/salt/utils/filebuffer.py index deab2c11135..eefc1a3e036 100644 --- a/salt/utils/filebuffer.py +++ b/salt/utils/filebuffer.py @@ -11,8 +11,8 @@ from __future__ import absolute_import # Import salt libs -import salt.ext.six as six -import salt.utils +from salt.ext import six +import salt.utils.files from salt.exceptions import SaltException @@ -57,7 +57,7 @@ class BufferedReader(object): if 'a' in mode or 'w' in mode: raise InvalidFileMode("Cannot open file in write or append mode") self.__path = path - self.__file = salt.utils.fopen(self.__path, mode) # pylint: disable=resource-leakage + self.__file = salt.utils.files.fopen(self.__path, mode) # pylint: disable=resource-leakage self.__max_in_mem = max_in_mem self.__chunk_size = chunk_size self.__buffered = None diff --git a/salt/utils/files.py b/salt/utils/files.py index 235f6b44366..1d7068987a2 100644 --- a/salt/utils/files.py +++ b/salt/utils/files.py @@ -7,24 +7,37 @@ import contextlib import errno import logging import os +import re import shutil +import stat import subprocess import tempfile import time +import urllib -# Import salt libs -import salt.utils +# Import Salt libs +import salt.utils # Can be removed when backup_minion is moved +import salt.utils.path +import salt.utils.platform import salt.modules.selinux from salt.exceptions import CommandExecutionError, FileLockError, MinionError +from salt.utils.decorators.jinja import jinja_filter # Import 3rd-party libs from salt.ext import six +from salt.ext.six.moves import range +try: + import fcntl + HAS_FCNTL = True +except ImportError: + # fcntl is not available on windows + HAS_FCNTL = False log = logging.getLogger(__name__) -TEMPFILE_PREFIX = '__salt.tmp.' REMOTE_PROTOS = ('http', 'https', 'ftp', 'swift', 's3') VALID_PROTOS = ('salt', 'file') + REMOTE_PROTOS +TEMPFILE_PREFIX = '__salt.tmp.' def guess_archive_type(name): @@ -44,20 +57,20 @@ def guess_archive_type(name): def mkstemp(*args, **kwargs): ''' - Helper function which does exactly what `tempfile.mkstemp()` does but - accepts another argument, `close_fd`, which, by default, is true and closes + Helper function which does exactly what ``tempfile.mkstemp()`` does but + accepts another argument, ``close_fd``, which, by default, is true and closes the fd before returning the file path. Something commonly done throughout Salt's code. ''' if 'prefix' not in kwargs: - kwargs['prefix'] = TEMPFILE_PREFIX + kwargs['prefix'] = '__salt.tmp.' close_fd = kwargs.pop('close_fd', True) - fd_, fpath = tempfile.mkstemp(*args, **kwargs) + fd_, f_path = tempfile.mkstemp(*args, **kwargs) if close_fd is False: - return (fd_, fpath) + return fd_, f_path os.close(fd_) del fd_ - return fpath + return f_path def recursive_copy(source, dest): @@ -68,7 +81,7 @@ def recursive_copy(source, dest): (identical to cp -r on a unix machine) ''' for root, _, files in os.walk(source): - path_from_source = root.replace(source, '').lstrip('/') + path_from_source = root.replace(source, '').lstrip(os.sep) target_directory = os.path.join(dest, path_from_source) if not os.path.exists(target_directory): os.makedirs(target_directory) @@ -107,7 +120,7 @@ def copyfile(source, dest, backup_mode='', cachedir=''): # Get current file stats to they can be replicated after the new file is # moved to the destination path. fstat = None - if not salt.utils.is_windows(): + if not salt.utils.platform.is_windows(): try: fstat = os.stat(dest) except OSError: @@ -117,7 +130,7 @@ def copyfile(source, dest, backup_mode='', cachedir=''): os.chown(dest, fstat.st_uid, fstat.st_gid) os.chmod(dest, fstat.st_mode) # If SELINUX is available run a restorecon on the file - rcon = salt.utils.which('restorecon') + rcon = salt.utils.path.which('restorecon') if rcon: policy = False try: @@ -125,7 +138,7 @@ def copyfile(source, dest, backup_mode='', cachedir=''): except (ImportError, CommandExecutionError): pass if policy == 'Enforcing': - with salt.utils.fopen(os.devnull, 'w') as dev_null: + with fopen(os.devnull, 'w') as dev_null: cmd = [rcon, dest] subprocess.call(cmd, stdout=dev_null, stderr=dev_null) if os.path.isfile(tgt): @@ -253,3 +266,275 @@ def wait_lock(path, lock_fn=None, timeout=5, sleep=0.1, time_start=None): if obtained_lock: os.remove(lock_fn) log.trace('Write lock for %s (%s) released', path, lock_fn) + + +@contextlib.contextmanager +def set_umask(mask): + ''' + Temporarily set the umask and restore once the contextmanager exits + ''' + if salt.utils.platform.is_windows(): + # Don't attempt on Windows + yield + else: + try: + orig_mask = os.umask(mask) + yield + finally: + os.umask(orig_mask) + + +def fopen(*args, **kwargs): + ''' + Wrapper around open() built-in to set CLOEXEC on the fd. + + This flag specifies that the file descriptor should be closed when an exec + function is invoked; + + When a file descriptor is allocated (as with open or dup), this bit is + initially cleared on the new file descriptor, meaning that descriptor will + survive into the new program after exec. + + NB! We still have small race condition between open and fcntl. + ''' + binary = None + # ensure 'binary' mode is always used on Windows in Python 2 + if ((six.PY2 and salt.utils.platform.is_windows() and 'binary' not in kwargs) or + kwargs.pop('binary', False)): + if len(args) > 1: + args = list(args) + if 'b' not in args[1]: + args[1] += 'b' + elif kwargs.get('mode', None): + if 'b' not in kwargs['mode']: + kwargs['mode'] += 'b' + else: + # the default is to read + kwargs['mode'] = 'rb' + elif six.PY3 and 'encoding' not in kwargs: + # In Python 3, if text mode is used and the encoding + # is not specified, set the encoding to 'utf-8'. + binary = False + if len(args) > 1: + args = list(args) + if 'b' in args[1]: + binary = True + if kwargs.get('mode', None): + if 'b' in kwargs['mode']: + binary = True + if not binary: + kwargs['encoding'] = __salt_system_encoding__ + + if six.PY3 and not binary and not kwargs.get('newline', None): + kwargs['newline'] = '' + + f_handle = open(*args, **kwargs) # pylint: disable=resource-leakage + + if is_fcntl_available(): + # modify the file descriptor on systems with fcntl + # unix and unix-like systems only + try: + FD_CLOEXEC = fcntl.FD_CLOEXEC # pylint: disable=C0103 + except AttributeError: + FD_CLOEXEC = 1 # pylint: disable=C0103 + old_flags = fcntl.fcntl(f_handle.fileno(), fcntl.F_GETFD) + fcntl.fcntl(f_handle.fileno(), fcntl.F_SETFD, old_flags | FD_CLOEXEC) + + return f_handle + + +@contextlib.contextmanager +def flopen(*args, **kwargs): + ''' + Shortcut for fopen with lock and context manager. + ''' + with fopen(*args, **kwargs) as f_handle: + try: + if is_fcntl_available(check_sunos=True): + fcntl.flock(f_handle.fileno(), fcntl.LOCK_SH) + yield f_handle + finally: + if is_fcntl_available(check_sunos=True): + fcntl.flock(f_handle.fileno(), fcntl.LOCK_UN) + + +@contextlib.contextmanager +def fpopen(*args, **kwargs): + ''' + Shortcut for fopen with extra uid, gid, and mode options. + + Supported optional Keyword Arguments: + + mode + Explicit mode to set. Mode is anything os.chmod would accept + as input for mode. Works only on unix/unix-like systems. + + uid + The uid to set, if not set, or it is None or -1 no changes are + made. Same applies if the path is already owned by this uid. + Must be int. Works only on unix/unix-like systems. + + gid + The gid to set, if not set, or it is None or -1 no changes are + made. Same applies if the path is already owned by this gid. + Must be int. Works only on unix/unix-like systems. + + ''' + # Remove uid, gid and mode from kwargs if present + uid = kwargs.pop('uid', -1) # -1 means no change to current uid + gid = kwargs.pop('gid', -1) # -1 means no change to current gid + mode = kwargs.pop('mode', None) + with fopen(*args, **kwargs) as f_handle: + path = args[0] + d_stat = os.stat(path) + + if hasattr(os, 'chown'): + # if uid and gid are both -1 then go ahead with + # no changes at all + if (d_stat.st_uid != uid or d_stat.st_gid != gid) and \ + [i for i in (uid, gid) if i != -1]: + os.chown(path, uid, gid) + + if mode is not None: + mode_part = stat.S_IMODE(d_stat.st_mode) + if mode_part != mode: + os.chmod(path, (d_stat.st_mode ^ mode_part) | mode) + + yield f_handle + + +def safe_rm(tgt): + ''' + Safely remove a file + ''' + try: + os.remove(tgt) + except (IOError, OSError): + pass + + +def rm_rf(path): + ''' + Platform-independent recursive delete. Includes code from + http://stackoverflow.com/a/2656405 + ''' + def _onerror(func, path, exc_info): + ''' + Error handler for `shutil.rmtree`. + + If the error is due to an access error (read only file) + it attempts to add write permission and then retries. + + If the error is for another reason it re-raises the error. + + Usage : `shutil.rmtree(path, onerror=onerror)` + ''' + if salt.utils.platform.is_windows() and not os.access(path, os.W_OK): + # Is the error an access error ? + os.chmod(path, stat.S_IWUSR) + func(path) + else: + raise # pylint: disable=E0704 + if os.path.islink(path) or not os.path.isdir(path): + os.remove(path) + else: + shutil.rmtree(path, onerror=_onerror) + + +@jinja_filter('is_empty') +def is_empty(filename): + ''' + Is a file empty? + ''' + try: + return os.stat(filename).st_size == 0 + except OSError: + # Non-existent file or permission denied to the parent dir + return False + + +def is_fcntl_available(check_sunos=False): + ''' + Simple function to check if the ``fcntl`` module is available or not. + + If ``check_sunos`` is passed as ``True`` an additional check to see if host is + SunOS is also made. For additional information see: http://goo.gl/159FF8 + ''' + if check_sunos and salt.utils.platform.is_sunos(): + return False + return HAS_FCNTL + + +def safe_filename_leaf(file_basename): + ''' + Input the basename of a file, without the directory tree, and returns a safe name to use + i.e. only the required characters are converted by urllib.quote + If the input is a PY2 String, output a PY2 String. If input is Unicode output Unicode. + For consistency all platforms are treated the same. Hard coded to utf8 as its ascii compatible + windows is \\ / : * ? " < > | posix is / + + .. versionadded:: 2017.7.2 + ''' + def _replace(re_obj): + return urllib.quote(re_obj.group(0), safe=u'') + if not isinstance(file_basename, six.text_type): + # the following string is not prefixed with u + return re.sub('[\\\\:/*?"<>|]', + _replace, + six.text_type(file_basename, 'utf8').encode('ascii', 'backslashreplace')) + # the following string is prefixed with u + return re.sub(u'[\\\\:/*?"<>|]', _replace, file_basename, flags=re.UNICODE) + + +def safe_filepath(file_path_name): + ''' + Input the full path and filename, splits on directory separator and calls safe_filename_leaf for + each part of the path. + + .. versionadded:: 2017.7.2 + ''' + (drive, path) = os.path.splitdrive(file_path_name) + path = os.sep.join([safe_filename_leaf(file_section) for file_section in file_path_name.rsplit(os.sep)]) + if drive: + return os.sep.join([drive, path]) + else: + return path + + +@jinja_filter('is_text_file') +def is_text_file(fp_, blocksize=512): + ''' + Uses heuristics to guess whether the given file is text or binary, + by reading a single block of bytes from the file. + If more than 30% of the chars in the block are non-text, or there + are NUL ('\x00') bytes in the block, assume this is a binary file. + ''' + int2byte = (lambda x: bytes((x,))) if six.PY3 else chr + text_characters = ( + b''.join(int2byte(i) for i in range(32, 127)) + + b'\n\r\t\f\b') + try: + block = fp_.read(blocksize) + except AttributeError: + # This wasn't an open filehandle, so treat it as a file path and try to + # open the file + try: + with fopen(fp_, 'rb') as fp2_: + block = fp2_.read(blocksize) + except IOError: + # Unable to open file, bail out and return false + return False + if b'\x00' in block: + # Files with null bytes are binary + return False + elif not block: + # An empty file is considered a valid text file + return True + try: + block.decode('utf-8') + return True + except UnicodeDecodeError: + pass + + nontext = block.translate(None, text_characters) + return float(len(nontext)) / len(block) <= 0.30 diff --git a/salt/utils/find.py b/salt/utils/find.py index 67c9820d4d3..565ee996c3d 100644 --- a/salt/utils/find.py +++ b/salt/utils/find.py @@ -102,10 +102,12 @@ except ImportError: pass # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six # Import salt libs import salt.utils +import salt.utils.args +import salt.utils.stringutils import salt.defaults.exitcodes from salt.utils.filebuffer import BufferedReader @@ -561,8 +563,8 @@ class ExecOption(Option): def execute(self, fullpath, fstat, test=False): try: command = self.command.replace('{}', fullpath) - print(salt.utils.shlex_split(command)) - p = Popen(salt.utils.shlex_split(command), + print(salt.utils.args.shlex_split(command)) + p = Popen(salt.utils.args.shlex_split(command), stdout=PIPE, stderr=PIPE) (out, err) = p.communicate() @@ -570,8 +572,8 @@ class ExecOption(Option): log.error( 'Error running command: {0}\n\n{1}'.format( command, - salt.utils.to_str(err))) - return "{0}:\n{1}\n".format(command, salt.utils.to_str(out)) + salt.utils.stringutils.to_str(err))) + return "{0}:\n{1}\n".format(command, salt.utils.stringutils.to_str(out)) except Exception as e: log.error( diff --git a/salt/utils/fsutils.py b/salt/utils/fsutils.py index 3894219396c..049ab6c0a2c 100644 --- a/salt/utils/fsutils.py +++ b/salt/utils/fsutils.py @@ -13,11 +13,11 @@ import os import logging # Import Salt libs -import salt.utils +import salt.utils.files from salt.exceptions import CommandExecutionError # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six log = logging.getLogger(__name__) @@ -41,7 +41,7 @@ def _get_mounts(fs_type=None): List mounted filesystems. ''' mounts = {} - with salt.utils.fopen("/proc/mounts") as fhr: + with salt.utils.files.fopen('/proc/mounts') as fhr: for line in fhr.readlines(): device, mntpnt, fstype, options, fs_freq, fs_passno = line.strip().split(" ") if fs_type and fstype != fs_type: diff --git a/salt/utils/gitfs.py b/salt/utils/gitfs.py index 00320afb23a..a0b0b20ca11 100644 --- a/salt/utils/gitfs.py +++ b/salt/utils/gitfs.py @@ -21,24 +21,31 @@ from datetime import datetime # Import salt libs import salt.utils +import salt.utils.configparser +import salt.utils.files import salt.utils.itertools +import salt.utils.path +import salt.utils.platform +import salt.utils.stringutils import salt.utils.url +import salt.utils.versions import salt.fileserver +from salt.config import DEFAULT_MASTER_OPTS as _DEFAULT_MASTER_OPTS from salt.utils.odict import OrderedDict from salt.utils.process import os_is_running as pid_exists from salt.exceptions import ( FileserverConfigError, GitLockError, - GitRemoteError, get_error_message ) from salt.utils.event import tagify from salt.utils.versions import LooseVersion as _LooseVersion # Import third party libs -import salt.ext.six as six +from salt.ext import six + +VALID_REF_TYPES = _DEFAULT_MASTER_OPTS['gitfs_ref_types'] -VALID_PROVIDERS = ('pygit2', 'gitpython') # Optional per-remote params that can only be used on a per-remote basis, and # thus do not have defaults in salt/config.py. PER_REMOTE_ONLY = ('name',) @@ -116,11 +123,13 @@ def enforce_types(key, val): non_string_params = { 'ssl_verify': bool, 'insecure_auth': bool, + 'disable_saltenv_mapping': bool, 'env_whitelist': 'stringlist', 'env_blacklist': 'stringlist', 'saltenv_whitelist': 'stringlist', 'saltenv_blacklist': 'stringlist', 'refspecs': 'stringlist', + 'ref_types': 'stringlist', } def _find_global(key): @@ -166,7 +175,7 @@ class GitProvider(object): directly. self.provider should be set in the sub-class' __init__ function before - invoking GitProvider.__init__(). + invoking the parent class' __init__. ''' def __init__(self, opts, remote, per_remote_defaults, per_remote_only, override_params, cache_root, role='gitfs'): @@ -318,10 +327,32 @@ class GitProvider(object): setattr(self, '_' + key, self.conf[key]) self.add_conf_overlay(key) + if not hasattr(self, 'refspecs'): + # This was not specified as a per-remote overrideable parameter + # when instantiating an instance of a GitBase subclass. Make sure + # that we set this attribute so we at least have a sane default and + # are able to fetch. + key = '{0}_refspecs'.format(self.role) + try: + default_refspecs = _DEFAULT_MASTER_OPTS[key] + except KeyError: + log.critical( + 'The \'%s\' option has no default value in ' + 'salt/config/__init__.py.', key + ) + failhard(self.role) + + setattr(self, 'refspecs', default_refspecs) + log.debug( + 'The \'refspecs\' option was not explicitly defined as a ' + 'configurable parameter. Falling back to %s for %s remote ' + '\'%s\'.', default_refspecs, self.role, self.id + ) + for item in ('env_whitelist', 'env_blacklist'): val = getattr(self, item, None) if val: - salt.utils.warn_until( + salt.utils.versions.warn_until( 'Neon', 'The gitfs_{0} config option (and {0} per-remote config ' 'option) have been renamed to gitfs_salt{0} (and ' @@ -333,6 +364,23 @@ class GitProvider(object): # params as attributes delattr(self, 'conf') + # Normalize components of the ref_types configuration and check for + # invalid configuration. + if hasattr(self, 'ref_types'): + self.ref_types = [x.lower() for x in self.ref_types] + invalid_ref_types = [x for x in self.ref_types + if x not in VALID_REF_TYPES] + if invalid_ref_types: + log.critical( + 'The following ref_types for %s remote \'%s\' are ' + 'invalid: %s. The supported values are: %s', + self.role, + self.id, + ', '.join(invalid_ref_types), + ', '.join(VALID_REF_TYPES), + ) + failhard(self.role) + if not isinstance(self.url, six.string_types): log.critical( 'Invalid %s remote \'%s\'. Remotes must be strings, you ' @@ -347,13 +395,13 @@ class GitProvider(object): else: self.hash = hash_type(self.id).hexdigest() self.cachedir_basename = getattr(self, 'name', self.hash) - self.cachedir = salt.utils.path_join(cache_root, self.cachedir_basename) - self.linkdir = salt.utils.path_join(cache_root, + self.cachedir = salt.utils.path.join(cache_root, self.cachedir_basename) + self.linkdir = salt.utils.path.join(cache_root, 'links', self.cachedir_basename) try: # Remove linkdir if it exists - salt.utils.rm_rf(self.linkdir) + salt.utils.files.rm_rf(self.linkdir) except OSError: pass @@ -384,21 +432,24 @@ class GitProvider(object): else: env_set.add('base' if rname == self.base else rname) + use_branches = 'branch' in self.ref_types + use_tags = 'tag' in self.ref_types + ret = set() for ref in refs: ref = re.sub('^refs/', '', ref) rtype, rname = ref.split('/', 1) - if rtype == 'remotes': + if rtype == 'remotes' and use_branches: parted = rname.partition('/') rname = parted[2] if parted[2] else parted[0] _check_ref(ret, rname) - elif rtype == 'tags': + elif rtype == 'tags' and use_tags: _check_ref(ret, rname) return ret def _get_lock_file(self, lock_type='update'): - return salt.utils.path_join(self.gitdir, lock_type + '.lk') + return salt.utils.path.join(self.gitdir, lock_type + '.lk') @classmethod def add_conf_overlay(cls, name): @@ -424,9 +475,9 @@ class GitProvider(object): return None # Return the all_saltenvs branch/tag if it is configured + per_saltenv_ref = _get_per_saltenv(tgt_env) try: all_saltenvs_ref = self.all_saltenvs - per_saltenv_ref = _get_per_saltenv(tgt_env) if per_saltenv_ref and all_saltenvs_ref != per_saltenv_ref: log.debug( 'The per-saltenv configuration has mapped the ' @@ -444,8 +495,16 @@ class GitProvider(object): if tgt_env == 'base': return self.base + elif self.disable_saltenv_mapping: + if per_saltenv_ref is None: + log.debug( + 'saltenv mapping is diabled for %s remote \'%s\' ' + 'and saltenv \'%s\' is not explicitly mapped', + self.role, self.id, tgt_env + ) + return per_saltenv_ref else: - return _get_per_saltenv(tgt_env) or tgt_env + return per_saltenv_ref or tgt_env if name in saltenv_conf: return strip_sep(saltenv_conf[name]) @@ -456,12 +515,6 @@ class GitProvider(object): return strip_sep(getattr(self, '_' + name)) setattr(cls, name, _getconf) - def add_refspecs(self, *refspecs): - ''' - This function must be overridden in a sub-class - ''' - raise NotImplementedError() - def check_root(self): ''' Check if the relative root path exists in the checked-out copy of the @@ -471,7 +524,7 @@ class GitProvider(object): # No need to pass an environment to self.root() here since per-saltenv # configuration is a gitfs-only feature and check_root() is not used # for gitfs. - root_dir = salt.utils.path_join(self.cachedir, self.root()).rstrip(os.sep) + root_dir = salt.utils.path.join(self.cachedir, self.root()).rstrip(os.sep) if os.path.isdir(root_dir): return root_dir log.error( @@ -488,7 +541,7 @@ class GitProvider(object): cmd_str = 'git remote prune origin' cmd = subprocess.Popen( shlex.split(cmd_str), - close_fds=not salt.utils.is_windows(), + close_fds=not salt.utils.platform.is_windows(), cwd=os.path.dirname(self.gitdir), stdout=subprocess.PIPE, stderr=subprocess.STDOUT) @@ -554,55 +607,95 @@ class GitProvider(object): success.append(msg) return success, failed - def configure_refspecs(self): + def enforce_git_config(self): ''' - Ensure that the configured refspecs are set + For the config options which need to be maintained in the git config, + ensure that the git config file is configured as desired. ''' - try: - refspecs = set(self.get_refspecs()) - except (git.exc.GitCommandError, GitRemoteError) as exc: - log.error( - 'Failed to get refspecs for %s remote \'%s\': %s', - self.role, - self.id, - exc - ) - return - - desired_refspecs = set(self.refspecs) - to_delete = refspecs - desired_refspecs if refspecs else set() - if to_delete: - # There is no native unset support in Pygit2, and GitPython just - # wraps the CLI anyway. So we'll just use the git CLI to - # --unset-all the config value. Then, we will add back all - # configured refspecs. This is more foolproof than trying to remove - # specific refspecs, as removing specific ones necessitates - # formulating a regex to match, and the fact that slashes and - # asterisks are in refspecs complicates this. - cmd_str = 'git config --unset-all remote.origin.fetch' - cmd = subprocess.Popen( - shlex.split(cmd_str), - close_fds=not salt.utils.is_windows(), - cwd=os.path.dirname(self.gitdir), - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - output = cmd.communicate()[0] - if cmd.returncode != 0: - log.error( - 'Failed to unset git config value for %s remote \'%s\'. ' - 'Output from \'%s\' follows:\n%s', - self.role, self.id, cmd_str, output - ) - return - # Since we had to remove all refspecs, we now need to add all - # desired refspecs to achieve the desired configuration. - to_add = desired_refspecs + git_config = os.path.join(self.gitdir, 'config') + conf = salt.utils.configparser.GitConfigParser() + if not conf.read(git_config): + log.error('Failed to read from git config file %s', git_config) else: - # We didn't need to delete any refspecs, so we'll only need to add - # the desired refspecs that aren't currently configured. - to_add = desired_refspecs - refspecs + # We are currently enforcing the following git config items: + # 1. Fetch URL + # 2. refspecs used in fetch + # 3. http.sslVerify + conf_changed = False + remote_section = 'remote "origin"' - self.add_refspecs(*to_add) + # 1. URL + try: + url = conf.get(remote_section, 'url') + except salt.utils.configparser.NoSectionError: + # First time we've init'ed this repo, we need to add the + # section for the remote to the git config + conf.add_section(remote_section) + conf_changed = True + url = None + log.debug( + 'Current fetch URL for %s remote \'%s\': %s (desired: %s)', + self.role, self.id, url, self.url + ) + if url != self.url: + conf.set(remote_section, 'url', self.url) + log.debug( + 'Fetch URL for %s remote \'%s\' set to %s', + self.role, self.id, self.url + ) + conf_changed = True + + # 2. refspecs + try: + refspecs = sorted( + conf.get(remote_section, 'fetch', as_list=True)) + except salt.utils.configparser.NoOptionError: + # No 'fetch' option present in the remote section. Should never + # happen, but if it does for some reason, don't let it cause a + # traceback. + refspecs = [] + desired_refspecs = sorted(self.refspecs) + log.debug( + 'Current refspecs for %s remote \'%s\': %s (desired: %s)', + self.role, self.id, refspecs, desired_refspecs + ) + if refspecs != desired_refspecs: + conf.set_multivar(remote_section, 'fetch', self.refspecs) + log.debug( + 'Refspecs for %s remote \'%s\' set to %s', + self.role, self.id, desired_refspecs + ) + conf_changed = True + + # 3. http.sslVerify + try: + ssl_verify = conf.get('http', 'sslVerify') + except salt.utils.configparser.NoSectionError: + conf.add_section('http') + ssl_verify = None + except salt.utils.configparser.NoOptionError: + ssl_verify = None + desired_ssl_verify = six.text_type(self.ssl_verify).lower() + log.debug( + 'Current http.sslVerify for %s remote \'%s\': %s (desired: %s)', + self.role, self.id, ssl_verify, desired_ssl_verify + ) + if ssl_verify != desired_ssl_verify: + conf.set('http', 'sslVerify', desired_ssl_verify) + log.debug( + 'http.sslVerify for %s remote \'%s\' set to %s', + self.role, self.id, desired_ssl_verify + ) + conf_changed = True + + # Write changes, if necessary + if conf_changed: + with salt.utils.files.fopen(git_config, 'w') as fp_: + conf.write(fp_) + log.debug( + 'Config updates for %s remote \'%s\' written to %s', + self.role, self.id, git_config + ) def fetch(self): ''' @@ -645,7 +738,7 @@ class GitProvider(object): os.write(fh_, six.b(str(os.getpid()))) except (OSError, IOError) as exc: if exc.errno == errno.EEXIST: - with salt.utils.fopen(self._get_lock_file(lock_type), 'r') as fd_: + with salt.utils.files.fopen(self._get_lock_file(lock_type), 'r') as fd_: try: pid = int(fd_.readline().rstrip()) except ValueError: @@ -816,12 +909,6 @@ class GitProvider(object): else target return self.branch - def get_refspecs(self): - ''' - This function must be overridden in a sub-class - ''' - raise NotImplementedError() - def get_tree(self, tgt_env): ''' Return a tree object for the specified environment @@ -833,7 +920,7 @@ class GitProvider(object): if tgt_ref is None: return None - for ref_type in ('branch', 'tag', 'sha'): + for ref_type in self.ref_types: try: func_name = 'get_tree_from_{0}'.format(ref_type) func = getattr(self, func_name) @@ -893,25 +980,10 @@ class GitPython(GitProvider): def __init__(self, opts, remote, per_remote_defaults, per_remote_only, override_params, cache_root, role='gitfs'): self.provider = 'gitpython' - GitProvider.__init__(self, opts, remote, per_remote_defaults, - per_remote_only, override_params, cache_root, role) - - def add_refspecs(self, *refspecs): - ''' - Add the specified refspecs to the "origin" remote - ''' - for refspec in refspecs: - try: - self.repo.git.config('--add', 'remote.origin.fetch', refspec) - log.debug( - 'Added refspec \'%s\' to %s remote \'%s\'', - refspec, self.role, self.id - ) - except git.exc.GitCommandError as exc: - log.error( - 'Failed to add refspec \'%s\' to %s remote \'%s\': %s', - refspec, self.role, self.id, exc - ) + super(GitPython, self).__init__( + opts, remote, per_remote_defaults, per_remote_only, + override_params, cache_root, role + ) def checkout(self): ''' @@ -999,30 +1071,8 @@ class GitPython(GitProvider): log.error(_INVALID_REPO.format(self.cachedir, self.url, self.role)) return new - self.gitdir = salt.utils.path_join(self.repo.working_dir, '.git') - - if not self.repo.remotes: - try: - self.repo.create_remote('origin', self.url) - except os.error: - # This exception occurs when two processes are trying to write - # to the git config at once, go ahead and pass over it since - # this is the only write. This should place a lock down. - pass - else: - new = True - - try: - ssl_verify = self.repo.git.config('--get', 'http.sslVerify') - except git.exc.GitCommandError: - ssl_verify = '' - desired_ssl_verify = str(self.ssl_verify).lower() - if ssl_verify != desired_ssl_verify: - self.repo.git.config('http.sslVerify', desired_ssl_verify) - - # Ensure that refspecs for the "origin" remote are set up as configured - if hasattr(self, 'refspecs'): - self.configure_refspecs() + self.gitdir = salt.utils.path.join(self.repo.working_dir, '.git') + self.enforce_git_config() return new @@ -1042,7 +1092,7 @@ class GitPython(GitProvider): relpath = lambda path: os.path.relpath(path, self.root(tgt_env)) else: relpath = lambda path: path - add_mountpoint = lambda path: salt.utils.path_join( + add_mountpoint = lambda path: salt.utils.path.join( self.mountpoint(tgt_env), path, use_posixpath=True) for blob in tree.traverse(): if isinstance(blob, git.Tree): @@ -1095,7 +1145,7 @@ class GitPython(GitProvider): new_objs = True cleaned = self.clean_stale_refs() - return bool(new_objs or cleaned) + return True if (new_objs or cleaned) else None def file_list(self, tgt_env): ''' @@ -1115,7 +1165,7 @@ class GitPython(GitProvider): relpath = lambda path: os.path.relpath(path, self.root(tgt_env)) else: relpath = lambda path: path - add_mountpoint = lambda path: salt.utils.path_join( + add_mountpoint = lambda path: salt.utils.path.join( self.mountpoint(tgt_env), path, use_posixpath=True) for file_blob in tree.traverse(): if not isinstance(file_blob, git.Blob): @@ -1137,7 +1187,7 @@ class GitPython(GitProvider): ''' tree = self.get_tree(tgt_env) if not tree: - # Branch/tag/SHA not found + # Branch/tag/SHA not found in repo return None, None, None blob = None depth = 0 @@ -1158,7 +1208,7 @@ class GitPython(GitProvider): stream.seek(0) link_tgt = stream.read() stream.close() - path = salt.utils.path_join( + path = salt.utils.path.join( os.path.dirname(path), link_tgt, use_posixpath=True) else: blob = file_blob @@ -1174,13 +1224,6 @@ class GitPython(GitProvider): return blob, blob.hexsha, blob.mode return None, None, None - def get_refspecs(self): - ''' - Return the configured refspecs - ''' - refspecs = self.repo.git.config('--get-all', 'remote.origin.fetch') - return [x.strip() for x in refspecs.splitlines()] - def get_tree_from_branch(self, ref): ''' Return a git.Tree object matching a head ref fetched into @@ -1217,7 +1260,7 @@ class GitPython(GitProvider): ''' Using the blob object, write the file to the destination path ''' - with salt.utils.fopen(dest, 'wb+') as fp_: + with salt.utils.files.fopen(dest, 'wb+') as fp_: blob.stream_data(fp_) @@ -1228,29 +1271,10 @@ class Pygit2(GitProvider): def __init__(self, opts, remote, per_remote_defaults, per_remote_only, override_params, cache_root, role='gitfs'): self.provider = 'pygit2' - GitProvider.__init__(self, opts, remote, per_remote_defaults, - per_remote_only, override_params, cache_root, role) - - def add_refspecs(self, *refspecs): - ''' - Add the specified refspecs to the "origin" remote - ''' - for refspec in refspecs: - try: - self.repo.config.set_multivar( - 'remote.origin.fetch', - 'FOO', - refspec - ) - log.debug( - 'Added refspec \'%s\' to %s remote \'%s\'', - refspec, self.role, self.id - ) - except Exception as exc: - log.error( - 'Failed to add refspec \'%s\' to %s remote \'%s\': %s', - refspec, self.role, self.id, exc - ) + super(Pygit2, self).__init__( + opts, remote, per_remote_defaults, per_remote_only, + override_params, cache_root, role + ) def checkout(self): ''' @@ -1477,31 +1501,8 @@ class Pygit2(GitProvider): log.error(_INVALID_REPO.format(self.cachedir, self.url, self.role)) return new - self.gitdir = salt.utils.path_join(self.repo.workdir, '.git') - - if not self.repo.remotes: - try: - self.repo.create_remote('origin', self.url) - except os.error: - # This exception occurs when two processes are trying to write - # to the git config at once, go ahead and pass over it since - # this is the only write. This should place a lock down. - pass - else: - new = True - - try: - ssl_verify = self.repo.config.get_bool('http.sslVerify') - except KeyError: - ssl_verify = None - if ssl_verify != self.ssl_verify: - self.repo.config.set_multivar('http.sslVerify', - '', - str(self.ssl_verify).lower()) - - # Ensure that refspecs for the "origin" remote are set up as configured - if hasattr(self, 'refspecs'): - self.configure_refspecs() + self.gitdir = salt.utils.path.join(self.repo.workdir, '.git') + self.enforce_git_config() return new @@ -1522,11 +1523,11 @@ class Pygit2(GitProvider): if not isinstance(blob, pygit2.Tree): continue blobs.append( - salt.utils.path_join(prefix, entry.name, use_posixpath=True) + salt.utils.path.join(prefix, entry.name, use_posixpath=True) ) if len(blob): _traverse( - blob, blobs, salt.utils.path_join( + blob, blobs, salt.utils.path.join( prefix, entry.name, use_posixpath=True) ) @@ -1548,7 +1549,7 @@ class Pygit2(GitProvider): blobs = [] if len(tree): _traverse(tree, blobs, self.root(tgt_env)) - add_mountpoint = lambda path: salt.utils.path_join( + add_mountpoint = lambda path: salt.utils.path.join( self.mountpoint(tgt_env), path, use_posixpath=True) for blob in blobs: ret.add(add_mountpoint(relpath(blob))) @@ -1620,7 +1621,9 @@ class Pygit2(GitProvider): log.debug('%s remote \'%s\' is up-to-date', self.role, self.id) refs_post = self.repo.listall_references() cleaned = self.clean_stale_refs(local_refs=refs_post) - return bool(received_objects or refs_pre != refs_post or cleaned) + return True \ + if (received_objects or refs_pre != refs_post or cleaned) \ + else None def file_list(self, tgt_env): ''' @@ -1637,7 +1640,7 @@ class Pygit2(GitProvider): continue obj = self.repo[entry.oid] if isinstance(obj, pygit2.Blob): - repo_path = salt.utils.path_join( + repo_path = salt.utils.path.join( prefix, entry.name, use_posixpath=True) blobs.setdefault('files', []).append(repo_path) if stat.S_ISLNK(tree[entry.name].filemode): @@ -1645,7 +1648,7 @@ class Pygit2(GitProvider): blobs.setdefault('symlinks', {})[repo_path] = link_tgt elif isinstance(obj, pygit2.Tree): _traverse( - obj, blobs, salt.utils.path_join( + obj, blobs, salt.utils.path.join( prefix, entry.name, use_posixpath=True) ) @@ -1671,7 +1674,7 @@ class Pygit2(GitProvider): blobs = {} if len(tree): _traverse(tree, blobs, self.root(tgt_env)) - add_mountpoint = lambda path: salt.utils.path_join( + add_mountpoint = lambda path: salt.utils.path.join( self.mountpoint(tgt_env), path, use_posixpath=True) for repo_path in blobs.get('files', []): files.add(add_mountpoint(relpath(repo_path))) @@ -1704,7 +1707,7 @@ class Pygit2(GitProvider): # the symlink and set path to the location indicated # in the blob data. link_tgt = self.repo[entry.oid].data - path = salt.utils.path_join( + path = salt.utils.path.join( os.path.dirname(path), link_tgt, use_posixpath=True) else: blob = self.repo[entry.oid] @@ -1719,14 +1722,6 @@ class Pygit2(GitProvider): return blob, blob.hex, mode return None, None, None - def get_refspecs(self): - ''' - Return the configured refspecs - ''' - if not [x for x in self.repo.config if x.startswith('remote.origin.')]: - raise GitRemoteError('\'origin\' remote not not present') - return list(self.repo.config.get_multivar('remote.origin.fetch')) - def get_tree_from_branch(self, ref): ''' Return a pygit2.Tree object matching a head ref fetched into @@ -1759,6 +1754,7 @@ class Pygit2(GitProvider): def setup_callbacks(self): ''' + Assign attributes for pygit2 callbacks ''' # pygit2 radically changed fetching in 0.23.2 pygit2_version = pygit2.__version__ @@ -1909,15 +1905,21 @@ class Pygit2(GitProvider): ''' Using the blob object, write the file to the destination path ''' - with salt.utils.fopen(dest, 'wb+') as fp_: + with salt.utils.files.fopen(dest, 'wb+') as fp_: fp_.write(blob.data) +GIT_PROVIDERS = { + 'pygit2': Pygit2, + 'gitpython': GitPython, +} + + class GitBase(object): ''' Base class for gitfs/git_pillar ''' - def __init__(self, opts, valid_providers=VALID_PROVIDERS, cache_root=None): + def __init__(self, opts, git_providers=None, cache_root=None): ''' IMPORTANT: If specifying a cache_root, understand that this is also where the remotes will be cloned. A non-default cache_root is only @@ -1925,17 +1927,18 @@ class GitBase(object): out into the winrepo locations and not within the cachedir. ''' self.opts = opts - self.valid_providers = valid_providers - self.get_provider() + self.git_providers = git_providers if git_providers is not None \ + else GIT_PROVIDERS + self.verify_provider() if cache_root is not None: self.cache_root = self.remote_root = cache_root else: - self.cache_root = salt.utils.path_join(self.opts['cachedir'], + self.cache_root = salt.utils.path.join(self.opts['cachedir'], self.role) - self.remote_root = salt.utils.path_join(self.cache_root, 'remotes') - self.env_cache = salt.utils.path_join(self.cache_root, 'envs.p') - self.hash_cachedir = salt.utils.path_join(self.cache_root, 'hash') - self.file_list_cachedir = salt.utils.path_join( + self.remote_root = salt.utils.path.join(self.cache_root, 'remotes') + self.env_cache = salt.utils.path.join(self.cache_root, 'envs.p') + self.hash_cachedir = salt.utils.path.join(self.cache_root, 'hash') + self.file_list_cachedir = salt.utils.path.join( self.opts['cachedir'], 'file_lists', self.role) def init_remotes(self, remotes, per_remote_overrides, @@ -1984,7 +1987,7 @@ class GitBase(object): self.remotes = [] for remote in remotes: - repo_obj = self.provider_class( + repo_obj = self.git_providers[self.provider]( self.opts, remote, per_remote_defaults, @@ -2080,7 +2083,7 @@ class GitBase(object): for item in cachedir_ls: if item in ('hash', 'refs'): continue - path = salt.utils.path_join(self.cache_root, item) + path = salt.utils.path.join(self.cache_root, item) if os.path.isdir(path): to_remove.append(path) failed = [] @@ -2213,7 +2216,7 @@ class GitBase(object): if refresh_env_cache: new_envs = self.envs(ignore_cache=True) serial = salt.payload.Serial(self.opts) - with salt.utils.fopen(self.env_cache, 'wb+') as fp_: + with salt.utils.files.fopen(self.env_cache, 'wb+') as fp_: fp_.write(serial.dumps(new_envs)) log.trace('Wrote env cache data to {0}'.format(self.env_cache)) @@ -2238,7 +2241,7 @@ class GitBase(object): # Hash file won't exist if no files have yet been served up pass - def get_provider(self): + def verify_provider(self): ''' Determine which provider to use ''' @@ -2259,12 +2262,12 @@ class GitBase(object): # Should only happen if someone does something silly like # set the provider to a numeric value. desired_provider = str(desired_provider).lower() - if desired_provider not in self.valid_providers: + if desired_provider not in self.git_providers: log.critical( 'Invalid {0}_provider \'{1}\'. Valid choices are: {2}' .format(self.role, desired_provider, - ', '.join(self.valid_providers)) + ', '.join(self.git_providers)) ) failhard(self.role) elif desired_provider == 'pygit2' and self.verify_pygit2(): @@ -2277,17 +2280,13 @@ class GitBase(object): .format(self.role) ) failhard(self.role) - if self.provider == 'pygit2': - self.provider_class = Pygit2 - elif self.provider == 'gitpython': - self.provider_class = GitPython def verify_gitpython(self, quiet=False): ''' Check if GitPython is available and at a compatible version (>= 0.3.0) ''' def _recommend(): - if HAS_PYGIT2 and 'pygit2' in self.valid_providers: + if HAS_PYGIT2 and 'pygit2' in self.git_providers: log.error(_RECOMMEND_PYGIT2.format(self.role)) if not HAS_GITPYTHON: @@ -2298,7 +2297,7 @@ class GitBase(object): ) _recommend() return False - elif 'gitpython' not in self.valid_providers: + elif 'gitpython' not in self.git_providers: return False # pylint: disable=no-member @@ -2315,7 +2314,7 @@ class GitBase(object): git.__version__ ) ) - if not salt.utils.which('git'): + if not salt.utils.path.which('git'): errors.append( 'The git command line utility is required when using the ' '\'gitpython\' {0}_provider.'.format(self.role) @@ -2338,7 +2337,7 @@ class GitBase(object): Pygit2 must be at least 0.20.3 and libgit2 must be at least 0.20.0. ''' def _recommend(): - if HAS_GITPYTHON and 'gitpython' in self.valid_providers: + if HAS_GITPYTHON and 'gitpython' in self.git_providers: log.error(_RECOMMEND_GITPYTHON.format(self.role)) if not HAS_PYGIT2: @@ -2349,7 +2348,7 @@ class GitBase(object): ) _recommend() return False - elif 'pygit2' not in self.valid_providers: + elif 'pygit2' not in self.git_providers: return False # pylint: disable=no-member @@ -2379,7 +2378,7 @@ class GitBase(object): pygit2.LIBGIT2_VERSION ) ) - if not salt.utils.which('git'): + if not salt.utils.path.which('git'): errors.append( 'The git command line utility is required when using the ' '\'pygit2\' {0}_provider.'.format(self.role) @@ -2400,9 +2399,9 @@ class GitBase(object): ''' Write the remote_map.txt ''' - remote_map = salt.utils.path_join(self.cache_root, 'remote_map.txt') + remote_map = salt.utils.path.join(self.cache_root, 'remote_map.txt') try: - with salt.utils.fopen(remote_map, 'w+') as fp_: + with salt.utils.files.fopen(remote_map, 'w+') as fp_: timestamp = \ datetime.now().strftime('%d %b %Y %H:%M:%S.%f') fp_.write( @@ -2468,7 +2467,7 @@ class GitFS(GitBase): ''' def __init__(self, opts): self.role = 'gitfs' - GitBase.__init__(self, opts) + super(GitFS, self).__init__(opts) def dir_list(self, load): ''' @@ -2490,7 +2489,8 @@ class GitFS(GitBase): ret = set() for repo in self.remotes: repo_envs = set() - repo_envs.update(repo.envs()) + if not repo.disable_saltenv_mapping: + repo_envs.update(repo.envs()) for env_list in six.itervalues(repo.saltenv_revmap): repo_envs.update(env_list) ret.update([x for x in repo_envs if repo.env_is_exposed(x)]) @@ -2504,17 +2504,17 @@ class GitFS(GitBase): fnd = {'path': '', 'rel': ''} if os.path.isabs(path) or \ - (not salt.utils.is_hex(tgt_env) and tgt_env not in self.envs()): + (not salt.utils.stringutils.is_hex(tgt_env) and tgt_env not in self.envs()): return fnd - dest = salt.utils.path_join(self.cache_root, 'refs', tgt_env, path) - hashes_glob = salt.utils.path_join(self.hash_cachedir, + dest = salt.utils.path.join(self.cache_root, 'refs', tgt_env, path) + hashes_glob = salt.utils.path.join(self.hash_cachedir, tgt_env, '{0}.hash.*'.format(path)) - blobshadest = salt.utils.path_join(self.hash_cachedir, + blobshadest = salt.utils.path.join(self.hash_cachedir, tgt_env, '{0}.hash.blob_sha1'.format(path)) - lk_fn = salt.utils.path_join(self.hash_cachedir, + lk_fn = salt.utils.path.join(self.hash_cachedir, tgt_env, '{0}.lk'.format(path)) destdir = os.path.dirname(dest) @@ -2540,7 +2540,7 @@ class GitFS(GitBase): continue repo_path = path[len(repo.mountpoint(tgt_env)):].lstrip(os.sep) if repo.root(tgt_env): - repo_path = salt.utils.path_join(repo.root(tgt_env), repo_path) + repo_path = salt.utils.path.join(repo.root(tgt_env), repo_path) blob, blob_hexsha, blob_mode = repo.find_file(repo_path, tgt_env) if blob is None: @@ -2562,13 +2562,13 @@ class GitFS(GitBase): salt.fileserver.wait_lock(lk_fn, dest) if os.path.isfile(blobshadest) and os.path.isfile(dest): - with salt.utils.fopen(blobshadest, 'r') as fp_: + with salt.utils.files.fopen(blobshadest, 'r') as fp_: sha = fp_.read() if sha == blob_hexsha: fnd['rel'] = path fnd['path'] = dest return _add_file_stat(fnd, blob_mode) - with salt.utils.fopen(lk_fn, 'w+') as fp_: + with salt.utils.files.fopen(lk_fn, 'w+') as fp_: fp_.write('') for filename in glob.glob(hashes_glob): try: @@ -2577,7 +2577,7 @@ class GitFS(GitBase): pass # Write contents of file to their destination in the FS cache repo.write_file(blob, dest) - with salt.utils.fopen(blobshadest, 'w+') as fp_: + with salt.utils.files.fopen(blobshadest, 'w+') as fp_: fp_.write(blob_hexsha) try: os.remove(lk_fn) @@ -2596,12 +2596,7 @@ class GitFS(GitBase): Return a chunk from a file based on the data received ''' if 'env' in load: - salt.utils.warn_until( - 'Oxygen', - 'Parameter \'env\' has been detected in the argument list. This ' - 'parameter is no longer used and has been replaced by \'saltenv\' ' - 'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.' - ) + # "env" is not supported; Use "saltenv". load.pop('env') ret = {'data': '', @@ -2620,7 +2615,7 @@ class GitFS(GitBase): ret['dest'] = fnd['rel'] gzip = load.get('gzip', None) fpath = os.path.normpath(fnd['path']) - with salt.utils.fopen(fpath, 'rb') as fp_: + with salt.utils.files.fopen(fpath, 'rb') as fp_: fp_.seek(load['loc']) data = fp_.read(self.opts['file_buffer_size']) if data and six.PY3 and not salt.utils.is_bin_file(fpath): @@ -2636,12 +2631,7 @@ class GitFS(GitBase): Return a file hash, the hash type is set in the master config file ''' if 'env' in load: - salt.utils.warn_until( - 'Oxygen', - 'Parameter \'env\' has been detected in the argument list. This ' - 'parameter is no longer used and has been replaced by \'saltenv\' ' - 'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.' - ) + # "env" is not supported; Use "saltenv". load.pop('env') if not all(x in load for x in ('path', 'saltenv')): @@ -2649,7 +2639,7 @@ class GitFS(GitBase): ret = {'hash_type': self.opts['hash_type']} relpath = fnd['rel'] path = fnd['path'] - hashdest = salt.utils.path_join(self.hash_cachedir, + hashdest = salt.utils.path.join(self.hash_cachedir, load['saltenv'], '{0}.hash.{1}'.format(relpath, self.opts['hash_type'])) @@ -2657,11 +2647,11 @@ class GitFS(GitBase): if not os.path.exists(os.path.dirname(hashdest)): os.makedirs(os.path.dirname(hashdest)) ret['hsum'] = salt.utils.get_hash(path, self.opts['hash_type']) - with salt.utils.fopen(hashdest, 'w+') as fp_: + with salt.utils.files.fopen(hashdest, 'w+') as fp_: fp_.write(ret['hsum']) return ret else: - with salt.utils.fopen(hashdest, 'rb') as fp_: + with salt.utils.files.fopen(hashdest, 'rb') as fp_: ret['hsum'] = fp_.read() return ret @@ -2670,12 +2660,7 @@ class GitFS(GitBase): Return a dict containing the file lists for files and dirs ''' if 'env' in load: - salt.utils.warn_until( - 'Oxygen', - 'Parameter \'env\' has been detected in the argument list. This ' - 'parameter is no longer used and has been replaced by \'saltenv\' ' - 'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.' - ) + # "env" is not supported; Use "saltenv". load.pop('env') if not os.path.isdir(self.file_list_cachedir): @@ -2688,11 +2673,11 @@ class GitFS(GitBase): ) ) return [] - list_cache = salt.utils.path_join( + list_cache = salt.utils.path.join( self.file_list_cachedir, '{0}.p'.format(load['saltenv'].replace(os.path.sep, '_|-')) ) - w_lock = salt.utils.path_join( + w_lock = salt.utils.path.join( self.file_list_cachedir, '.{0}.w'.format(load['saltenv'].replace(os.path.sep, '_|-')) ) @@ -2704,7 +2689,7 @@ class GitFS(GitBase): return cache_match if refresh_cache: ret = {'files': set(), 'symlinks': {}, 'dirs': set()} - if salt.utils.is_hex(load['saltenv']) \ + if salt.utils.stringutils.is_hex(load['saltenv']) \ or load['saltenv'] in self.envs(): for repo in self.remotes: repo_files, repo_symlinks = repo.file_list(load['saltenv']) @@ -2744,15 +2729,10 @@ class GitFS(GitBase): Return a dict of all symlinks based on a given path in the repo ''' if 'env' in load: - salt.utils.warn_until( - 'Oxygen', - 'Parameter \'env\' has been detected in the argument list. This ' - 'parameter is no longer used and has been replaced by \'saltenv\' ' - 'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.' - ) + # "env" is not supported; Use "saltenv". load.pop('env') - if not salt.utils.is_hex(load['saltenv']) \ + if not salt.utils.stringutils.is_hex(load['saltenv']) \ and load['saltenv'] not in self.envs(): return {} if 'prefix' in load: @@ -2771,7 +2751,7 @@ class GitPillar(GitBase): ''' def __init__(self, opts): self.role = 'git_pillar' - GitBase.__init__(self, opts) + super(GitPillar, self).__init__(opts) def checkout(self): ''' @@ -2799,7 +2779,7 @@ class GitPillar(GitBase): ''' Ensure that the mountpoint is linked to the passed cachedir ''' - lcachelink = salt.utils.path_join(repo.linkdir, repo._mountpoint) + lcachelink = salt.utils.path.join(repo.linkdir, repo._mountpoint) if not os.path.islink(lcachelink): ldirname = os.path.dirname(lcachelink) try: @@ -2829,7 +2809,7 @@ class GitPillar(GitBase): # A file or dir already exists at this path, remove it and # then re-attempt to create the symlink try: - salt.utils.rm_rf(lcachelink) + salt.utils.files.rm_rf(lcachelink) except OSError as exc: log.error( 'Failed to remove file/dir at path %s: %s', @@ -2854,18 +2834,6 @@ class GitPillar(GitBase): return False return True - def update(self): - ''' - Execute a git fetch on all of the repos. In this case, simply execute - self.fetch_remotes() from the parent class. - - This function only exists to make the git_pillar update code in - master.py (salt.master.Maintenance.handle_git_pillar) less complicated, - once the legacy git_pillar code is purged we can remove this function - and just run pillar.fetch_remotes() there. - ''' - return self.fetch_remotes() - class WinRepo(GitBase): ''' @@ -2873,7 +2841,7 @@ class WinRepo(GitBase): ''' def __init__(self, opts, winrepo_dir): self.role = 'winrepo' - GitBase.__init__(self, opts, cache_root=winrepo_dir) + super(WinRepo, self).__init__(opts, cache_root=winrepo_dir) def checkout(self): ''' diff --git a/salt/utils/github.py b/salt/utils/github.py index a7de81d30fc..6db18c72e21 100644 --- a/salt/utils/github.py +++ b/salt/utils/github.py @@ -10,7 +10,7 @@ import salt.utils.http import logging # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six log = logging.getLogger(__name__) diff --git a/salt/utils/gzip_util.py b/salt/utils/gzip_util.py index 19604a5d48e..56be3135522 100644 --- a/salt/utils/gzip_util.py +++ b/salt/utils/gzip_util.py @@ -11,10 +11,10 @@ from __future__ import absolute_import import gzip # Import Salt libs -import salt.utils +import salt.utils.files # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six from salt.ext.six import BytesIO @@ -96,7 +96,7 @@ def compress_file(fh_, compresslevel=9, chunk_size=1048576): bytes_read = ogz.write(fh_.read(chunk_size)) except AttributeError: # Open the file and re-attempt the read - fh_ = salt.utils.fopen(fh_, 'rb') + fh_ = salt.utils.files.fopen(fh_, 'rb') bytes_read = ogz.write(fh_.read(chunk_size)) yield buf.getvalue() finally: diff --git a/salt/utils/hashutils.py b/salt/utils/hashutils.py index 9e99af1100c..341058e80fc 100644 --- a/salt/utils/hashutils.py +++ b/salt/utils/hashutils.py @@ -8,12 +8,13 @@ from __future__ import absolute_import import base64 import hashlib import hmac +import random # Import Salt libs -import salt.ext.six as six -import salt.utils +from salt.ext import six +import salt.utils.stringutils -from salt.utils.decorators import jinja_filter +from salt.utils.decorators.jinja import jinja_filter @jinja_filter('base64_encode') @@ -25,9 +26,9 @@ def base64_b64encode(instr): newline ('\\n') characters in the encoded output. ''' if six.PY3: - b = salt.utils.to_bytes(instr) + b = salt.utils.stringutils.to_bytes(instr) b64 = base64.b64encode(b) - return salt.utils.to_str(b64) + return salt.utils.stringutils.to_str(b64) return base64.b64encode(instr) @@ -37,10 +38,10 @@ def base64_b64decode(instr): Decode a base64-encoded string using the "modern" Python interface. ''' if six.PY3: - b = salt.utils.to_bytes(instr) + b = salt.utils.stringutils.to_bytes(instr) data = base64.b64decode(b) try: - return salt.utils.to_str(data) + return salt.utils.stringutils.to_str(data) except UnicodeDecodeError: return data return base64.b64decode(instr) @@ -55,9 +56,9 @@ def base64_encodestring(instr): at the end of the encoded string. ''' if six.PY3: - b = salt.utils.to_bytes(instr) + b = salt.utils.stringutils.to_bytes(instr) b64 = base64.encodebytes(b) - return salt.utils.to_str(b64) + return salt.utils.stringutils.to_str(b64) return base64.encodestring(instr) @@ -67,10 +68,10 @@ def base64_decodestring(instr): ''' if six.PY3: - b = salt.utils.to_bytes(instr) + b = salt.utils.stringutils.to_bytes(instr) data = base64.decodebytes(b) try: - return salt.utils.to_str(data) + return salt.utils.stringutils.to_str(data) except UnicodeDecodeError: return data return base64.decodestring(instr) @@ -82,7 +83,7 @@ def md5_digest(instr): Generate an md5 hash of a given string. ''' if six.PY3: - b = salt.utils.to_bytes(instr) + b = salt.utils.stringutils.to_bytes(instr) return hashlib.md5(b).hexdigest() return hashlib.md5(instr).hexdigest() @@ -93,7 +94,7 @@ def sha256_digest(instr): Generate an sha256 hash of a given string. ''' if six.PY3: - b = salt.utils.to_bytes(instr) + b = salt.utils.stringutils.to_bytes(instr) return hashlib.sha256(b).hexdigest() return hashlib.sha256(instr).hexdigest() @@ -104,7 +105,7 @@ def sha512_digest(instr): Generate an sha512 hash of a given string ''' if six.PY3: - b = salt.utils.to_bytes(instr) + b = salt.utils.stringutils.to_bytes(instr) return hashlib.sha512(b).hexdigest() return hashlib.sha512(instr).hexdigest() @@ -116,9 +117,9 @@ def hmac_signature(string, shared_secret, challenge_hmac): Returns a boolean if the verification succeeded or failed. ''' if six.PY3: - msg = salt.utils.to_bytes(string) - key = salt.utils.to_bytes(shared_secret) - challenge = salt.utils.to_bytes(challenge_hmac) + msg = salt.utils.stringutils.to_bytes(string) + key = salt.utils.stringutils.to_bytes(shared_secret) + challenge = salt.utils.stringutils.to_bytes(challenge_hmac) else: msg = string key = shared_secret @@ -126,3 +127,15 @@ def hmac_signature(string, shared_secret, challenge_hmac): hmac_hash = hmac.new(key, msg, hashlib.sha256) valid_hmac = base64.b64encode(hmac_hash.digest()) return valid_hmac == challenge + + +@jinja_filter('rand_str') # Remove this for Neon +@jinja_filter('random_hash') +def random_hash(size=9999999999, hash_type=None): + ''' + Return a hash of a randomized data from random.SystemRandom() + ''' + if not hash_type: + hash_type = 'md5' + hasher = getattr(hashlib, hash_type) + return hasher(salt.utils.stringutils.to_bytes(str(random.SystemRandom().randint(0, size)))).hexdigest() diff --git a/salt/utils/http.py b/salt/utils/http.py index 935bfb3a7f2..f0b6f0c690a 100644 --- a/salt/utils/http.py +++ b/salt/utils/http.py @@ -15,6 +15,7 @@ import os.path import pprint import socket import yaml +import re import ssl try: @@ -35,19 +36,22 @@ except ImportError: HAS_MATCHHOSTNAME = False # Import salt libs -import salt.utils -import salt.utils.xmlutil as xml -import salt.utils.args -import salt.loader import salt.config +import salt.loader +import salt.syspaths +import salt.utils # Can be removed once refresh_dns is moved +import salt.utils.args +import salt.utils.files +import salt.utils.platform +import salt.utils.stringutils import salt.version +import salt.utils.xmlutil as xml from salt._compat import ElementTree as ET from salt.template import compile_template -from salt.utils.decorators import jinja_filter -from salt import syspaths +from salt.utils.decorators.jinja import jinja_filter # Import 3rd party libs -import salt.ext.six as six +from salt.ext import six # pylint: disable=import-error,no-name-in-module import salt.ext.six.moves.http_client import salt.ext.six.moves.http_cookiejar @@ -145,11 +149,11 @@ def query(url, if opts is None: if node == 'master': opts = salt.config.master_config( - os.path.join(syspaths.CONFIG_DIR, 'master') + os.path.join(salt.syspaths.CONFIG_DIR, 'master') ) elif node == 'minion': opts = salt.config.minion_config( - os.path.join(syspaths.CONFIG_DIR, 'minion') + os.path.join(salt.syspaths.CONFIG_DIR, 'minion') ) else: opts = {} @@ -157,6 +161,10 @@ def query(url, if not backend: backend = opts.get('backend', 'tornado') + match = re.match(r'https?://((25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(25[0-5]|2[0-4]\d|[01]?\d\d?)($|/)', url) + if not match: + salt.utils.refresh_dns() + if backend == 'requests': if HAS_REQUESTS is False: ret['error'] = ('http.query has been set to use requests, but the ' @@ -218,9 +226,9 @@ def query(url, header_list = [] if cookie_jar is None: - cookie_jar = os.path.join(opts.get('cachedir', syspaths.CACHE_DIR), 'cookies.txt') + cookie_jar = os.path.join(opts.get('cachedir', salt.syspaths.CACHE_DIR), 'cookies.txt') if session_cookie_jar is None: - session_cookie_jar = os.path.join(opts.get('cachedir', syspaths.CACHE_DIR), 'cookies.session.p') + session_cookie_jar = os.path.join(opts.get('cachedir', salt.syspaths.CACHE_DIR), 'cookies.session.p') if persist_session is True and HAS_MSGPACK: # TODO: This is hackish; it will overwrite the session cookie jar with @@ -228,12 +236,12 @@ def query(url, # proper cookie jar. Unfortunately, since session cookies do not # contain expirations, they can't be stored in a proper cookie jar. if os.path.isfile(session_cookie_jar): - with salt.utils.fopen(session_cookie_jar, 'rb') as fh_: + with salt.utils.files.fopen(session_cookie_jar, 'rb') as fh_: session_cookies = msgpack.load(fh_) if isinstance(session_cookies, dict): header_dict.update(session_cookies) else: - with salt.utils.fopen(session_cookie_jar, 'wb') as fh_: + with salt.utils.files.fopen(session_cookie_jar, 'wb') as fh_: msgpack.dump('', fh_) for header in header_list: @@ -456,8 +464,8 @@ def query(url, # We want to use curl_http if we have a proxy defined if proxy_host and proxy_port: if HAS_CURL_HTTPCLIENT is False: - ret['error'] = ('proxy_host and proxy_port has been set. This requires pycurl, but the ' - 'pycurl library does not seem to be installed') + ret['error'] = ('proxy_host and proxy_port has been set. This requires pycurl and tornado, ' + 'but the libraries does not seem to be installed') log.error(ret['error']) return ret @@ -544,11 +552,11 @@ def query(url, 'incompatibilities between requests and logging.').format(exc)) if text_out is not None: - with salt.utils.fopen(text_out, 'w') as tof: + with salt.utils.files.fopen(text_out, 'w') as tof: tof.write(result_text) if headers_out is not None and os.path.exists(headers_out): - with salt.utils.fopen(headers_out, 'w') as hof: + with salt.utils.files.fopen(headers_out, 'w') as hof: hof.write(result_headers) if cookies is not None: @@ -557,7 +565,7 @@ def query(url, if persist_session is True and HAS_MSGPACK: # TODO: See persist_session above if 'set-cookie' in result_headers: - with salt.utils.fopen(session_cookie_jar, 'wb') as fh_: + with salt.utils.files.fopen(session_cookie_jar, 'wb') as fh_: session_cookies = result_headers.get('set-cookie', None) if session_cookies is not None: msgpack.dump({'Cookie': session_cookies}, fh_) @@ -596,7 +604,7 @@ def query(url, return ret if decode_type == 'json': - ret['dict'] = json.loads(salt.utils.to_str(result_text)) + ret['dict'] = json.loads(salt.utils.stringutils.to_str(result_text)) elif decode_type == 'xml': ret['dict'] = [] items = ET.fromstring(result_text) @@ -608,7 +616,7 @@ def query(url, text = True if decode_out: - with salt.utils.fopen(decode_out, 'w') as dof: + with salt.utils.files.fopen(decode_out, 'w') as dof: dof.write(result_text) if text is True: @@ -633,7 +641,7 @@ def get_ca_bundle(opts=None): if opts_bundle is not None and os.path.exists(opts_bundle): return opts_bundle - file_roots = opts.get('file_roots', {'base': [syspaths.SRV_ROOT_DIR]}) + file_roots = opts.get('file_roots', {'base': [salt.syspaths.SRV_ROOT_DIR]}) # Please do not change the order without good reason @@ -661,7 +669,7 @@ def get_ca_bundle(opts=None): if os.path.exists(path): return path - if salt.utils.is_windows() and HAS_CERTIFI: + if salt.utils.platform.is_windows() and HAS_CERTIFI: return certifi.where() return None @@ -733,7 +741,7 @@ def update_ca_bundle( ) ) try: - with salt.utils.fopen(cert_file, 'r') as fcf: + with salt.utils.files.fopen(cert_file, 'r') as fcf: merge_content = '\n'.join((merge_content, fcf.read())) except IOError as exc: log.error( @@ -745,7 +753,7 @@ def update_ca_bundle( if merge_content: log.debug('Appending merge_files to {0}'.format(target)) try: - with salt.utils.fopen(target, 'a') as tfp: + with salt.utils.files.fopen(target, 'a') as tfp: tfp.write('\n') tfp.write(merge_content) except IOError as exc: @@ -769,11 +777,12 @@ def _render(template, render, renderer, template_dict, opts): blacklist = opts.get('renderer_blacklist') whitelist = opts.get('renderer_whitelist') ret = compile_template(template, rend, renderer, blacklist, whitelist, **template_dict) - ret = ret.read() + if salt.utils.stringio.is_readable(ret): + ret = ret.read() if str(ret).startswith('#!') and not str(ret).startswith('#!/'): ret = str(ret).split('\n', 1)[1] return ret - with salt.utils.fopen(template, 'r') as fh_: + with salt.utils.files.fopen(template, 'r') as fh_: return fh_.read() diff --git a/salt/utils/iam.py b/salt/utils/iam.py index a148fb4f04b..2e7986c69bb 100644 --- a/salt/utils/iam.py +++ b/salt/utils/iam.py @@ -11,7 +11,7 @@ import logging import time import pprint from salt.ext.six.moves import range -import salt.ext.six as six +from salt.ext import six try: import requests diff --git a/salt/utils/itertools.py b/salt/utils/itertools.py index 60bc853b344..7048b57821f 100644 --- a/salt/utils/itertools.py +++ b/salt/utils/itertools.py @@ -8,7 +8,7 @@ from __future__ import absolute_import import re # Import Salt libs -import salt.utils +import salt.utils.files def split(orig, sep=None): @@ -53,7 +53,7 @@ def read_file(fh_, chunk_size=1048576): chunk = fh_.read(chunk_size) except AttributeError: # Open the file and re-attempt the read - fh_ = salt.utils.fopen(fh_, 'rb') # pylint: disable=W8470 + fh_ = salt.utils.files.fopen(fh_, 'rb') # pylint: disable=W8470 chunk = fh_.read(chunk_size) if not chunk: break diff --git a/salt/utils/jid.py b/salt/utils/jid.py index 3f4ef296a26..b65293d8d50 100644 --- a/salt/utils/jid.py +++ b/salt/utils/jid.py @@ -9,12 +9,22 @@ import os from salt.ext import six +LAST_JID_DATETIME = None -def gen_jid(): + +def gen_jid(opts): ''' Generate a jid ''' - return '{0:%Y%m%d%H%M%S%f}'.format(datetime.datetime.now()) + global LAST_JID_DATETIME # pylint: disable=global-statement + + if not opts.get('unique_jid', False): + return '{0:%Y%m%d%H%M%S%f}'.format(datetime.datetime.now()) + jid_dt = datetime.datetime.now() + if LAST_JID_DATETIME and LAST_JID_DATETIME >= jid_dt: + jid_dt = LAST_JID_DATETIME + datetime.timedelta(microseconds=1) + LAST_JID_DATETIME = jid_dt + return '{0:%Y%m%d%H%M%S%f}_{1}'.format(jid_dt, os.getpid()) def is_jid(jid): @@ -23,10 +33,10 @@ def is_jid(jid): ''' if not isinstance(jid, six.string_types): return False - if len(jid) != 20: + if len(jid) != 20 and (len(jid) <= 21 or jid[20] != '_'): return False try: - int(jid) + int(jid[:20]) return True except ValueError: return False @@ -37,7 +47,7 @@ def jid_to_time(jid): Convert a salt job id into the time when the job was invoked ''' jid = str(jid) - if len(jid) != 20: + if len(jid) != 20 and (len(jid) <= 21 or jid[20] != '_'): return '' year = jid[:4] month = jid[4:6] @@ -45,7 +55,7 @@ def jid_to_time(jid): hour = jid[8:10] minute = jid[10:12] second = jid[12:14] - micro = jid[14:] + micro = jid[14:20] ret = '{0}, {1} {2} {3}:{4}:{5}.{6}'.format(year, months[int(month)], diff --git a/salt/utils/jinja.py b/salt/utils/jinja.py index 37bfb6a3ae2..38896e2b810 100644 --- a/salt/utils/jinja.py +++ b/salt/utils/jinja.py @@ -8,18 +8,18 @@ from __future__ import absolute_import import collections import json import logging +import os.path import pipes import pprint import re import uuid from functools import wraps -from os import path from xml.dom import minidom from xml.etree.ElementTree import Element, SubElement, tostring # Import third party libs import jinja2 -import salt.ext.six as six +from salt.ext import six import yaml from jinja2 import BaseLoader, Markup, TemplateNotFound, nodes from jinja2.environment import TemplateModule @@ -27,12 +27,12 @@ from jinja2.exceptions import TemplateRuntimeError from jinja2.ext import Extension # Import salt libs -import salt import salt.fileclient -import salt.utils +import salt.utils.files import salt.utils.url -from salt.utils.decorators import jinja_filter, jinja_test +from salt.utils.decorators.jinja import jinja_filter, jinja_test, jinja_global from salt.utils.odict import OrderedDict +from salt.exceptions import TemplateError log = logging.getLogger(__name__) @@ -75,7 +75,7 @@ class SaltCacheLoader(BaseLoader): else: self.searchpath = opts['file_roots'][saltenv] else: - self.searchpath = [path.join(opts['cachedir'], 'files', saltenv)] + self.searchpath = [os.path.join(opts['cachedir'], 'files', saltenv)] log.debug('Jinja search path: %s', self.searchpath) self._file_client = None self.cached = [] @@ -117,7 +117,7 @@ class SaltCacheLoader(BaseLoader): self.check_cache(template) if environment and template: - tpldir = path.dirname(template).replace('\\', '/') + tpldir = os.path.dirname(template).replace('\\', '/') tpldata = { 'tplfile': template, 'tpldir': '.' if tpldir == '' else tpldir, @@ -127,15 +127,15 @@ class SaltCacheLoader(BaseLoader): # pylint: disable=cell-var-from-loop for spath in self.searchpath: - filepath = path.join(spath, template) + filepath = os.path.join(spath, template) try: - with salt.utils.fopen(filepath, 'rb') as ifile: + with salt.utils.files.fopen(filepath, 'rb') as ifile: contents = ifile.read().decode(self.encoding) - mtime = path.getmtime(filepath) + mtime = os.path.getmtime(filepath) def uptodate(): try: - return path.getmtime(filepath) == mtime + return os.path.getmtime(filepath) == mtime except OSError: return False return contents, filepath, uptodate @@ -182,6 +182,12 @@ class PrintableDict(OrderedDict): return '{' + ', '.join(output) + '}' +# Additional globals +@jinja_global('raise') +def jinja_raise(msg): + raise TemplateError(msg) + + # Additional tests @jinja_test('match') def test_match(txt, rgx, ignorecase=False, multiline=False): diff --git a/salt/utils/job.py b/salt/utils/job.py index c37e034c32e..a10098019aa 100644 --- a/salt/utils/job.py +++ b/salt/utils/job.py @@ -18,7 +18,7 @@ def store_job(opts, load, event=None, mminion=None): Store job information using the configured master_job_cache ''' # Generate EndTime - endtime = salt.utils.jid.jid_to_time(salt.utils.jid.gen_jid()) + endtime = salt.utils.jid.jid_to_time(salt.utils.jid.gen_jid(opts)) # If the return data is invalid, just ignore it if any(key not in load for key in ('return', 'jid', 'id')): return False diff --git a/salt/utils/kickstart.py b/salt/utils/kickstart.py index 9de44850635..f8df726c086 100644 --- a/salt/utils/kickstart.py +++ b/salt/utils/kickstart.py @@ -8,7 +8,7 @@ from __future__ import absolute_import import yaml import shlex import argparse # pylint: disable=minimum-python-version -import salt.utils +import salt.utils.files from salt.ext.six.moves import range @@ -898,7 +898,7 @@ def mksls(src, dst=None): mode = 'command' sls = {} ks_opts = {} - with salt.utils.fopen(src, 'r') as fh_: + with salt.utils.files.fopen(src, 'r') as fh_: for line in fh_: if line.startswith('#'): continue @@ -1175,7 +1175,7 @@ def mksls(src, dst=None): sls[package] = {'pkg': ['absent']} if dst: - with salt.utils.fopen(dst, 'w') as fp_: + with salt.utils.files.fopen(dst, 'w') as fp_: fp_.write(yaml.safe_dump(sls, default_flow_style=False)) else: return yaml.safe_dump(sls, default_flow_style=False) diff --git a/salt/utils/locales.py b/salt/utils/locales.py index 6da3b3e26ae..c114a985aee 100644 --- a/salt/utils/locales.py +++ b/salt/utils/locales.py @@ -3,14 +3,17 @@ the locale utils used by salt ''' +# Import Python libs from __future__ import absolute_import - import sys -import salt.utils -import salt.ext.six as six +# Import Salt libs +import salt.utils.stringutils from salt.utils.decorators import memoize as real_memoize +# Import 3rd-party libs +from salt.ext import six + @real_memoize def get_encodings(): @@ -42,7 +45,7 @@ def sdecode(string_): encodings = get_encodings() for encoding in encodings: try: - decoded = salt.utils.to_unicode(string_, encoding) + decoded = salt.utils.stringutils.to_unicode(string_, encoding) if isinstance(decoded, six.string_types): # Make sure unicode string ops work u' ' + decoded # pylint: disable=W0104 @@ -115,17 +118,3 @@ def normalize_locale(loc): comps['codeset'] = comps['codeset'].lower().replace('-', '') comps['charmap'] = '' return join_locale(comps) - - -def decode_recursively(object_): - if isinstance(object_, list): - return [decode_recursively(o) for o in object_] - if isinstance(object_, tuple): - return tuple([decode_recursively(o) for o in object_]) - if isinstance(object_, dict): - return dict([(decode_recursively(key), decode_recursively(value)) - for key, value in salt.ext.six.iteritems(object_)]) - elif isinstance(object_, six.string_types): - return sdecode(object_) - else: - return object_ diff --git a/salt/utils/mac_utils.py b/salt/utils/mac_utils.py index 399afb6ba6c..baf6ca16c01 100644 --- a/salt/utils/mac_utils.py +++ b/salt/utils/mac_utils.py @@ -13,6 +13,9 @@ import time # Import Salt Libs import salt.utils +import salt.utils.args +import salt.utils.platform +import salt.utils.stringutils import salt.utils.timed_subprocess import salt.grains.extra from salt.ext import six @@ -34,7 +37,7 @@ def __virtual__(): ''' Load only on Mac OS ''' - if not salt.utils.is_darwin(): + if not salt.utils.platform.is_darwin(): return (False, 'The mac_utils utility could not be loaded: ' 'utility only works on MacOS systems.') @@ -51,7 +54,7 @@ def _run_all(cmd): ''' if not isinstance(cmd, list): - cmd = salt.utils.shlex_split(cmd, posix=False) + cmd = salt.utils.args.shlex_split(cmd, posix=False) for idx, item in enumerate(cmd): if not isinstance(cmd[idx], six.string_types): @@ -95,9 +98,9 @@ def _run_all(cmd): out, err = proc.stdout, proc.stderr if out is not None: - out = salt.utils.to_str(out).rstrip() + out = salt.utils.stringutils.to_str(out).rstrip() if err is not None: - err = salt.utils.to_str(err).rstrip() + err = salt.utils.stringutils.to_str(err).rstrip() ret['pid'] = proc.process.pid ret['retcode'] = proc.process.returncode @@ -190,7 +193,7 @@ def validate_enabled(enabled): :return: "on" or "off" or errors :rtype: str ''' - if isinstance(enabled, str): + if isinstance(enabled, six.string_types): if enabled.lower() not in ['on', 'off', 'yes', 'no']: msg = '\nMac Power: Invalid String Value for Enabled.\n' \ 'String values must be \'on\' or \'off\'/\'yes\' or \'no\'.\n' \ diff --git a/salt/utils/master.py b/salt/utils/master.py index c48137ce4ce..b57a81f93ac 100644 --- a/salt/utils/master.py +++ b/salt/utils/master.py @@ -22,6 +22,8 @@ import salt.pillar import salt.utils import salt.utils.atomicfile import salt.utils.minions +import salt.utils.verify +import salt.utils.versions import salt.payload from salt.exceptions import SaltException import salt.config @@ -29,7 +31,7 @@ from salt.utils.cache import CacheCli as cache_cli from salt.utils.process import MultiprocessingProcess # Import third party libs -import salt.ext.six as six +from salt.ext import six try: import zmq HAS_ZMQ = True @@ -78,7 +80,7 @@ class MasterPillarUtil(object): # remember to remove the expr_form argument from this function when # performing the cleanup on this deprecation. if expr_form is not None: - salt.utils.warn_until( + salt.utils.versions.warn_until( 'Fluorine', 'the target type should be passed using the \'tgt_type\' ' 'argument instead of \'expr_form\'. Support for using ' @@ -122,7 +124,7 @@ class MasterPillarUtil(object): 'and enfore_mine_cache are both disabled.') return mine_data if not minion_ids: - minion_ids = self.cache.ls('minions') + minion_ids = self.cache.list('minions') for minion_id in minion_ids: if not salt.utils.verify.valid_id(self.opts, minion_id): continue @@ -141,7 +143,7 @@ class MasterPillarUtil(object): 'enabled.') return grains, pillars if not minion_ids: - minion_ids = self.cache.ls('minions') + minion_ids = self.cache.list('minions') for minion_id in minion_ids: if not salt.utils.verify.valid_id(self.opts, minion_id): continue @@ -246,7 +248,8 @@ class MasterPillarUtil(object): # Return a list of minion ids that match the target and tgt_type minion_ids = [] ckminions = salt.utils.minions.CkMinions(self.opts) - minion_ids = ckminions.check_minions(self.tgt, self.tgt_type) + _res = ckminions.check_minions(self.tgt, self.tgt_type) + minion_ids = _res['minions'] if len(minion_ids) == 0: log.debug('No minions matched for tgt="{0}" and tgt_type="{1}"'.format(self.tgt, self.tgt_type)) return {} @@ -364,7 +367,7 @@ class MasterPillarUtil(object): # in the same file, 'data.p' grains, pillars = self._get_cached_minion_data(*minion_ids) try: - c_minions = self.cache.ls('minions') + c_minions = self.cache.list('minions') for minion_id in minion_ids: if not salt.utils.verify.valid_id(self.opts, minion_id): continue @@ -610,7 +613,7 @@ class ConnectedCache(MultiprocessingProcess): log.debug('ConCache Received request: {0}'.format(msg)) # requests to the minion list are send as str's - if isinstance(msg, str): + if isinstance(msg, six.string_types): if msg == 'minions': # Send reply back to client reply = serial.dumps(self.minions) @@ -642,7 +645,7 @@ class ConnectedCache(MultiprocessingProcess): data = new_c_data[0] - if isinstance(data, str): + if isinstance(data, six.string_types): if data not in self.minions: log.debug('ConCache Adding minion {0} to cache'.format(new_c_data[0])) self.minions.append(data) diff --git a/salt/utils/minion.py b/salt/utils/minion.py index def96c9ac1f..67ae2f09b18 100644 --- a/salt/utils/minion.py +++ b/salt/utils/minion.py @@ -10,11 +10,13 @@ import logging import threading # Import Salt Libs -import salt.utils import salt.payload +import salt.utils.files +import salt.utils.platform +import salt.utils.process # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six log = logging.getLogger(__name__) @@ -53,7 +55,7 @@ def cache_jobs(opts, jid, ret): jdir = os.path.dirname(fn_) if not os.path.isdir(jdir): os.makedirs(jdir) - with salt.utils.fopen(fn_, 'w+b') as fp_: + with salt.utils.files.fopen(fn_, 'w+b') as fp_: fp_.write(serial.dumps(ret)) @@ -64,7 +66,7 @@ def _read_proc_file(path, opts): serial = salt.payload.Serial(opts) current_thread = threading.currentThread().name pid = os.getpid() - with salt.utils.fopen(path, 'rb') as fp_: + with salt.utils.files.fopen(path, 'rb') as fp_: buf = fp_.read() fp_.close() if buf: @@ -129,7 +131,7 @@ def _check_cmdline(data): For non-Linux systems we punt and just return True ''' - if not salt.utils.is_linux(): + if not salt.utils.platform.is_linux(): return True pid = data.get('pid') if not pid: @@ -140,7 +142,7 @@ def _check_cmdline(data): if not os.path.isfile(path): return False try: - with salt.utils.fopen(path, 'rb') as fp_: + with salt.utils.files.fopen(path, 'rb') as fp_: if six.b('salt') in fp_.read(): return True except (OSError, IOError): diff --git a/salt/utils/minions.py b/salt/utils/minions.py index f4de4e92d99..0a49f2c24d4 100644 --- a/salt/utils/minions.py +++ b/salt/utils/minions.py @@ -14,14 +14,16 @@ import logging # Import salt libs import salt.payload import salt.utils +import salt.utils.files +import salt.utils.network +import salt.utils.versions from salt.defaults import DEFAULT_TARGET_DELIM from salt.exceptions import CommandExecutionError, SaltCacheError import salt.auth.ldap import salt.cache -import salt.ext.six as six +from salt.ext import six # Import 3rd-party libs -import salt.ext.six as six if six.PY3: import ipaddress else: @@ -75,7 +77,7 @@ def get_minion_data(minion, opts): if opts.get('minion_data_cache', False): cache = salt.cache.factory(opts) if minion is None: - for id_ in cache.ls('minions'): + for id_ in cache.list('minions'): data = cache.fetch('minions/{0}'.format(id_), 'data') if data is None: continue @@ -199,7 +201,8 @@ class CkMinions(object): ''' Return the minions found by looking via globs ''' - return fnmatch.filter(self._pki_minions(), expr) + return {'minions': fnmatch.filter(self._pki_minions(), expr), + 'missing': []} def _check_list_minions(self, expr, greedy): # pylint: disable=unused-argument ''' @@ -208,14 +211,16 @@ class CkMinions(object): if isinstance(expr, six.string_types): expr = [m for m in expr.split(',') if m] minions = self._pki_minions() - return [x for x in expr if x in minions] + return {'minions': [x for x in expr if x in minions], + 'missing': [x for x in expr if x not in minions]} def _check_pcre_minions(self, expr, greedy): # pylint: disable=unused-argument ''' Return the minions found by looking via regular expressions ''' reg = re.compile(expr) - return [m for m in self._pki_minions() if reg.match(m)] + return {'minions': [m for m in self._pki_minions() if reg.match(m)], + 'missing': []} def _pki_minions(self): ''' @@ -227,7 +232,7 @@ class CkMinions(object): try: if self.opts['key_cache'] and os.path.exists(pki_cache_fn): log.debug('Returning cached minion list') - with salt.utils.fopen(pki_cache_fn) as fn_: + with salt.utils.files.fopen(pki_cache_fn) as fn_: return self.serial.load(fn_) else: for fn_ in salt.utils.isorted(os.listdir(os.path.join(self.opts['pki_dir'], self.acc))): @@ -263,7 +268,8 @@ class CkMinions(object): elif cache_enabled: minions = list_cached_minions() else: - return [] + return {'minions': [], + 'missing': []} if cache_enabled: if greedy: @@ -271,7 +277,8 @@ class CkMinions(object): else: cminions = minions if not cminions: - return minions + return {'minions': minions, + 'missing': []} minions = set(minions) for id_ in cminions: if greedy and id_ not in minions: @@ -289,7 +296,8 @@ class CkMinions(object): exact_match=exact_match): minions.remove(id_) minions = list(minions) - return minions + return {'minions': minions, + 'missing': []} def _check_grain_minions(self, expr, delimiter, greedy): ''' @@ -342,17 +350,19 @@ class CkMinions(object): if greedy: minions = self._pki_minions() elif cache_enabled: - minions = self.cache.ls('minions') + minions = self.cache.list('minions') else: - return [] + return {'minions': [], + 'missing': []} if cache_enabled: if greedy: - cminions = self.cache.ls('minions') + cminions = self.cache.list('minions') else: cminions = minions if cminions is None: - return minions + return {'minions': minions, + 'missing': []} tgt = expr try: @@ -364,7 +374,8 @@ class CkMinions(object): tgt = ipaddress.ip_network(tgt) except: # pylint: disable=bare-except log.error('Invalid IP/CIDR target: {0}'.format(tgt)) - return [] + return {'minions': [], + 'missing': []} proto = 'ipv{0}'.format(tgt.version) minions = set(minions) @@ -385,7 +396,8 @@ class CkMinions(object): if not match and id_ in minions: minions.remove(id_) - return list(minions) + return {'minions': list(minions), + 'missing': []} def _check_range_minions(self, expr, greedy): ''' @@ -410,11 +422,14 @@ class CkMinions(object): for fn_ in salt.utils.isorted(os.listdir(os.path.join(self.opts['pki_dir'], self.acc))): if not fn_.startswith('.') and os.path.isfile(os.path.join(self.opts['pki_dir'], self.acc, fn_)): mlist.append(fn_) - return mlist + return {'minions': mlist, + 'missing': []} elif cache_enabled: - return self.cache.ls('minions') + return {'minions': self.cache.list('minions'), + 'missing': []} else: - return list() + return {'minions': [], + 'missing': []} def _check_compound_pillar_exact_minions(self, expr, delimiter, greedy): ''' @@ -435,10 +450,9 @@ class CkMinions(object): ''' Return the minions found by looking via compound matcher ''' - log.debug('_check_compound_minions({0}, {1}, {2}, {3})'.format(expr, delimiter, greedy, pillar_exact)) if not isinstance(expr, six.string_types) and not isinstance(expr, (list, tuple)): log.error('Compound target that is neither string, list nor tuple') - return [] + return {'minions': [], 'missing': []} minions = set(self._pki_minions()) log.debug('minions: {0}'.format(minions)) @@ -459,6 +473,7 @@ class CkMinions(object): results = [] unmatched = [] opers = ['and', 'or', 'not', '(', ')'] + missing = [] if isinstance(expr, six.string_types): words = expr.split() @@ -473,7 +488,7 @@ class CkMinions(object): if results: if results[-1] == '(' and word in ('and', 'or'): log.error('Invalid beginning operator after "(": {0}'.format(word)) - return [] + return {'minions': [], 'missing': []} if word == 'not': if not results[-1] in ('&', '|', '('): results.append('&') @@ -493,7 +508,7 @@ class CkMinions(object): log.error('Invalid compound expr (unexpected ' 'right parenthesis): {0}' .format(expr)) - return [] + return {'minions': [], 'missing': []} results.append(word) unmatched.pop() if unmatched and unmatched[-1] == '-': @@ -502,7 +517,7 @@ class CkMinions(object): else: # Won't get here, unless oper is added log.error('Unhandled oper in compound expr: {0}' .format(expr)) - return [] + return {'minions': [], 'missing': []} else: # seq start with oper, fail if word == 'not': @@ -518,13 +533,13 @@ class CkMinions(object): 'Expression may begin with' ' binary operator: {0}'.format(word) ) - return [] + return {'minions': [], 'missing': []} elif target_info and target_info['engine']: if 'N' == target_info['engine']: # Nodegroups should already be expanded/resolved to other engines log.error('Detected nodegroup expansion failure of "{0}"'.format(word)) - return [] + return {'minions': [], 'missing': []} engine = ref.get(target_info['engine']) if not engine: # If an unknown engine is called at any time, fail out @@ -535,21 +550,24 @@ class CkMinions(object): word, ) ) - return [] + return {'minions': [], 'missing': []} engine_args = [target_info['pattern']] if target_info['engine'] in ('G', 'P', 'I', 'J'): engine_args.append(target_info['delimiter'] or ':') engine_args.append(greedy) - results.append(str(set(engine(*engine_args)))) + _results = engine(*engine_args) + results.append(str(set(_results['minions']))) + missing.extend(_results['missing']) if unmatched and unmatched[-1] == '-': results.append(')') unmatched.pop() else: # The match is not explicitly defined, evaluate as a glob - results.append(str(set(self._check_glob_minions(word, True)))) + _results = self._check_glob_minions(word, True) + results.append(str(set(_results['minions']))) if unmatched and unmatched[-1] == '-': results.append(')') unmatched.pop() @@ -561,12 +579,14 @@ class CkMinions(object): log.debug('Evaluating final compound matching expr: {0}' .format(results)) try: - return list(eval(results)) # pylint: disable=W0123 + minions = list(eval(results)) # pylint: disable=W0123 + return {'minions': minions, 'missing': missing} except Exception: log.error('Invalid compound target: {0}'.format(expr)) - return [] + return {'minions': [], 'missing': []} - return list(minions) + return {'minions': list(minions), + 'missing': []} def connected_ids(self, subset=None, show_ipv4=False, include_localhost=False): ''' @@ -574,7 +594,7 @@ class CkMinions(object): ''' minions = set() if self.opts.get('minion_data_cache', False): - search = self.cache.ls('minions') + search = self.cache.list('minions') if search is None: return minions addrs = salt.utils.network.local_port_tcp(int(self.opts['publish_port'])) @@ -618,7 +638,7 @@ class CkMinions(object): for fn_ in salt.utils.isorted(os.listdir(os.path.join(self.opts['pki_dir'], self.acc))): if not fn_.startswith('.') and os.path.isfile(os.path.join(self.opts['pki_dir'], self.acc, fn_)): mlist.append(fn_) - return mlist + return {'minions': mlist, 'missing': []} def check_minions(self, expr, @@ -631,6 +651,7 @@ class CkMinions(object): match the regex, this will then be used to parse the returns to make sure everyone has checked back in. ''' + try: if expr is None: expr = '' @@ -642,15 +663,15 @@ class CkMinions(object): 'pillar_exact', 'compound', 'compound_pillar_exact'): - minions = check_func(expr, delimiter, greedy) + _res = check_func(expr, delimiter, greedy) else: - minions = check_func(expr, greedy) + _res = check_func(expr, greedy) except Exception: log.exception( 'Failed matching available minions with {0} pattern: {1}' .format(tgt_type, expr)) - minions = [] - return minions + _res = {'minions': [], 'missing': []} + return _res def _expand_matching(self, auth_entry): ref = {'G': 'grain', @@ -670,7 +691,8 @@ class CkMinions(object): v_matcher = ref.get(target_info['engine']) v_expr = target_info['pattern'] - return set(self.check_minions(v_expr, v_matcher)) + _res = self.check_minions(v_expr, v_matcher) + return set(_res['minions']) def validate_tgt(self, valid, expr, tgt_type, minions=None, expr_form=None): ''' @@ -680,7 +702,7 @@ class CkMinions(object): # remember to remove the expr_form argument from this function when # performing the cleanup on this deprecation. if expr_form is not None: - salt.utils.warn_until( + salt.utils.versions.warn_until( 'Fluorine', 'the target type should be passed using the \'tgt_type\' ' 'argument instead of \'expr_form\'. Support for using ' @@ -690,7 +712,8 @@ class CkMinions(object): v_minions = self._expand_matching(valid) if minions is None: - minions = set(self.check_minions(expr, tgt_type)) + _res = self.check_minions(expr, tgt_type) + minions = set(_res['minions']) else: minions = set(minions) d_bool = not bool(minions.difference(v_minions)) @@ -705,7 +728,7 @@ class CkMinions(object): functions ''' vals = [] - if isinstance(fun, str): + if isinstance(fun, six.string_types): fun = [fun] for func in fun: try: @@ -721,6 +744,13 @@ class CkMinions(object): ''' Read in the form and determine which auth check routine to execute ''' + # This function is only called from salt.auth.Authorize(), which is also + # deprecated and will be removed in Neon. + salt.utils.versions.warn_until( + 'Neon', + 'The \'any_auth\' function has been deprecated. Support for this ' + 'function will be removed in Salt {version}.' + ) if form == 'publish': return self.auth_check( auth_list, @@ -731,6 +761,7 @@ class CkMinions(object): return self.spec_check( auth_list, fun, + arg, form) def auth_check_expanded(self, @@ -779,8 +810,12 @@ class CkMinions(object): v_tgt_type = 'pillar_exact' elif tgt_type.lower() == 'compound': v_tgt_type = 'compound_pillar_exact' - v_minions = set(self.check_minions(tgt, v_tgt_type)) - minions = set(self.check_minions(tgt, tgt_type)) + _res = self.check_minions(tgt, v_tgt_type) + v_minions = set(_res['minions']) + + _res = self.check_minions(tgt, tgt_type) + minions = set(_res['minions']) + mismatch = bool(minions.difference(v_minions)) # If the non-exact match gets more minions than the exact match # then pillar globbing or PCRE is being used, and we have a @@ -867,8 +902,12 @@ class CkMinions(object): v_tgt_type = 'pillar_exact' elif tgt_type.lower() == 'compound': v_tgt_type = 'compound_pillar_exact' - v_minions = set(self.check_minions(tgt, v_tgt_type)) - minions = set(self.check_minions(tgt, tgt_type)) + _res = self.check_minions(tgt, v_tgt_type) + v_minions = set(_res['minions']) + + _res = self.check_minions(tgt, tgt_type) + minions = set(_res['minions']) + mismatch = bool(minions.difference(v_minions)) # If the non-exact match gets more minions than the exact match # then pillar globbing or PCRE is being used, and we have a @@ -900,73 +939,15 @@ class CkMinions(object): tgt_type, minions=minions): # Minions are allowed, verify function in allowed list - if isinstance(ind[valid], six.string_types): - if self.match_check(ind[valid], fun): - return True - elif isinstance(ind[valid], list): - for cond in ind[valid]: - # Function name match - if isinstance(cond, six.string_types): - if self.match_check(cond, fun): - return True - # Function and args match - elif isinstance(cond, dict): - if len(cond) != 1: - # Invalid argument - continue - fcond = next(six.iterkeys(cond)) - # cond: { - # 'mod.func': { - # 'args': [ - # 'one.*', 'two\\|three'], - # 'kwargs': { - # 'functioin': 'teach\\|feed', - # 'user': 'mother\\|father' - # } - # } - # } - if self.match_check(fcond, - fun): # check key that is function name match - acond = cond[fcond] - if not isinstance(acond, dict): - # Invalid argument - continue - # whitelist args, kwargs - arg_list = args[num] - cond_args = acond.get('args', []) - good = True - for i, cond_arg in enumerate(cond_args): - if len(arg_list) <= i: - good = False - break - if cond_arg is None: # None == '.*' i.e. allow any - continue - if not self.match_check(cond_arg, - arg_list[i]): - good = False - break - if not good: - continue - # Check kwargs - cond_kwargs = acond.get('kwargs', {}) - arg_kwargs = {} - for a in arg_list: - if isinstance(a, - dict) and '__kwarg__' in a: - arg_kwargs = a - break - for k, v in six.iteritems(cond_kwargs): - if k not in arg_kwargs: - good = False - break - if v is None: # None == '.*' i.e. allow any - continue - if not self.match_check(v, - arg_kwargs[k]): - good = False - break - if good: - return True + fun_args = args[num] + fun_kwargs = fun_args[-1] if fun_args else None + if isinstance(fun_kwargs, dict) and '__kwarg__' in fun_kwargs: + fun_args = list(fun_args) # copy on modify + del fun_args[-1] + else: + fun_kwargs = None + if self.__fun_check(ind[valid], fun, fun_args, fun_kwargs): + return True except TypeError: return False return False @@ -985,100 +966,111 @@ class CkMinions(object): auth_list.append(matcher) return auth_list - def wheel_check(self, auth_list, fun): + def wheel_check(self, auth_list, fun, args): ''' Check special API permissions ''' - comps = fun.split('.') - if len(comps) != 2: - return False - mod = comps[0] - fun = comps[1] - for ind in auth_list: - if isinstance(ind, six.string_types): - if ind.startswith('@') and ind[1:] == mod: - return True - if ind == '@wheel': - return True - if ind == '@wheels': - return True - elif isinstance(ind, dict): - if len(ind) != 1: - continue - valid = next(six.iterkeys(ind)) - if valid.startswith('@') and valid[1:] == mod: - if isinstance(ind[valid], six.string_types): - if self.match_check(ind[valid], fun): - return True - elif isinstance(ind[valid], list): - for regex in ind[valid]: - if self.match_check(regex, fun): - return True - return False + return self.spec_check(auth_list, fun, args, 'wheel') - def runner_check(self, auth_list, fun): + def runner_check(self, auth_list, fun, args): ''' Check special API permissions ''' - comps = fun.split('.') - if len(comps) != 2: - return False - mod = comps[0] - fun = comps[1] - for ind in auth_list: - if isinstance(ind, six.string_types): - if ind.startswith('@') and ind[1:] == mod: - return True - if ind == '@runners': - return True - if ind == '@runner': - return True - elif isinstance(ind, dict): - if len(ind) != 1: - continue - valid = next(six.iterkeys(ind)) - if valid.startswith('@') and valid[1:] == mod: - if isinstance(ind[valid], six.string_types): - if self.match_check(ind[valid], fun): - return True - elif isinstance(ind[valid], list): - for regex in ind[valid]: - if self.match_check(regex, fun): - return True - return False + return self.spec_check(auth_list, fun, args, 'runner') - def spec_check(self, auth_list, fun, form): + def spec_check(self, auth_list, fun, args, form): ''' Check special API permissions ''' if form != 'cloud': comps = fun.split('.') if len(comps) != 2: - return False - mod = comps[0] - fun = comps[1] + # Hint at a syntax error when command is passed improperly, + # rather than returning an authentication error of some kind. + # See Issue #21969 for more information. + return {'error': {'name': 'SaltInvocationError', + 'message': 'A command invocation error occurred: Check syntax.'}} + mod_name = comps[0] + fun_name = comps[1] else: - mod = fun + fun_name = mod_name = fun for ind in auth_list: if isinstance(ind, six.string_types): - if ind.startswith('@') and ind[1:] == mod: - return True - if ind == '@{0}'.format(form): - return True - if ind == '@{0}s'.format(form): - return True + if ind[0] == '@': + if ind[1:] == mod_name or ind[1:] == form or ind == '@{0}s'.format(form): + return True elif isinstance(ind, dict): if len(ind) != 1: continue valid = next(six.iterkeys(ind)) - if valid.startswith('@') and valid[1:] == mod: - if isinstance(ind[valid], six.string_types): - if self.match_check(ind[valid], fun): + if valid[0] == '@': + if valid[1:] == mod_name: + if self.__fun_check(ind[valid], fun_name, args.get('arg'), args.get('kwarg')): return True - elif isinstance(ind[valid], list): - for regex in ind[valid]: - if self.match_check(regex, fun): - return True + if valid[1:] == form or valid == '@{0}s'.format(form): + if self.__fun_check(ind[valid], fun, args.get('arg'), args.get('kwarg')): + return True + return False + + def __fun_check(self, valid, fun, args=None, kwargs=None): + ''' + Check the given function name (fun) and its arguments (args) against the list of conditions. + ''' + if not isinstance(valid, list): + valid = [valid] + for cond in valid: + # Function name match + if isinstance(cond, six.string_types): + if self.match_check(cond, fun): + return True + # Function and args match + elif isinstance(cond, dict): + if len(cond) != 1: + # Invalid argument + continue + fname_cond = next(six.iterkeys(cond)) + if self.match_check(fname_cond, fun): # check key that is function name match + if self.__args_check(cond[fname_cond], args, kwargs): + return True + return False + + def __args_check(self, valid, args=None, kwargs=None): + ''' + valid is a dicts: {'args': [...], 'kwargs': {...}} or a list of such dicts. + ''' + if not isinstance(valid, list): + valid = [valid] + for cond in valid: + if not isinstance(cond, dict): + # Invalid argument + continue + # whitelist args, kwargs + cond_args = cond.get('args', []) + good = True + for i, cond_arg in enumerate(cond_args): + if args is None or len(args) <= i: + good = False + break + if cond_arg is None: # None == '.*' i.e. allow any + continue + if not self.match_check(cond_arg, str(args[i])): + good = False + break + if not good: + continue + # Check kwargs + cond_kwargs = cond.get('kwargs', {}) + for k, v in six.iteritems(cond_kwargs): + if kwargs is None or k not in kwargs: + good = False + break + if v is None: # None == '.*' i.e. allow any + continue + if not self.match_check(v, str(kwargs[k])): + good = False + break + if good: + return True return False @@ -1090,9 +1082,10 @@ def mine_get(tgt, fun, tgt_type='glob', opts=None): ret = {} serial = salt.payload.Serial(opts) checker = CkMinions(opts) - minions = checker.check_minions( + _res = checker.check_minions( tgt, tgt_type) + minions = _res['minions'] cache = salt.cache.factory(opts) for minion in minions: mdata = cache.fetch('minions/{0}'.format(minion), 'mine') diff --git a/salt/utils/mount.py b/salt/utils/mount.py new file mode 100644 index 00000000000..09dbc8bfc19 --- /dev/null +++ b/salt/utils/mount.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- +''' +Common functions for managing mounts +''' + +# Import python libs +from __future__ import absolute_import +import logging +import os +import yaml + +# Import Salt libs +import salt.utils.files +import salt.utils.stringutils +import salt.utils.versions + +from salt.utils.yamldumper import SafeOrderedDumper + +log = logging.getLogger(__name__) + + +def _read_file(path): + ''' + Reads and returns the contents of a text file + ''' + try: + with salt.utils.files.fopen(path, 'rb') as contents: + return yaml.safe_load(contents.read()) + except (OSError, IOError): + return {} + + +def get_cache(opts): + ''' + Return the mount cache file location. + ''' + return os.path.join(opts['cachedir'], 'mounts') + + +def read_cache(opts): + ''' + Write the mount cache file. + ''' + cache_file = get_cache(opts) + return _read_file(cache_file) + + +def write_cache(cache, opts): + ''' + Write the mount cache file. + ''' + cache_file = get_cache(opts) + + try: + _cache = salt.utils.stringutils.to_bytes( + yaml.dump( + cache, + Dumper=SafeOrderedDumper + ) + ) + with salt.utils.files.fopen(cache_file, 'wb+') as fp_: + fp_.write(_cache) + return True + except (IOError, OSError): + log.error('Failed to cache mounts', + exc_info_on_loglevel=logging.DEBUG) + return False diff --git a/salt/utils/msazure.py b/salt/utils/msazure.py index 7488f3077c7..ec72617331b 100644 --- a/salt/utils/msazure.py +++ b/salt/utils/msazure.py @@ -19,7 +19,7 @@ except ImportError: pass # Import salt libs -import salt.ext.six as six +from salt.ext import six from salt.exceptions import SaltSystemExit log = logging.getLogger(__name__) diff --git a/salt/utils/napalm.py b/salt/utils/napalm.py index c69a92432e3..0a8922136ff 100644 --- a/salt/utils/napalm.py +++ b/salt/utils/napalm.py @@ -14,17 +14,19 @@ Utils for the NAPALM modules and proxy. .. versionadded:: 2017.7.0 ''' - +# Import Python libs from __future__ import absolute_import - import traceback import logging +import importlib from functools import wraps -log = logging.getLogger(__file__) -import salt.utils +# Import Salt libs +import salt.output +import salt.utils.platform -# Import third party lib +# Import 3rd-party libs +from salt.ext import six try: # will try to import NAPALM # https://github.com/napalm-automation/napalm @@ -44,14 +46,14 @@ try: except ImportError: HAS_CONN_CLOSED_EXC_CLASS = False -from salt.ext import six as six +log = logging.getLogger(__file__) def is_proxy(opts): ''' Is this a NAPALM proxy? ''' - return salt.utils.is_proxy() and opts.get('proxy', {}).get('proxytype') == 'napalm' + return salt.utils.platform.is_proxy() and opts.get('proxy', {}).get('proxytype') == 'napalm' def is_always_alive(opts): @@ -72,7 +74,7 @@ def is_minion(opts): ''' Is this a NAPALM straight minion? ''' - return not salt.utils.is_proxy() and 'napalm' in opts + return not salt.utils.platform.is_proxy() and 'napalm' in opts def virtual(opts, virtualname, filename): @@ -242,6 +244,11 @@ def get_device_opts(opts, salt_obj=None): network_device = {} # by default, look in the proxy config details device_dict = opts.get('proxy', {}) or opts.get('napalm', {}) + if opts.get('proxy') or opts.get('napalm'): + opts['multiprocessing'] = device_dict.get('multiprocessing', False) + # Most NAPALM drivers are SSH-based, so multiprocessing should default to False. + # But the user can be allows to have a different value for the multiprocessing, which will + # override the opts. if salt_obj and not device_dict: # get the connection details from the opts device_dict = salt_obj['config.merge']('napalm') @@ -264,6 +271,7 @@ def get_device_opts(opts, salt_obj=None): network_device['TIMEOUT'] = device_dict.get('timeout', 60) network_device['OPTIONAL_ARGS'] = device_dict.get('optional_args', {}) network_device['ALWAYS_ALIVE'] = device_dict.get('always_alive', True) + network_device['PROVIDER'] = device_dict.get('provider') network_device['UP'] = False # get driver object form NAPALM if 'config_lock' not in network_device['OPTIONAL_ARGS']: @@ -281,7 +289,24 @@ def get_device(opts, salt_obj=None): ''' log.debug('Setting up NAPALM connection') network_device = get_device_opts(opts, salt_obj=salt_obj) - _driver_ = napalm_base.get_network_driver(network_device.get('DRIVER_NAME')) + provider_lib = napalm_base + if network_device.get('PROVIDER'): + # In case the user requires a different provider library, + # other than napalm-base. + # For example, if napalm-base does not satisfy the requirements + # and needs to be enahanced with more specific features, + # we may need to define a custom library on top of napalm-base + # with the constraint that it still needs to provide the + # `get_network_driver` function. However, even this can be + # extended later, if really needed. + # Configuration example: + # provider: napalm_base_example + try: + provider_lib = importlib.import_module(network_device.get('PROVIDER')) + except ImportError as ierr: + log.error('Unable to import {0}'.format(network_device.get('PROVIDER')), exc_info=True) + log.error('Falling back to napalm-base') + _driver_ = provider_lib.get_network_driver(network_device.get('DRIVER_NAME')) try: network_device['DRIVER'] = _driver_( network_device.get('HOSTNAME', ''), @@ -330,11 +355,11 @@ def proxy_napalm_wrap(func): # the execution modules will make use of this variable from now on # previously they were accessing the device properties through the __proxy__ object always_alive = opts.get('proxy', {}).get('always_alive', True) - if salt.utils.is_proxy() and always_alive: + if salt.utils.platform.is_proxy() and always_alive: # if it is running in a proxy and it's using the default always alive behaviour, # will get the cached copy of the network device wrapped_global_namespace['napalm_device'] = proxy['napalm.get_device']() - elif salt.utils.is_proxy() and not always_alive: + elif salt.utils.platform.is_proxy() and not always_alive: # if still proxy, but the user does not want the SSH session always alive # get a new device instance # which establishes a new connection @@ -408,58 +433,58 @@ def default_ret(name): return ret -def loaded_ret(ret, loaded, test, debug): +def loaded_ret(ret, loaded, test, debug, compliance_report=False, opts=None): ''' Return the final state output. - ret The initial state output structure. - loaded The loaded dictionary. ''' # Always get the comment - ret.update({ - 'comment': loaded.get('comment', '') - }) + changes = {} pchanges = {} + ret['comment'] = loaded['comment'] + if 'diff' in loaded: + changes['diff'] = loaded['diff'] + pchanges['diff'] = loaded['diff'] + if 'compliance_report' in loaded: + if compliance_report: + changes['compliance_report'] = loaded['compliance_report'] + pchanges['compliance_report'] = loaded['compliance_report'] + if debug and 'loaded_config' in loaded: + changes['loaded_config'] = loaded['loaded_config'] + pchanges['loaded_config'] = loaded['loaded_config'] + ret['pchanges'] = pchanges + if changes.get('diff'): + ret['comment'] = '{comment_base}\n\nConfiguration diff:\n\n{diff}'.format(comment_base=ret['comment'], + diff=changes['diff']) + if changes.get('loaded_config'): + ret['comment'] = '{comment_base}\n\nLoaded config:\n\n{loaded_cfg}'.format( + comment_base=ret['comment'], + loaded_cfg=changes['loaded_config']) + if changes.get('compliance_report'): + ret['comment'] = '{comment_base}\n\nCompliance report:\n\n{compliance}'.format( + comment_base=ret['comment'], + compliance=salt.output.string_format(changes['compliance_report'], 'nested', opts=opts)) if not loaded.get('result', False): # Failure of some sort return ret - if debug: - # Always check for debug - pchanges.update({ - 'loaded_config': loaded.get('loaded_config', '') - }) - ret.update({ - "pchanges": pchanges - }) if not loaded.get('already_configured', True): # We're making changes - pchanges.update({ - "diff": loaded.get('diff', '') - }) - ret.update({ - 'pchanges': pchanges - }) if test: - for k, v in pchanges.items(): - ret.update({ - "comment": "{}:\n{}\n\n{}".format(k, v, ret.get("comment", '')) - }) - ret.update({ - 'result': None, - }) + ret['result'] = None return ret # Not test, changes were applied ret.update({ 'result': True, - 'changes': pchanges, - 'comment': "Configuration changed!\n{}".format(ret.get('comment', '')) + 'changes': changes, + 'comment': "Configuration changed!\n{}".format(loaded['comment']) }) return ret # No changes ret.update({ - 'result': True + 'result': True, + 'changes': {} }) return ret diff --git a/salt/utils/network.py b/salt/utils/network.py index 9cc80abe15b..119aeed25ab 100644 --- a/salt/utils/network.py +++ b/salt/utils/network.py @@ -16,7 +16,7 @@ import subprocess from string import ascii_letters, digits # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import range # pylint: disable=import-error,redefined-builtin # Attempt to import wmi try: @@ -26,12 +26,16 @@ except ImportError: pass # Import salt libs -import salt.utils +import salt.utils.args +import salt.utils.files +import salt.utils.path +import salt.utils.platform +import salt.utils.stringutils from salt._compat import ipaddress -from salt.utils.decorators import jinja_filter +from salt.utils.decorators.jinja import jinja_filter # inet_pton does not exist in Windows, this is a workaround -if salt.utils.is_windows(): +if salt.utils.platform.is_windows(): from salt.ext import win_inet_pton # pylint: disable=unused-import log = logging.getLogger(__name__) @@ -137,13 +141,13 @@ def _generate_minion_id(): r'{win}\system32\drivers\etc\hosts'.format(win=os.getenv('WINDIR'))]: if not os.path.exists(f_name): continue - with salt.utils.fopen(f_name) as f_hdl: + with salt.utils.files.fopen(f_name) as f_hdl: for hst in (line.strip().split('#')[0].strip().split() or None for line in f_hdl.read().split(os.linesep)): if hst and (hst[0][:4] in ['127.', '::1'] or len(hst) == 1): hosts.extend(hst) # include public and private ipaddresses - return hosts.extend([addr for addr in salt.utils.network.ip_addrs() + return hosts.extend([addr for addr in ip_addrs() if not ipaddress.ip_address(addr).is_loopback]) @@ -206,6 +210,21 @@ def ip_to_host(ip): # pylint: enable=C0103 +def is_reachable_host(entity_name): + ''' + Returns a bool telling if the entity name is a reachable host (IPv4/IPv6/FQDN/etc). + :param hostname: + :return: + ''' + try: + assert type(socket.getaddrinfo(entity_name, 0, 0, 0, 0)) == list + ret = True + except socket.gaierror: + ret = False + + return ret + + def is_ip(ip): ''' Returns a bool telling if the passed IP is a valid IPv4 or IPv6 address. @@ -462,6 +481,11 @@ def _network_hosts(ip_addr_entry): def network_hosts(value, options=None, version=None): ''' Return the list of hosts within a network. + + .. note:: + + When running this command with a large IPv6 network, the command will + take a long time to gather all of the hosts. ''' ipaddr_filter_out = _filter_ipaddr(value, options=options, version=version) if not ipaddr_filter_out: @@ -675,7 +699,7 @@ def _interfaces_ifconfig(out): piface = re.compile(r'^([^\s:]+)') pmac = re.compile('.*?(?:HWaddr|ether|address:|lladdr) ([0-9a-fA-F:]+)') - if salt.utils.is_sunos(): + if salt.utils.platform.is_sunos(): pip = re.compile(r'.*?(?:inet\s+)([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)(.*)') pip6 = re.compile('.*?(?:inet6 )([0-9a-fA-F:]+)') pmask6 = re.compile(r'.*?(?:inet6 [0-9a-fA-F:]+/(\d+)).*') @@ -702,7 +726,7 @@ def _interfaces_ifconfig(out): iface = miface.group(1) if mmac: data['hwaddr'] = mmac.group(1) - if salt.utils.is_sunos(): + if salt.utils.platform.is_sunos(): expand_mac = [] for chunk in data['hwaddr'].split(':'): expand_mac.append('0{0}'.format(chunk) if len(chunk) < 2 else '{0}'.format(chunk)) @@ -734,11 +758,11 @@ def _interfaces_ifconfig(out): mmask6 = pmask6.match(line) if mmask6: addr_obj['prefixlen'] = mmask6.group(1) or mmask6.group(2) - if not salt.utils.is_sunos(): + if not salt.utils.platform.is_sunos(): ipv6scope = mmask6.group(3) or mmask6.group(4) addr_obj['scope'] = ipv6scope.lower() if ipv6scope is not None else ipv6scope # SunOS sometimes has ::/0 as inet6 addr when using addrconf - if not salt.utils.is_sunos() \ + if not salt.utils.platform.is_sunos() \ or addr_obj['address'] != '::' \ and addr_obj['prefixlen'] != 0: data['inet6'].append(addr_obj) @@ -767,8 +791,8 @@ def linux_interfaces(): Obtain interface information for *NIX/BSD variants ''' ifaces = dict() - ip_path = salt.utils.which('ip') - ifconfig_path = None if ip_path else salt.utils.which('ifconfig') + ip_path = salt.utils.path.which('ip') + ifconfig_path = None if ip_path else salt.utils.path.which('ifconfig') if ip_path: cmd1 = subprocess.Popen( '{0} link show'.format(ip_path), @@ -783,15 +807,15 @@ def linux_interfaces(): stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate()[0] ifaces = _interfaces_ip("{0}\n{1}".format( - salt.utils.to_str(cmd1), - salt.utils.to_str(cmd2))) + salt.utils.stringutils.to_str(cmd1), + salt.utils.stringutils.to_str(cmd2))) elif ifconfig_path: cmd = subprocess.Popen( '{0} -a'.format(ifconfig_path), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate()[0] - ifaces = _interfaces_ifconfig(salt.utils.to_str(cmd)) + ifaces = _interfaces_ifconfig(salt.utils.stringutils.to_str(cmd)) return ifaces @@ -894,7 +918,7 @@ def interfaces(): ''' Return a dictionary of information about all the interfaces on the minion ''' - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): return win_interfaces() else: return linux_interfaces() @@ -985,7 +1009,7 @@ def hw_addr(iface): Added support for AIX ''' - if salt.utils.is_aix(): + if salt.utils.platform.is_aix(): return _hw_addr_aix iface_info, error = _get_iface_info(iface) @@ -1015,7 +1039,8 @@ def interface_ip(iface): iface_info, error = _get_iface_info(iface) if error is False: - return iface_info.get(iface, {}).get('inet', {})[0].get('address', '') + inet = iface_info.get(iface, {}).get('inet', None) + return inet[0].get('address', '') if inet else '' else: return error @@ -1206,7 +1231,7 @@ def active_tcp(): ret = {} for statf in ['/proc/net/tcp', '/proc/net/tcp6']: if os.path.isfile(statf): - with salt.utils.fopen(statf, 'rb') as fp_: + with salt.utils.files.fopen(statf, 'rb') as fp_: for line in fp_: if line.strip().startswith('sl'): continue @@ -1241,7 +1266,7 @@ def _remotes_on(port, which_end): for statf in ['/proc/net/tcp', '/proc/net/tcp6']: if os.path.isfile(statf): proc_available = True - with salt.utils.fopen(statf, 'r') as fp_: + with salt.utils.files.fopen(statf, 'r') as fp_: for line in fp_: if line.strip().startswith('sl'): continue @@ -1251,17 +1276,17 @@ def _remotes_on(port, which_end): ret.add(iret[sl]['remote_addr']) if not proc_available: # Fallback to use OS specific tools - if salt.utils.is_sunos(): + if salt.utils.platform.is_sunos(): return _sunos_remotes_on(port, which_end) - if salt.utils.is_freebsd(): + if salt.utils.platform.is_freebsd(): return _freebsd_remotes_on(port, which_end) - if salt.utils.is_netbsd(): + if salt.utils.platform.is_netbsd(): return _netbsd_remotes_on(port, which_end) - if salt.utils.is_openbsd(): + if salt.utils.platform.is_openbsd(): return _openbsd_remotes_on(port, which_end) - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): return _windows_remotes_on(port, which_end) - if salt.utils.is_aix(): + if salt.utils.platform.is_aix(): return _aix_remotes_on(port, which_end) return _linux_remotes_on(port, which_end) @@ -1308,7 +1333,7 @@ def _sunos_remotes_on(port, which_end): log.error('Failed netstat') raise - lines = salt.utils.to_str(data).split('\n') + lines = salt.utils.stringutils.to_str(data).split('\n') for line in lines: if 'ESTABLISHED' not in line: continue @@ -1348,13 +1373,13 @@ def _freebsd_remotes_on(port, which_end): remotes = set() try: - cmd = salt.utils.shlex_split('sockstat -4 -c -p {0}'.format(port)) + cmd = salt.utils.args.shlex_split('sockstat -4 -c -p {0}'.format(port)) data = subprocess.check_output(cmd) # pylint: disable=minimum-python-version except subprocess.CalledProcessError as ex: log.error('Failed "sockstat" with returncode = {0}'.format(ex.returncode)) raise - lines = salt.utils.to_str(data).split('\n') + lines = salt.utils.stringutils.to_str(data).split('\n') for line in lines: chunks = line.split() @@ -1408,13 +1433,13 @@ def _netbsd_remotes_on(port, which_end): remotes = set() try: - cmd = salt.utils.shlex_split('sockstat -4 -c -n -p {0}'.format(port)) + cmd = salt.utils.args.shlex_split('sockstat -4 -c -n -p {0}'.format(port)) data = subprocess.check_output(cmd) # pylint: disable=minimum-python-version except subprocess.CalledProcessError as ex: log.error('Failed "sockstat" with returncode = {0}'.format(ex.returncode)) raise - lines = salt.utils.to_str(data).split('\n') + lines = salt.utils.stringutils.to_str(data).split('\n') for line in lines: chunks = line.split() @@ -1503,7 +1528,7 @@ def _windows_remotes_on(port, which_end): log.error('Failed netstat') raise - lines = salt.utils.to_str(data).split('\n') + lines = salt.utils.stringutils.to_str(data).split('\n') for line in lines: if 'ESTABLISHED' not in line: continue @@ -1550,7 +1575,7 @@ def _linux_remotes_on(port, which_end): log.error('Failed "lsof" with returncode = {0}'.format(ex.returncode)) raise - lines = salt.utils.to_str(data).split('\n') + lines = salt.utils.stringutils.to_str(data).split('\n') for line in lines: chunks = line.split() if not chunks: @@ -1609,7 +1634,7 @@ def _aix_remotes_on(port, which_end): log.error('Failed netstat') raise - lines = salt.utils.to_str(data).split('\n') + lines = salt.utils.stringutils.to_str(data).split('\n') for line in lines: if 'ESTABLISHED' not in line: continue diff --git a/salt/utils/odict.py b/salt/utils/odict.py index 78382562323..86b8d2b4c1b 100644 --- a/salt/utils/odict.py +++ b/salt/utils/odict.py @@ -25,7 +25,7 @@ from __future__ import absolute_import from collections import Callable # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six try: # pylint: disable=E0611,minimum-python-version diff --git a/salt/utils/openstack/neutron.py b/salt/utils/openstack/neutron.py index 671d9216afc..54fd2ec57a7 100644 --- a/salt/utils/openstack/neutron.py +++ b/salt/utils/openstack/neutron.py @@ -9,7 +9,7 @@ from __future__ import absolute_import, with_statement import logging # Import third party libs -import salt.ext.six as six +from salt.ext import six # pylint: disable=import-error HAS_NEUTRON = False try: diff --git a/salt/utils/openstack/nova.py b/salt/utils/openstack/nova.py index 82deb5769cc..eea3305cf7c 100644 --- a/salt/utils/openstack/nova.py +++ b/salt/utils/openstack/nova.py @@ -10,7 +10,7 @@ import logging import time # Import third party libs -import salt.ext.six as six +from salt.ext import six HAS_NOVA = False # pylint: disable=import-error try: @@ -36,7 +36,8 @@ except ImportError: # pylint: enable=import-error # Import salt libs -import salt.utils +import salt.utils.cloud +import salt.utils.files from salt.exceptions import SaltCloudSystemExit from salt.utils.versions import LooseVersion as _LooseVersion @@ -289,8 +290,9 @@ class SaltNova(object): self.session = keystoneauth1.session.Session(auth=options, verify=verify) conn = client.Client(version=self.version, session=self.session, **self.client_kwargs) self.kwargs['auth_token'] = conn.client.session.get_token() - self.catalog = conn.client.session.get('/auth/catalog', endpoint_filter={'service_type': 'identity'}).json().get('catalog', []) - if conn.client.get_endpoint(service_type='identity').endswith('v3'): + identity_service_type = kwargs.get('identity_service_type', 'identity') + self.catalog = conn.client.session.get('/auth/catalog', endpoint_filter={'service_type': identity_service_type}).json().get('catalog', []) + if conn.client.get_endpoint(service_type=identity_service_type).endswith('v3'): self._v3_setup(region_name) else: self._v2_setup(region_name) @@ -763,7 +765,7 @@ class SaltNova(object): ''' nt_ks = self.compute_conn if pubfile: - with salt.utils.fopen(pubfile, 'r') as fp_: + with salt.utils.files.fopen(pubfile, 'r') as fp_: pubkey = fp_.read() if not pubkey: return False diff --git a/salt/utils/openstack/swift.py b/salt/utils/openstack/swift.py index 53e5e2a36cb..aebca6fd347 100644 --- a/salt/utils/openstack/swift.py +++ b/salt/utils/openstack/swift.py @@ -14,7 +14,7 @@ from os.path import dirname, isdir from errno import EEXIST # Import Salt libs -import salt.utils +import salt.utils.files # Get logging started log = logging.getLogger(__name__) @@ -177,7 +177,7 @@ class SaltSwift(object): dirpath = dirname(local_file) if dirpath and not isdir(dirpath): mkdirs(dirpath) - fp = salt.utils.fopen(local_file, 'wb') # pylint: disable=resource-leakage + fp = salt.utils.files.fopen(local_file, 'wb') # pylint: disable=resource-leakage read_length = 0 for chunk in body: @@ -200,7 +200,7 @@ class SaltSwift(object): Upload a file to Swift ''' try: - with salt.utils.fopen(local_file, 'rb') as fp_: + with salt.utils.files.fopen(local_file, 'rb') as fp_: self.conn.put_object(cont, obj, fp_) return True except Exception as exc: diff --git a/salt/utils/oset.py b/salt/utils/oset.py new file mode 100644 index 00000000000..677181f307e --- /dev/null +++ b/salt/utils/oset.py @@ -0,0 +1,204 @@ +# -*- coding: utf-8 -*- +""" + +Available at repository https://github.com/LuminosoInsight/ordered-set + + salt.utils.oset + ~~~~~~~~~~~~~~~~ + +An OrderedSet is a custom MutableSet that remembers its order, so that every +entry has an index that can be looked up. + +Based on a recipe originally posted to ActiveState Recipes by Raymond Hettiger, +and released under the MIT license. + +Rob Speer's changes are as follows: + + - changed the content from a doubly-linked list to a regular Python list. + Seriously, who wants O(1) deletes but O(N) lookups by index? + - add() returns the index of the added item + - index() just returns the index of an item + - added a __getstate__ and __setstate__ so it can be pickled + - added __getitem__ +""" +from __future__ import absolute_import +import collections + +SLICE_ALL = slice(None) +__version__ = '2.0.1' + + +def is_iterable(obj): + """ + Are we being asked to look up a list of things, instead of a single thing? + We check for the `__iter__` attribute so that this can cover types that + don't have to be known by this module, such as NumPy arrays. + + Strings, however, should be considered as atomic values to look up, not + iterables. The same goes for tuples, since they are immutable and therefore + valid entries. + + We don't need to check for the Python 2 `unicode` type, because it doesn't + have an `__iter__` attribute anyway. + """ + return hasattr(obj, '__iter__') and not isinstance(obj, str) and not isinstance(obj, tuple) + + +class OrderedSet(collections.MutableSet): + """ + An OrderedSet is a custom MutableSet that remembers its order, so that + every entry has an index that can be looked up. + """ + def __init__(self, iterable=None): + self.items = [] + self.map = {} + if iterable is not None: + self |= iterable + + def __len__(self): + return len(self.items) + + def __getitem__(self, index): + """ + Get the item at a given index. + + If `index` is a slice, you will get back that slice of items. If it's + the slice [:], exactly the same object is returned. (If you want an + independent copy of an OrderedSet, use `OrderedSet.copy()`.) + + If `index` is an iterable, you'll get the OrderedSet of items + corresponding to those indices. This is similar to NumPy's + "fancy indexing". + """ + if index == SLICE_ALL: + return self + elif hasattr(index, '__index__') or isinstance(index, slice): + result = self.items[index] + if isinstance(result, list): + return OrderedSet(result) + else: + return result + elif is_iterable(index): + return OrderedSet([self.items[i] for i in index]) + else: + raise TypeError("Don't know how to index an OrderedSet by {}".format(repr(index))) + + def copy(self): + return OrderedSet(self) + + def __getstate__(self): + if len(self) == 0: + # The state can't be an empty list. + # We need to return a truthy value, or else __setstate__ won't be run. + # + # This could have been done more gracefully by always putting the state + # in a tuple, but this way is backwards- and forwards- compatible with + # previous versions of OrderedSet. + return (None,) + else: + return list(self) + + def __setstate__(self, state): + if state == (None,): + self.__init__([]) + else: + self.__init__(state) + + def __contains__(self, key): + return key in self.map + + def add(self, key): + """ + Add `key` as an item to this OrderedSet, then return its index. + + If `key` is already in the OrderedSet, return the index it already + had. + """ + if key not in self.map: + self.map[key] = len(self.items) + self.items.append(key) + return self.map[key] + append = add + + def update(self, sequence): + """ + Update the set with the given iterable sequence, then return the index + of the last element inserted. + """ + item_index = None + try: + for item in sequence: + item_index = self.add(item) + except TypeError: + raise ValueError("Argument needs to be an iterable, got {}".format(type(sequence))) + return item_index + + def index(self, key): + """ + Get the index of a given entry, raising an IndexError if it's not + present. + + `key` can be an iterable of entries that is not a string, in which case + this returns a list of indices. + """ + if is_iterable(key): + return [self.index(subkey) for subkey in key] + return self.map[key] + + def pop(self): + """ + Remove and return the last element from the set. + + Raises KeyError if the set is empty. + """ + if not self.items: + raise KeyError('Set is empty') + + elem = self.items[-1] + del self.items[-1] + del self.map[elem] + return elem + + def discard(self, key): + """ + Remove an element. Do not raise an exception if absent. + + The MutableSet mixin uses this to implement the .remove() method, which + *does* raise an error when asked to remove a non-existent item. + """ + if key in self: + i = self.map[key] + del self.items[i] + del self.map[key] + for k, v in self.map.items(): + if v >= i: + self.map[k] = v - 1 + + def clear(self): + """ + Remove all items from this OrderedSet. + """ + del self.items[:] + self.map.clear() + + def __iter__(self): + return iter(self.items) + + def __reversed__(self): + return reversed(self.items) + + def __repr__(self): + if not self: + return "{}()".format(self.__class__.__name__) + return "{}({})".format(self.__class__.__name__, repr(list(self))) + + def __eq__(self, other): + if isinstance(other, OrderedSet): + return len(self) == len(other) and self.items == other.items + try: + other_as_set = set(other) + except TypeError: + # If `other` can't be converted into a set, it's not equal. + return False + else: + return set(self) == other_as_set diff --git a/salt/utils/parsers.py b/salt/utils/parsers.py index b440cc54ee0..c32bd4b14f3 100644 --- a/salt/utils/parsers.py +++ b/salt/utils/parsers.py @@ -32,14 +32,16 @@ import salt.syspaths as syspaths import salt.version as version import salt.utils import salt.utils.args -import salt.utils.xdg +import salt.utils.files import salt.utils.jid -from salt.utils import kinds +import salt.utils.kinds as kinds +import salt.utils.platform +import salt.utils.xdg from salt.defaults import DEFAULT_TARGET_DELIM from salt.utils.validate.path import is_writeable from salt.utils.verify import verify_files import salt.exceptions -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import range # pylint: disable=import-error,redefined-builtin from salt.utils.yamldumper import SafeOrderedDumper @@ -806,7 +808,7 @@ class LogLevelMixIn(six.with_metaclass(MixInMeta, object)): # Log rotate options log_rotate_max_bytes = self.config.get('log_rotate_max_bytes', 0) log_rotate_backup_count = self.config.get('log_rotate_backup_count', 0) - if not salt.utils.is_windows(): + if not salt.utils.platform.is_windows(): # Not supported on platforms other than Windows. # Other platforms may use an external tool such as 'logrotate' if log_rotate_max_bytes != 0: @@ -825,7 +827,7 @@ class LogLevelMixIn(six.with_metaclass(MixInMeta, object)): self.config['log_rotate_backup_count'] = log_rotate_backup_count def setup_logfile_logger(self): - if salt.utils.is_windows() and self._setup_mp_logging_listener_: + if salt.utils.platform.is_windows() and self._setup_mp_logging_listener_: # On Windows when using a logging listener, all log file logging # will go through the logging listener. return @@ -849,7 +851,7 @@ class LogLevelMixIn(six.with_metaclass(MixInMeta, object)): log.set_logger_level(name, level) def __setup_extended_logging(self, *args): # pylint: disable=unused-argument - if salt.utils.is_windows() and self._setup_mp_logging_listener_: + if salt.utils.platform.is_windows() and self._setup_mp_logging_listener_: # On Windows when using a logging listener, all extended logging # will go through the logging listener. return @@ -866,7 +868,7 @@ class LogLevelMixIn(six.with_metaclass(MixInMeta, object)): ) def _setup_mp_logging_client(self, *args): # pylint: disable=unused-argument - if salt.utils.is_windows() and self._setup_mp_logging_listener_: + if salt.utils.platform.is_windows() and self._setup_mp_logging_listener_: # On Windows, all logging including console and # log file logging will go through the multiprocessing # logging listener if it exists. @@ -913,7 +915,7 @@ class LogLevelMixIn(six.with_metaclass(MixInMeta, object)): if getattr(self.options, 'daemon', False) is True: return - if salt.utils.is_windows() and self._setup_mp_logging_listener_: + if salt.utils.platform.is_windows() and self._setup_mp_logging_listener_: # On Windows when using a logging listener, all console logging # will go through the logging listener. return @@ -1007,7 +1009,7 @@ class DaemonMixIn(six.with_metaclass(MixInMeta, object)): if self.check_pidfile(): pid = self.get_pidfile() - if not salt.utils.is_windows(): + if not salt.utils.platform.is_windows(): if self.check_pidfile() and self.is_daemonized(pid) and not os.getppid() == pid: return True else: @@ -1056,6 +1058,13 @@ class TargetOptionsMixIn(six.with_metaclass(MixInMeta, object)): self, 'Target Options', 'Target selection options.' ) self.add_option_group(group) + group.add_option( + '-H', '--hosts', + default=False, + action='store_true', + dest='list_hosts', + help='List all known hosts to currently visible or other specified rosters' + ) group.add_option( '-E', '--pcre', default=False, @@ -1339,7 +1348,7 @@ class OutputOptionsMixIn(six.with_metaclass(MixInMeta, object)): if self.options.output_file is not None and self.options.output_file_append is False: if os.path.isfile(self.options.output_file): try: - with salt.utils.fopen(self.options.output_file, 'w') as ofh: + with salt.utils.files.fopen(self.options.output_file, 'w') as ofh: # Make this a zero length filename instead of removing # it. This way we keep the file permissions. ofh.write('') @@ -1538,7 +1547,7 @@ class CloudQueriesMixIn(six.with_metaclass(MixInMeta, object)): action='store', help='Display a list of configured profiles. Pass in a cloud ' 'provider to view the provider\'s associated profiles, ' - 'such as digital_ocean, or pass in "all" to list all the ' + 'such as digitalocean, or pass in "all" to list all the ' 'configured profiles.' ) self.add_option_group(group) @@ -2190,10 +2199,18 @@ class SaltCPOptionParser(six.with_metaclass(OptionParserMeta, def _mixin_setup(self): file_opts_group = optparse.OptionGroup(self, 'File Options') + file_opts_group.add_option( + '-C', '--chunked', + default=False, + dest='chunked', + action='store_true', + help='Use chunked files transfer. Supports big files, recursive ' + 'lookup and directories creation.' + ) file_opts_group.add_option( '-n', '--no-compression', default=True, - dest='compression', + dest='gzip', action='store_false', help='Disable gzip compression.' ) @@ -2214,7 +2231,6 @@ class SaltCPOptionParser(six.with_metaclass(OptionParserMeta, self.config['tgt'] = self.args[0] self.config['src'] = [os.path.realpath(x) for x in self.args[1:-1]] self.config['dest'] = self.args[-1] - self.config['gzip'] = True def setup_config(self): return config.master_config(self.get_config_file_path()) @@ -2386,6 +2402,16 @@ class SaltKeyOptionParser(six.with_metaclass(OptionParserMeta, 'Default: %default.') ) + self.add_option( + '--preserve-minions', + default=False, + help=('Setting this to True prevents the master from deleting ' + 'the minion cache when keys are deleted, this may have ' + 'security implications if compromised minions auth with ' + 'a previous deleted minion ID. ' + 'Default: %default.') + ) + key_options_group = optparse.OptionGroup( self, 'Key Generation Options' ) @@ -2479,12 +2505,19 @@ class SaltKeyOptionParser(six.with_metaclass(OptionParserMeta, return keys_config def process_rotate_aes_key(self): - if hasattr(self.options, 'rotate_aes_key') and isinstance(self.options.rotate_aes_key, str): + if hasattr(self.options, 'rotate_aes_key') and isinstance(self.options.rotate_aes_key, six.string_types): if self.options.rotate_aes_key.lower() == 'true': self.options.rotate_aes_key = True elif self.options.rotate_aes_key.lower() == 'false': self.options.rotate_aes_key = False + def process_preserve_minions(self): + if hasattr(self.options, 'preserve_minions') and isinstance(self.options.preserve_minions, str): + if self.options.preserve_minions.lower() == 'true': + self.options.preserve_minions = True + elif self.options.preserve_minions.lower() == 'false': + self.options.preserve_minions = False + def process_list(self): # Filter accepted list arguments as soon as possible if not self.options.list: @@ -2781,6 +2814,12 @@ class SaltRunOptionParser(six.with_metaclass(OptionParserMeta, action='store_true', help=('Start the runner operation and immediately return control.') ) + self.add_option( + '--skip-grains', + default=False, + action='store_true', + help=('Do not load grains.') + ) group = self.output_options_group = optparse.OptionGroup( self, 'Output Options', 'Configure your preferred output format.' ) @@ -3036,6 +3075,14 @@ class SaltSSHOptionParser(six.with_metaclass(OptionParserMeta, action='store_true', help='Run command via sudo.' ) + auth_group.add_option( + '--skip-roster', + dest='ssh_skip_roster', + default=False, + action='store_true', + help='If hostname is not found in the roster, do not store the information' + 'into the default roster file (flat).' + ) self.add_option_group(auth_group) scan_group = optparse.OptionGroup( diff --git a/salt/utils/path.py b/salt/utils/path.py index 53f94792b3e..b9700fdc3aa 100644 --- a/salt/utils/path.py +++ b/salt/utils/path.py @@ -4,15 +4,25 @@ Platform independent versions of some os/os.path functions. Gets around PY2's lack of support for reading NTFS links. ''' -# Import python lib +# Import python libs from __future__ import absolute_import +import collections import errno import logging import os +import posixpath +import re import struct +import sys + +# Import Salt libs +import salt.utils.args +import salt.utils.platform +import salt.utils.stringutils +from salt.utils.decorators import memoize as real_memoize +from salt.utils.decorators.jinja import jinja_filter # Import 3rd-party libs -import salt.utils from salt.ext import six try: @@ -29,7 +39,7 @@ def islink(path): ''' Equivalent to os.path.islink() ''' - if six.PY3 or not salt.utils.is_windows(): + if six.PY3 or not salt.utils.platform.is_windows(): return os.path.islink(path) if not HAS_WIN32FILE: @@ -64,7 +74,7 @@ def readlink(path): ''' Equivalent to os.readlink() ''' - if six.PY3 or not salt.utils.is_windows(): + if six.PY3 or not salt.utils.platform.is_windows(): return os.readlink(path) if not HAS_WIN32FILE: @@ -168,3 +178,133 @@ def _get_reparse_data(path): win32file.CloseHandle(fileHandle) return reparseData + + +@jinja_filter('which') +def which(exe=None): + ''' + Python clone of /usr/bin/which + ''' + def _is_executable_file_or_link(exe): + # check for os.X_OK doesn't suffice because directory may executable + return (os.access(exe, os.X_OK) and + (os.path.isfile(exe) or os.path.islink(exe))) + + if exe: + if _is_executable_file_or_link(exe): + # executable in cwd or fullpath + return exe + + ext_list = os.environ.get('PATHEXT', '.EXE').split(';') + + @real_memoize + def _exe_has_ext(): + ''' + Do a case insensitive test if exe has a file extension match in + PATHEXT + ''' + for ext in ext_list: + try: + pattern = r'.*\.' + ext.lstrip('.') + r'$' + re.match(pattern, exe, re.I).groups() + return True + except AttributeError: + continue + return False + + # Enhance POSIX path for the reliability at some environments, when $PATH is changing + # This also keeps order, where 'first came, first win' for cases to find optional alternatives + search_path = os.environ.get('PATH') and os.environ['PATH'].split(os.pathsep) or list() + for default_path in ['/bin', '/sbin', '/usr/bin', '/usr/sbin', '/usr/local/bin']: + if default_path not in search_path: + search_path.append(default_path) + os.environ['PATH'] = os.pathsep.join(search_path) + for path in search_path: + full_path = os.path.join(path, exe) + if _is_executable_file_or_link(full_path): + return full_path + elif salt.utils.platform.is_windows() and not _exe_has_ext(): + # On Windows, check for any extensions in PATHEXT. + # Allows both 'cmd' and 'cmd.exe' to be matched. + for ext in ext_list: + # Windows filesystem is case insensitive so we + # safely rely on that behavior + if _is_executable_file_or_link(full_path + ext): + return full_path + ext + log.trace('\'{0}\' could not be found in the following search path: \'{1}\''.format(exe, search_path)) + else: + log.error('No executable was passed to be searched by salt.utils.path.which()') + + return None + + +def which_bin(exes): + ''' + Scan over some possible executables and return the first one that is found + ''' + if not isinstance(exes, collections.Iterable): + return None + for exe in exes: + path = which(exe) + if not path: + continue + return path + return None + + +@jinja_filter('path_join') +def join(*parts, **kwargs): + ''' + This functions tries to solve some issues when joining multiple absolute + paths on both *nix and windows platforms. + + See tests/unit/utils/path_join_test.py for some examples on what's being + talked about here. + + The "use_posixpath" kwarg can be be used to force joining using poxixpath, + which is useful for Salt fileserver paths on Windows masters. + ''' + if six.PY3: + new_parts = [] + for part in parts: + new_parts.append(salt.utils.stringutils.to_str(part)) + parts = new_parts + + kwargs = salt.utils.args.clean_kwargs(**kwargs) + use_posixpath = kwargs.pop('use_posixpath', False) + if kwargs: + salt.utils.args.invalid_kwargs(kwargs) + + pathlib = posixpath if use_posixpath else os.path + + # Normalize path converting any os.sep as needed + parts = [pathlib.normpath(p) for p in parts] + + try: + root = parts.pop(0) + except IndexError: + # No args passed to func + return '' + + if not parts: + ret = root + else: + stripped = [p.lstrip(os.sep) for p in parts] + try: + ret = pathlib.join(root, *stripped) + except UnicodeDecodeError: + # This is probably Python 2 and one of the parts contains unicode + # characters in a bytestring. First try to decode to the system + # encoding. + try: + enc = __salt_system_encoding__ + except NameError: + enc = sys.stdin.encoding or sys.getdefaultencoding() + try: + ret = pathlib.join(root.decode(enc), + *[x.decode(enc) for x in stripped]) + except UnicodeDecodeError: + # Last resort, try UTF-8 + ret = pathlib.join(root.decode('UTF-8'), + *[x.decode('UTF-8') for x in stripped]) + return pathlib.normpath(ret) diff --git a/salt/utils/pkg/__init__.py b/salt/utils/pkg/__init__.py index 9cc675cc2dc..68e58646762 100644 --- a/salt/utils/pkg/__init__.py +++ b/salt/utils/pkg/__init__.py @@ -10,7 +10,9 @@ import os import re # Import Salt libs -import salt.utils +import salt.utils # Can be removed once is_true is moved +import salt.utils.files +import salt.utils.versions log = logging.getLogger(__name__) @@ -43,7 +45,7 @@ def write_rtag(opts): rtag_file = rtag(opts) if not os.path.exists(rtag_file): try: - with salt.utils.fopen(rtag_file, 'w+'): + with salt.utils.files.fopen(rtag_file, 'w+'): pass except OSError as exc: log.warning('Encountered error writing rtag: %s', exc.__str__()) @@ -85,7 +87,7 @@ def match_version(desired, available, cmp_func=None, ignore_epoch=False): if not oper: oper = '==' for candidate in available: - if salt.utils.compare_versions(ver1=candidate, + if salt.utils.versions.compare(ver1=candidate, oper=oper, ver2=version, cmp_func=cmp_func, diff --git a/salt/utils/pkg/deb.py b/salt/utils/pkg/deb.py index 615319d6e38..7cd170a4a3d 100644 --- a/salt/utils/pkg/deb.py +++ b/salt/utils/pkg/deb.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- ''' -Common functions for working with RPM packages +Common functions for working with deb packages ''' # Import python libs diff --git a/salt/utils/pkg/rpm.py b/salt/utils/pkg/rpm.py index 21d12146adb..c5ee94aee21 100644 --- a/salt/utils/pkg/rpm.py +++ b/salt/utils/pkg/rpm.py @@ -6,9 +6,14 @@ Common functions for working with RPM packages # Import python libs from __future__ import absolute_import import collections +import datetime import logging import subprocess +# Import 3rd-party libs +from salt.ext import six +from salt.ext.six.moves import range # pylint: disable=redefined-builtin + log = logging.getLogger(__name__) # These arches compiled from the rpmUtils.arch python module source @@ -30,7 +35,7 @@ ARCHES = ARCHES_64 + ARCHES_32 + ARCHES_PPC + ARCHES_S390 + \ ARCHES_ALPHA + ARCHES_ARM + ARCHES_SH # EPOCHNUM can't be used until RHEL5 is EOL as it is not present -QUERYFORMAT = '%{NAME}_|-%{EPOCH}_|-%{VERSION}_|-%{RELEASE}_|-%{ARCH}_|-%{REPOID}' +QUERYFORMAT = '%{NAME}_|-%{EPOCH}_|-%{VERSION}_|-%{RELEASE}_|-%{ARCH}_|-%{REPOID}_|-%{INSTALLTIME}' def get_osarch(): @@ -55,15 +60,17 @@ def check_32(arch, osarch=None): return all(x in ARCHES_32 for x in (osarch, arch)) -def pkginfo(name, version, arch, repoid): +def pkginfo(name, version, arch, repoid, install_date=None, install_date_time_t=None): ''' Build and return a pkginfo namedtuple ''' pkginfo_tuple = collections.namedtuple( 'PkgInfo', - ('name', 'version', 'arch', 'repoid') + ('name', 'version', 'arch', 'repoid', 'install_date', + 'install_date_time_t') ) - return pkginfo_tuple(name, version, arch, repoid) + return pkginfo_tuple(name, version, arch, repoid, install_date, + install_date_time_t) def resolve_name(name, arch, osarch=None): @@ -85,7 +92,7 @@ def parse_pkginfo(line, osarch=None): pkginfo namedtuple. ''' try: - name, epoch, version, release, arch, repoid = line.split('_|-') + name, epoch, version, release, arch, repoid, install_time = line.split('_|-') # Handle unpack errors (should never happen with the queryformat we are # using, but can't hurt to be careful). except ValueError: @@ -97,4 +104,63 @@ def parse_pkginfo(line, osarch=None): if epoch not in ('(none)', '0'): version = ':'.join((epoch, version)) - return pkginfo(name, version, arch, repoid) + if install_time not in ('(none)', '0'): + install_date = datetime.datetime.utcfromtimestamp(int(install_time)).isoformat() + "Z" + install_date_time_t = int(install_time) + else: + install_date = None + install_date_time_t = None + + return pkginfo(name, version, arch, repoid, install_date, install_date_time_t) + + +def combine_comments(comments): + ''' + Given a list of comments, strings, a single comment or a single string, + return a single string of text containing all of the comments, prepending + the '#' and joining with newlines as necessary. + ''' + if not isinstance(comments, list): + comments = [comments] + for idx in range(len(comments)): + if not isinstance(comments[idx], six.string_types): + comments[idx] = str(comments[idx]) + comments[idx] = comments[idx].strip() + if not comments[idx].startswith('#'): + comments[idx] = '#' + comments[idx] + return '\n'.join(comments) + + +def version_to_evr(verstring): + ''' + Split the package version string into epoch, version and release. + Return this as tuple. + + The epoch is always not empty. The version and the release can be an empty + string if such a component could not be found in the version string. + + "2:1.0-1.2" => ('2', '1.0', '1.2) + "1.0" => ('0', '1.0', '') + "" => ('0', '', '') + ''' + if verstring in [None, '']: + return '0', '', '' + + idx_e = verstring.find(':') + if idx_e != -1: + try: + epoch = str(int(verstring[:idx_e])) + except ValueError: + # look, garbage in the epoch field, how fun, kill it + epoch = '0' # this is our fallback, deal + else: + epoch = '0' + idx_r = verstring.find('-') + if idx_r != -1: + version = verstring[idx_e + 1:idx_r] + release = verstring[idx_r + 1:] + else: + version = verstring[idx_e + 1:] + release = '' + + return epoch, version, release diff --git a/salt/utils/platform.py b/salt/utils/platform.py new file mode 100644 index 00000000000..2da117e62b6 --- /dev/null +++ b/salt/utils/platform.py @@ -0,0 +1,158 @@ +# -*- coding: utf-8 -*- +''' +Functions for identifying which platform a machine is +''' +# Import Python libs +from __future__ import absolute_import +import os +import subprocess +import sys + +# Import Salt libs +from salt.utils.decorators import memoize as real_memoize + + +@real_memoize +def is_windows(): + ''' + Simple function to return if a host is Windows or not + ''' + return sys.platform.startswith('win') + + +@real_memoize +def is_proxy(): + ''' + Return True if this minion is a proxy minion. + Leverages the fact that is_linux() and is_windows + both return False for proxies. + TODO: Need to extend this for proxies that might run on + other Unices + ''' + import __main__ as main + # This is a hack. If a proxy minion is started by other + # means, e.g. a custom script that creates the minion objects + # then this will fail. + ret = False + try: + # Changed this from 'salt-proxy in main...' to 'proxy in main...' + # to support the testsuite's temp script that is called 'cli_salt_proxy' + if 'proxy' in main.__file__: + ret = True + except AttributeError: + pass + return ret + + +@real_memoize +def is_linux(): + ''' + Simple function to return if a host is Linux or not. + Note for a proxy minion, we need to return something else + ''' + return sys.platform.startswith('linux') + + +@real_memoize +def is_darwin(): + ''' + Simple function to return if a host is Darwin (macOS) or not + ''' + return sys.platform.startswith('darwin') + + +@real_memoize +def is_sunos(): + ''' + Simple function to return if host is SunOS or not + ''' + return sys.platform.startswith('sunos') + + +@real_memoize +def is_smartos(): + ''' + Simple function to return if host is SmartOS (Illumos) or not + ''' + if not is_sunos(): + return False + else: + return os.uname()[3].startswith('joyent_') + + +@real_memoize +def is_smartos_globalzone(): + ''' + Function to return if host is SmartOS (Illumos) global zone or not + ''' + if not is_smartos(): + return False + else: + cmd = ['zonename'] + try: + zonename = subprocess.Popen( + cmd, shell=False, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + except OSError: + return False + if zonename.returncode: + return False + if zonename.stdout.read().strip() == 'global': + return True + + return False + + +@real_memoize +def is_smartos_zone(): + ''' + Function to return if host is SmartOS (Illumos) and not the gz + ''' + if not is_smartos(): + return False + else: + cmd = ['zonename'] + try: + zonename = subprocess.Popen( + cmd, shell=False, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + except OSError: + return False + if zonename.returncode: + return False + if zonename.stdout.read().strip() == 'global': + return False + + return True + + +@real_memoize +def is_freebsd(): + ''' + Simple function to return if host is FreeBSD or not + ''' + return sys.platform.startswith('freebsd') + + +@real_memoize +def is_netbsd(): + ''' + Simple function to return if host is NetBSD or not + ''' + return sys.platform.startswith('netbsd') + + +@real_memoize +def is_openbsd(): + ''' + Simple function to return if host is OpenBSD or not + ''' + return sys.platform.startswith('openbsd') + + +@real_memoize +def is_aix(): + ''' + Simple function to return if host is AIX or not + ''' + return sys.platform.startswith('aix') diff --git a/salt/utils/preseed.py b/salt/utils/preseed.py index 444e442756b..acebe16dea5 100644 --- a/salt/utils/preseed.py +++ b/salt/utils/preseed.py @@ -7,7 +7,7 @@ Utilities for managing Debian preseed from __future__ import absolute_import import yaml import shlex -import salt.utils +import salt.utils.files def mksls(src, dst=None): @@ -15,7 +15,7 @@ def mksls(src, dst=None): Convert a preseed file to an SLS file ''' ps_opts = {} - with salt.utils.fopen(src, 'r') as fh_: + with salt.utils.files.fopen(src, 'r') as fh_: for line in fh_: if line.startswith('#'): continue @@ -72,7 +72,7 @@ def mksls(src, dst=None): sls[iface]['nameservers'] = ps_opts['d-i']['netcfg']['get_nameservers']['argument'] if dst is not None: - with salt.utils.fopen(dst, 'w') as fh_: + with salt.utils.files.fopen(dst, 'w') as fh_: fh_.write(yaml.safe_dump(sls, default_flow_style=False)) else: return yaml.safe_dump(sls, default_flow_style=False) diff --git a/salt/utils/process.py b/salt/utils/process.py index aefe8731ee3..39c7e021457 100644 --- a/salt/utils/process.py +++ b/salt/utils/process.py @@ -15,17 +15,21 @@ import contextlib import subprocess import multiprocessing import multiprocessing.util +import socket # Import salt libs import salt.defaults.exitcodes -import salt.utils +import salt.utils # Can be removed once appendproctitle is moved +import salt.utils.files +import salt.utils.path +import salt.utils.platform import salt.log.setup import salt.defaults.exitcodes from salt.log.mixins import NewStyleClassMixIn # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import queue, range # pylint: disable=import-error,redefined-builtin from tornado import gen @@ -54,8 +58,22 @@ def notify_systemd(): try: import systemd.daemon except ImportError: - if salt.utils.which('systemd-notify') and systemd_notify_call('--booted'): - return systemd_notify_call('--ready') + if salt.utils.path.which('systemd-notify') \ + and systemd_notify_call('--booted'): + # Notify systemd synchronously + notify_socket = os.getenv('NOTIFY_SOCKET') + if notify_socket: + # Handle abstract namespace socket + if notify_socket.startswith('@'): + notify_socket = '\0{0}'.format(notify_socket[1:]) + try: + sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) + sock.connect(notify_socket) + sock.sendall('READY=1'.encode()) + sock.close() + except socket.error: + return systemd_notify_call('--ready') + return True return False if systemd.daemon.booted(): @@ -74,13 +92,13 @@ def set_pidfile(pidfile, user): if not os.path.isdir(pdir) and pdir: os.makedirs(pdir) try: - with salt.utils.fopen(pidfile, 'w+') as ofile: + with salt.utils.files.fopen(pidfile, 'w+') as ofile: ofile.write(str(os.getpid())) except IOError: pass log.debug(('Created pidfile: {0}').format(pidfile)) - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): return True import pwd # after confirming not running Windows @@ -128,10 +146,12 @@ def get_pidfile(pidfile): ''' Return the pid from a pidfile as an integer ''' - with salt.utils.fopen(pidfile) as pdf: + with salt.utils.files.fopen(pidfile) as pdf: pid = pdf.read() - - return int(pid) + if pid: + return int(pid) + else: + return def clean_proc(proc, wait_for_kill=10): @@ -277,7 +297,7 @@ class ProcessManager(object): if kwargs is None: kwargs = {} - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): # Need to ensure that 'log_queue' is correctly transferred to # processes that inherit from 'MultiprocessingProcess'. if type(MultiprocessingProcess) is type(tgt) and ( @@ -347,7 +367,7 @@ class ProcessManager(object): self._restart_processes = False def send_signal_to_processes(self, signal_): - if (salt.utils.is_windows() and + if (salt.utils.platform.is_windows() and signal_ in (signal.SIGTERM, signal.SIGINT)): # On Windows, the subprocesses automatically have their signal # handlers invoked. If you send one of these signals while the @@ -438,7 +458,7 @@ class ProcessManager(object): return signal.default_int_handler(signal.SIGTERM)(*args) else: return - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): if multiprocessing.current_process().name != 'MainProcess': # Since the main process will kill subprocesses by tree, # no need to do anything in the subprocesses. @@ -446,7 +466,7 @@ class ProcessManager(object): # call 'taskkill', it will leave a 'taskkill' zombie process. # We want to avoid this. return - with salt.utils.fopen(os.devnull, 'wb') as devnull: + with salt.utils.files.fopen(os.devnull, 'wb') as devnull: for pid, p_map in six.iteritems(self._process_map): # On Windows, we need to explicitly terminate sub-processes # because the processes don't have a sigterm handler. @@ -559,7 +579,7 @@ class MultiprocessingProcess(multiprocessing.Process, NewStyleClassMixIn): return instance def __init__(self, *args, **kwargs): - if (salt.utils.is_windows() and + if (salt.utils.platform.is_windows() and not hasattr(self, '_is_child') and self.__setstate__.__code__ is MultiprocessingProcess.__setstate__.__code__): @@ -590,7 +610,7 @@ class MultiprocessingProcess(multiprocessing.Process, NewStyleClassMixIn): # 'log_queue' from kwargs. super(MultiprocessingProcess, self).__init__(*args, **kwargs) - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): # On Windows, the multiprocessing.Process object is reinitialized # in the child process via the constructor. Due to this, methods # such as ident() and is_alive() won't work properly. So we use @@ -661,7 +681,7 @@ class MultiprocessingProcess(multiprocessing.Process, NewStyleClassMixIn): class SignalHandlingMultiprocessingProcess(MultiprocessingProcess): def __init__(self, *args, **kwargs): super(SignalHandlingMultiprocessingProcess, self).__init__(*args, **kwargs) - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): if hasattr(self, '_is_child'): # On Windows, no need to call register_after_fork(). # register_after_fork() would only work on Windows if called diff --git a/salt/utils/psutil_compat.py b/salt/utils/psutil_compat.py index a32712860aa..cbb9cd9da97 100644 --- a/salt/utils/psutil_compat.py +++ b/salt/utils/psutil_compat.py @@ -14,7 +14,7 @@ Built off of http://grodola.blogspot.com/2014/01/psutil-20-porting.html from __future__ import absolute_import # Import Salt libs -import salt.ext.six as six +from salt.ext import six # No exception handling, as we want ImportError if psutil doesn't exist import psutil # pylint: disable=3rd-party-module-not-gated diff --git a/salt/utils/pycrypto.py b/salt/utils/pycrypto.py index ea6d5370f75..f2220b368a6 100644 --- a/salt/utils/pycrypto.py +++ b/salt/utils/pycrypto.py @@ -29,6 +29,7 @@ except ImportError: # Import salt libs import salt.utils +import salt.utils.stringutils from salt.exceptions import SaltInvocationError @@ -43,7 +44,7 @@ def secure_password(length=20, use_random=True): pw += re.sub( r'\W', '', - salt.utils.to_str(CRand.get_random_bytes(1)) + salt.utils.stringutils.to_str(CRand.get_random_bytes(1)) ) else: pw += random.SystemRandom().choice(string.ascii_letters + string.digits) diff --git a/salt/utils/pydsl.py b/salt/utils/pydsl.py index 221b24e8264..fb62598ba15 100644 --- a/salt/utils/pydsl.py +++ b/salt/utils/pydsl.py @@ -89,11 +89,10 @@ from uuid import uuid4 as _uuid # Import salt libs from salt.utils.odict import OrderedDict -from salt.utils import warn_until from salt.state import HighState # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six REQUISITES = set('listen require watch prereq use listen_in require_in watch_in prereq_in use_in onchanges onfail'.split()) @@ -140,12 +139,7 @@ class Sls(object): def include(self, *sls_names, **kws): if 'env' in kws: - warn_until( - 'Oxygen', - 'Parameter \'env\' has been detected in the argument list. This ' - 'parameter is no longer used and has been replaced by \'saltenv\' ' - 'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.' - ) + # "env" is not supported; Use "saltenv". kws.pop('env') saltenv = kws.get('saltenv', self.saltenv) diff --git a/salt/utils/pyobjects.py b/salt/utils/pyobjects.py index 762080e993f..9306a3a3ed2 100644 --- a/salt/utils/pyobjects.py +++ b/salt/utils/pyobjects.py @@ -10,7 +10,8 @@ import inspect import logging from salt.utils.odict import OrderedDict -import salt.ext.six as six +from salt.utils.schema import Prepareable +from salt.ext import six REQUISITES = ('listen', 'onchanges', 'onfail', 'require', 'watch', 'use', 'listen_in', 'onchanges_in', 'onfail_in', 'require_in', 'watch_in', 'use_in') @@ -288,20 +289,30 @@ class SaltObject(object): return __wrapper__() -class MapMeta(type): +class MapMeta(six.with_metaclass(Prepareable, type)): ''' This is the metaclass for our Map class, used for building data maps based off of grain data. ''' + @classmethod + def __prepare__(metacls, name, bases): + return OrderedDict() + + def __new__(cls, name, bases, attrs): + c = type.__new__(cls, name, bases, attrs) + c.__ordered_attrs__ = attrs.keys() + return c + def __init__(cls, name, bases, nmspc): cls.__set_attributes__() super(MapMeta, cls).__init__(name, bases, nmspc) def __set_attributes__(cls): - match_groups = OrderedDict([]) + match_info = [] + grain_targets = set() # find all of our filters - for item in cls.__dict__: + for item in cls.__ordered_attrs__: if item[0] == '_': continue @@ -313,31 +324,56 @@ class MapMeta(type): # which grain are we filtering on grain = getattr(filt, '__grain__', 'os_family') - if grain not in match_groups: - match_groups[grain] = OrderedDict([]) + grain_targets.add(grain) # does the object pointed to have a __match__ attribute? # if so use it, otherwise use the name of the object # this is so that you can match complex values, which the python # class name syntax does not allow - if hasattr(filt, '__match__'): - match = filt.__match__ - else: - match = item + match = getattr(filt, '__match__', item) - match_groups[grain][match] = OrderedDict([]) + match_attrs = {} for name in filt.__dict__: - if name[0] == '_': - continue + if name[0] != '_': + match_attrs[name] = filt.__dict__[name] - match_groups[grain][match][name] = filt.__dict__[name] + match_info.append((grain, match, match_attrs)) + # Reorder based on priority + try: + if not hasattr(cls.priority, '__iter__'): + log.error('pyobjects: priority must be an iterable') + else: + new_match_info = [] + for grain in cls.priority: + # Using list() here because we will be modifying + # match_info during iteration + for index, item in list(enumerate(match_info)): + try: + if item[0] == grain: + # Add item to new list + new_match_info.append(item) + # Clear item from old list + match_info[index] = None + except TypeError: + # Already moved this item to new list + pass + # Add in any remaining items not defined in priority + new_match_info.extend([x for x in match_info if x is not None]) + # Save reordered list as the match_info list + match_info = new_match_info + except AttributeError: + pass + + # Check for matches and update the attrs dict accordingly attrs = {} - for grain in match_groups: - filtered = Map.__salt__['grains.filter_by'](match_groups[grain], - grain=grain) - if filtered: - attrs.update(filtered) + if match_info: + grain_vals = Map.__salt__['grains.item'](*grain_targets) + for grain, match, match_attrs in match_info: + if grain not in grain_vals: + continue + if grain_vals[grain] == match: + attrs.update(match_attrs) if hasattr(cls, 'merge'): pillar = Map.__salt__['pillar.get'](cls.merge) diff --git a/salt/utils/raetevent.py b/salt/utils/raetevent.py index 2e75443b879..2ed1428e6f2 100644 --- a/salt/utils/raetevent.py +++ b/salt/utils/raetevent.py @@ -18,19 +18,28 @@ import salt.payload import salt.loader import salt.state import salt.utils.event -from salt.utils import kinds +import salt.utils.kinds as kinds from salt import transport from salt import syspaths -from raet import raeting, nacling -from raet.lane.stacking import LaneStack -from raet.lane.yarding import RemoteYard + +try: + from raet import raeting, nacling + from raet.lane.stacking import LaneStack + from raet.lane.yarding import RemoteYard + HAS_RAET = True +except ImportError: + HAS_RAET = False # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six log = logging.getLogger(__name__) +def __virtual__(): + return HAS_RAET + + class RAETEvent(object): ''' The base class used to manage salt events diff --git a/salt/utils/raetlane.py b/salt/utils/raetlane.py index 7d6ffa8a830..f76d51b9d32 100644 --- a/salt/utils/raetlane.py +++ b/salt/utils/raetlane.py @@ -56,7 +56,7 @@ import time # Import Salt Libs import logging -from salt.utils import kinds +import salt.utils.kinds as kinds log = logging.getLogger(__name__) diff --git a/salt/utils/reactor.py b/salt/utils/reactor.py index de2b0750a11..58c7e98c231 100644 --- a/salt/utils/reactor.py +++ b/salt/utils/reactor.py @@ -12,12 +12,13 @@ import salt.state import salt.utils import salt.utils.cache import salt.utils.event +import salt.utils.files import salt.utils.process import salt.defaults.exitcodes # Import 3rd-party libs import yaml -import salt.ext.six as six +from salt.ext import six log = logging.getLogger(__name__) @@ -89,7 +90,7 @@ class Reactor(salt.utils.process.SignalHandlingMultiprocessingProcess, salt.stat reactors = [] if isinstance(self.opts['reactor'], six.string_types): try: - with salt.utils.fopen(self.opts['reactor']) as fp_: + with salt.utils.files.fopen(self.opts['reactor']) as fp_: react_map = yaml.safe_load(fp_.read()) except (OSError, IOError): log.error( @@ -126,7 +127,7 @@ class Reactor(salt.utils.process.SignalHandlingMultiprocessingProcess, salt.stat if isinstance(self.minion.opts['reactor'], six.string_types): log.debug('Reading reactors from yaml {0}'.format(self.opts['reactor'])) try: - with salt.utils.fopen(self.opts['reactor']) as fp_: + with salt.utils.files.fopen(self.opts['reactor']) as fp_: react_map = yaml.safe_load(fp_.read()) except (OSError, IOError): log.error( @@ -273,12 +274,19 @@ class ReactWrap(object): try: f_call = salt.utils.format_call(l_fun, low) kwargs = f_call.get('kwargs', {}) + if 'arg' not in kwargs: + kwargs['arg'] = [] + if 'kwarg' not in kwargs: + kwargs['kwarg'] = {} # TODO: Setting the user doesn't seem to work for actual remote publishes if low['state'] in ('runner', 'wheel'): # Update called function's low data with event user to # segregate events fired by reactor and avoid reaction loops kwargs['__user__'] = self.event_user + # Replace ``state`` kwarg which comes from high data compiler. + # It breaks some runner functions and seems unnecessary. + kwargs['__state__'] = kwargs.pop('state') l_fun(*f_call.get('args', ()), **kwargs) except Exception: diff --git a/salt/utils/rsax931.py b/salt/utils/rsax931.py index b7ecfcd0b85..f827cc6db82 100644 --- a/salt/utils/rsax931.py +++ b/salt/utils/rsax931.py @@ -3,17 +3,18 @@ Create and verify ANSI X9.31 RSA signatures using OpenSSL libcrypto ''' -# python libs +# Import Python libs from __future__ import absolute_import import glob import sys import os -# salt libs -import salt.utils +# Import Salt libs +import salt.utils.platform +import salt.utils.stringutils -# 3rd-party libs -import salt.ext.six as six +# Import 3rd-party libs +from salt.ext import six from ctypes import cdll, c_char_p, c_int, c_void_p, pointer, create_string_buffer from ctypes.util import find_library @@ -29,13 +30,13 @@ def _load_libcrypto(): ''' if sys.platform.startswith('win'): return cdll.LoadLibrary('libeay32') - elif getattr(sys, 'frozen', False) and salt.utils.is_smartos(): + elif getattr(sys, 'frozen', False) and salt.utils.platform.is_smartos(): return cdll.LoadLibrary(glob.glob(os.path.join( os.path.dirname(sys.executable), 'libcrypto.so*'))[0]) else: lib = find_library('crypto') - if not lib and salt.utils.is_sunos(): + if not lib and salt.utils.platform.is_sunos(): # Solaris-like distribution that use pkgsrc have # libraries in a non standard location. # (SmartOS, OmniOS, OpenIndiana, ...) @@ -98,7 +99,7 @@ class RSAX931Signer(object): :param str keydata: The RSA private key in PEM format ''' - keydata = salt.utils.to_bytes(keydata, 'ascii') + keydata = salt.utils.stringutils.to_bytes(keydata, 'ascii') self._bio = libcrypto.BIO_new_mem_buf(keydata, len(keydata)) self._rsa = c_void_p(libcrypto.RSA_new()) if not libcrypto.PEM_read_bio_RSAPrivateKey(self._bio, pointer(self._rsa), None, None): @@ -118,7 +119,7 @@ class RSAX931Signer(object): ''' # Allocate a buffer large enough for the signature. Freed by ctypes. buf = create_string_buffer(libcrypto.RSA_size(self._rsa)) - msg = salt.utils.to_bytes(msg) + msg = salt.utils.stringutils.to_bytes(msg) size = libcrypto.RSA_private_encrypt(len(msg), msg, buf, self._rsa, RSA_X931_PADDING) if size < 0: raise ValueError('Unable to encrypt message') @@ -135,7 +136,7 @@ class RSAX931Verifier(object): :param str pubdata: The RSA public key in PEM format ''' - pubdata = salt.utils.to_bytes(pubdata, 'ascii') + pubdata = salt.utils.stringutils.to_bytes(pubdata, 'ascii') pubdata = pubdata.replace(six.b('RSA '), six.b('')) self._bio = libcrypto.BIO_new_mem_buf(pubdata, len(pubdata)) self._rsa = c_void_p(libcrypto.RSA_new()) @@ -157,7 +158,7 @@ class RSAX931Verifier(object): ''' # Allocate a buffer large enough for the signature. Freed by ctypes. buf = create_string_buffer(libcrypto.RSA_size(self._rsa)) - signed = salt.utils.to_bytes(signed) + signed = salt.utils.stringutils.to_bytes(signed) size = libcrypto.RSA_public_decrypt(len(signed), signed, buf, self._rsa, RSA_X931_PADDING) if size < 0: raise ValueError('Unable to decrypt message') diff --git a/salt/utils/s3.py b/salt/utils/s3.py index bc12142254e..0bb198d3a64 100644 --- a/salt/utils/s3.py +++ b/salt/utils/s3.py @@ -19,6 +19,7 @@ except ImportError: # Import Salt libs import salt.utils import salt.utils.aws +import salt.utils.files import salt.utils.xmlutil as xml from salt._compat import ElementTree as ET from salt.exceptions import CommandExecutionError @@ -150,7 +151,7 @@ def query(key, keyid, method='GET', params=None, headers=None, try: if method == 'PUT': if local_file: - data = salt.utils.fopen(local_file, 'r') # pylint: disable=resource-leakage + data = salt.utils.files.fopen(local_file, 'r') # pylint: disable=resource-leakage result = requests.request(method, requesturl, headers=headers, @@ -233,7 +234,7 @@ def query(key, keyid, method='GET', params=None, headers=None, 'Failed to get file. {0}: {1}'.format(err_code, err_msg)) log.debug('Saving to local file: {0}'.format(local_file)) - with salt.utils.fopen(local_file, 'wb') as out: + with salt.utils.files.fopen(local_file, 'wb') as out: for chunk in result.iter_content(chunk_size=chunk_size): out.write(chunk) return 'Saved to local file: {0}'.format(local_file) diff --git a/salt/utils/schedule.py b/salt/utils/schedule.py index ad93d997238..d29a3bf3148 100644 --- a/salt/utils/schedule.py +++ b/salt/utils/schedule.py @@ -333,14 +333,20 @@ import logging import errno import random import yaml +import copy # Import Salt libs import salt.config -import salt.utils -import salt.utils.jid -import salt.utils.process +import salt.utils # Can be removed once appendproctitle and daemonize_if are moved import salt.utils.args +import salt.utils.error +import salt.utils.event +import salt.utils.files +import salt.utils.jid import salt.utils.minion +import salt.utils.platform +import salt.utils.process +import salt.utils.stringutils import salt.loader import salt.minion import salt.payload @@ -349,11 +355,10 @@ import salt.exceptions import salt.log.setup as log_setup import salt.defaults.exitcodes from salt.utils.odict import OrderedDict -from salt.utils.process import os_is_running, default_signals, SignalHandlingMultiprocessingProcess from salt.utils.yamldumper import SafeOrderedDumper # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six # pylint: disable=import-error try: @@ -380,7 +385,7 @@ class Schedule(object): ''' instance = None - def __new__(cls, opts, functions, returners=None, intervals=None, cleanup=None, proxy=None): + def __new__(cls, opts, functions, returners=None, intervals=None, cleanup=None, proxy=None, standalone=False): ''' Only create one instance of Schedule ''' @@ -390,33 +395,36 @@ class Schedule(object): # it in a WeakValueDictionary-- which will remove the item if no one # references it-- this forces a reference while we return to the caller cls.instance = object.__new__(cls) - cls.instance.__singleton_init__(opts, functions, returners, intervals, cleanup, proxy) + cls.instance.__singleton_init__(opts, functions, returners, intervals, cleanup, proxy, standalone) else: log.debug('Re-using Schedule') return cls.instance # has to remain empty for singletons, since __init__ will *always* be called - def __init__(self, opts, functions, returners=None, intervals=None, cleanup=None, proxy=None): + def __init__(self, opts, functions, returners=None, intervals=None, cleanup=None, proxy=None, standalone=False): pass # an init for the singleton instance to call - def __singleton_init__(self, opts, functions, returners=None, intervals=None, cleanup=None, proxy=None): + def __singleton_init__(self, opts, functions, returners=None, intervals=None, cleanup=None, proxy=None, standalone=False): self.opts = opts self.proxy = proxy self.functions = functions + self.standalone = standalone if isinstance(intervals, dict): self.intervals = intervals else: self.intervals = {} - if hasattr(returners, '__getitem__'): - self.returners = returners - else: - self.returners = returners.loader.gen_functions() + if not self.standalone: + if hasattr(returners, '__getitem__'): + self.returners = returners + else: + self.returners = returners.loader.gen_functions() self.time_offset = self.functions.get('timezone.get_offset', lambda: '0000')() self.schedule_returner = self.option('schedule_returner') # Keep track of the lowest loop interval needed in this variable self.loop_interval = six.MAXSIZE - clean_proc_dir(opts) + if not self.standalone: + clean_proc_dir(opts) if cleanup: for prefix in cleanup: self.delete_job_prefix(prefix) @@ -472,9 +480,9 @@ class Schedule(object): schedule_conf = os.path.join(minion_d_dir, '_schedule.conf') log.debug('Persisting schedule') try: - with salt.utils.fopen(schedule_conf, 'wb+') as fp_: + with salt.utils.files.fopen(schedule_conf, 'wb+') as fp_: fp_.write( - salt.utils.to_bytes( + salt.utils.stringutils.to_bytes( yaml.dump( {'schedule': self._get_schedule(include_pillar=False)}, Dumper=SafeOrderedDumper @@ -663,12 +671,12 @@ class Schedule(object): multiprocessing_enabled = self.opts.get('multiprocessing', True) if multiprocessing_enabled: - thread_cls = SignalHandlingMultiprocessingProcess + thread_cls = salt.utils.process.SignalHandlingMultiprocessingProcess else: thread_cls = threading.Thread if multiprocessing_enabled: - with default_signals(signal.SIGINT, signal.SIGTERM): + with salt.utils.process.default_signals(signal.SIGINT, signal.SIGTERM): proc = thread_cls(target=self.handle_func, args=(multiprocessing_enabled, func, data)) # Reset current signals before starting the process in # order not to inherit the current signal handlers @@ -742,7 +750,8 @@ class Schedule(object): ''' Execute this method in a multiprocess or thread ''' - if salt.utils.is_windows() or self.opts.get('transport') == 'zeromq': + if salt.utils.platform.is_windows() \ + or self.opts.get('transport') == 'zeromq': # Since function references can't be pickled and pickling # is required when spawning new processes on Windows, regenerate # the functions and returners. @@ -758,7 +767,7 @@ class Schedule(object): 'fun': func, 'fun_args': [], 'schedule': data['name'], - 'jid': salt.utils.jid.gen_jid()} + 'jid': salt.utils.jid.gen_jid(self.opts)} if 'metadata' in data: if isinstance(data['metadata'], dict): @@ -772,37 +781,39 @@ class Schedule(object): salt.utils.appendproctitle('{0} {1}'.format(self.__class__.__name__, ret['jid'])) - proc_fn = os.path.join( - salt.minion.get_proc_dir(self.opts['cachedir']), - ret['jid'] - ) + if not self.standalone: + proc_fn = os.path.join( + salt.minion.get_proc_dir(self.opts['cachedir']), + ret['jid'] + ) - # Check to see if there are other jobs with this - # signature running. If there are more than maxrunning - # jobs present then don't start another. - # If jid_include is False for this job we can ignore all this - # NOTE--jid_include defaults to True, thus if it is missing from the data - # dict we treat it like it was there and is True - if 'jid_include' not in data or data['jid_include']: - jobcount = 0 - for job in salt.utils.minion.running(self.opts): - if 'schedule' in job: - log.debug('schedule.handle_func: Checking job against ' - 'fun {0}: {1}'.format(ret['fun'], job)) - if ret['schedule'] == job['schedule'] and os_is_running(job['pid']): - jobcount += 1 - log.debug( - 'schedule.handle_func: Incrementing jobcount, now ' - '{0}, maxrunning is {1}'.format( - jobcount, data['maxrunning'])) - if jobcount >= data['maxrunning']: + # Check to see if there are other jobs with this + # signature running. If there are more than maxrunning + # jobs present then don't start another. + # If jid_include is False for this job we can ignore all this + # NOTE--jid_include defaults to True, thus if it is missing from the data + # dict we treat it like it was there and is True + if 'jid_include' not in data or data['jid_include']: + jobcount = 0 + for job in salt.utils.minion.running(self.opts): + if 'schedule' in job: + log.debug('schedule.handle_func: Checking job against ' + 'fun {0}: {1}'.format(ret['fun'], job)) + if ret['schedule'] == job['schedule'] \ + and salt.utils.process.os_is_running(job['pid']): + jobcount += 1 log.debug( - 'schedule.handle_func: The scheduled job {0} ' - 'was not started, {1} already running'.format( - ret['schedule'], data['maxrunning'])) - return False + 'schedule.handle_func: Incrementing jobcount, now ' + '{0}, maxrunning is {1}'.format( + jobcount, data['maxrunning'])) + if jobcount >= data['maxrunning']: + log.debug( + 'schedule.handle_func: The scheduled job {0} ' + 'was not started, {1} already running'.format( + ret['schedule'], data['maxrunning'])) + return False - if multiprocessing_enabled and not salt.utils.is_windows(): + if multiprocessing_enabled and not salt.utils.platform.is_windows(): # Reconfigure multiprocessing logging after daemonizing log_setup.setup_multiprocessing_logging() @@ -813,12 +824,13 @@ class Schedule(object): try: ret['pid'] = os.getpid() - if 'jid_include' not in data or data['jid_include']: - log.debug('schedule.handle_func: adding this job to the jobcache ' - 'with data {0}'.format(ret)) - # write this to /var/cache/salt/minion/proc - with salt.utils.fopen(proc_fn, 'w+b') as fp_: - fp_.write(salt.payload.Serial(self.opts).dumps(ret)) + if not self.standalone: + if 'jid_include' not in data or data['jid_include']: + log.debug('schedule.handle_func: adding this job to the jobcache ' + 'with data {0}'.format(ret)) + # write this to /var/cache/salt/minion/proc + with salt.utils.files.fopen(proc_fn, 'w+b') as fp_: + fp_.write(salt.payload.Serial(self.opts).dumps(ret)) args = tuple() if 'args' in data: @@ -841,40 +853,42 @@ class Schedule(object): if argspec.keywords: # this function accepts **kwargs, pack in the publish data for key, val in six.iteritems(ret): - kwargs['__pub_{0}'.format(key)] = val + if key is not 'kwargs': + kwargs['__pub_{0}'.format(key)] = copy.deepcopy(val) ret['return'] = self.functions[func](*args, **kwargs) - data_returner = data.get('returner', None) - if data_returner or self.schedule_returner: - if 'return_config' in data: - ret['ret_config'] = data['return_config'] - if 'return_kwargs' in data: - ret['ret_kwargs'] = data['return_kwargs'] - rets = [] - for returner in [data_returner, self.schedule_returner]: - if isinstance(returner, str): - rets.append(returner) - elif isinstance(returner, list): - rets.extend(returner) - # simple de-duplication with order retained - for returner in OrderedDict.fromkeys(rets): - ret_str = '{0}.returner'.format(returner) - if ret_str in self.returners: - ret['success'] = True - self.returners[ret_str](ret) - else: - log.info( - 'Job {0} using invalid returner: {1}. Ignoring.'.format( - func, returner + if not self.standalone: + # runners do not provide retcode + if 'retcode' in self.functions.pack['__context__']: + ret['retcode'] = self.functions.pack['__context__']['retcode'] + + ret['success'] = True + + data_returner = data.get('returner', None) + if data_returner or self.schedule_returner: + if 'return_config' in data: + ret['ret_config'] = data['return_config'] + if 'return_kwargs' in data: + ret['ret_kwargs'] = data['return_kwargs'] + rets = [] + for returner in [data_returner, self.schedule_returner]: + if isinstance(returner, six.string_types): + rets.append(returner) + elif isinstance(returner, list): + rets.extend(returner) + # simple de-duplication with order retained + for returner in OrderedDict.fromkeys(rets): + ret_str = '{0}.returner'.format(returner) + if ret_str in self.returners: + self.returners[ret_str](ret) + else: + log.info( + 'Job {0} using invalid returner: {1}. Ignoring.'.format( + func, returner + ) ) - ) - # runners do not provide retcode - if 'retcode' in self.functions.pack['__context__']: - ret['retcode'] = self.functions.pack['__context__']['retcode'] - - ret['success'] = True except Exception: log.exception("Unhandled exception running {0}".format(ret['fun'])) # Although catch-all exception handlers are bad, the exception here @@ -885,9 +899,10 @@ class Schedule(object): ret['success'] = False ret['retcode'] = 254 finally: - # Only attempt to return data to the master - # if the scheduled job is running on a minion. - if '__role' in self.opts and self.opts['__role'] == 'minion': + # Only attempt to return data to the master if the scheduled job is running + # on a master itself or a minion. + if '__role' in self.opts and self.opts['__role'] in ('master', 'minion'): + # The 'return_job' option is enabled by default even if not set if 'return_job' in data and not data['return_job']: pass else: @@ -914,23 +929,25 @@ class Schedule(object): except Exception as exc: log.exception("Unhandled exception firing event: {0}".format(exc)) - log.debug('schedule.handle_func: Removing {0}'.format(proc_fn)) - try: - os.unlink(proc_fn) - except OSError as exc: - if exc.errno == errno.EEXIST or exc.errno == errno.ENOENT: - # EEXIST and ENOENT are OK because the file is gone and that's what - # we wanted - pass - else: - log.error("Failed to delete '{0}': {1}".format(proc_fn, exc.errno)) - # Otherwise, failing to delete this file is not something - # we can cleanly handle. - raise - finally: - if multiprocessing_enabled: - # Let's make sure we exit the process! - sys.exit(salt.defaults.exitcodes.EX_GENERIC) + if not self.standalone: + log.debug('schedule.handle_func: Removing {0}'.format(proc_fn)) + + try: + os.unlink(proc_fn) + except OSError as exc: + if exc.errno == errno.EEXIST or exc.errno == errno.ENOENT: + # EEXIST and ENOENT are OK because the file is gone and that's what + # we wanted + pass + else: + log.error("Failed to delete '{0}': {1}".format(proc_fn, exc.errno)) + # Otherwise, failing to delete this file is not something + # we can cleanly handle. + raise + finally: + if multiprocessing_enabled: + # Let's make sure we exit the process! + sys.exit(salt.defaults.exitcodes.EX_GENERIC) def eval(self): ''' @@ -1127,21 +1144,28 @@ class Schedule(object): log.error('Invalid date string {0}. ' 'Ignoring job {1}.'.format(i, job)) continue - when = int(time.mktime(when__.timetuple())) - if when >= now: - _when.append(when) + _when.append(int(time.mktime(when__.timetuple()))) if data['_splay']: _when.append(data['_splay']) + # Sort the list of "whens" from earlier to later schedules _when.sort() + + for i in _when: + if i < now and len(_when) > 1: + # Remove all missed schedules except the latest one. + # We need it to detect if it was triggered previously. + _when.remove(i) + if _when: - # Grab the first element - # which is the next run time + # Grab the first element, which is the next run time or + # last scheduled time in the past. when = _when[0] if '_run' not in data: - data['_run'] = True + # Prevent run of jobs from the past + data['_run'] = bool(when >= now) if not data['_next_fire_time']: data['_next_fire_time'] = when @@ -1209,8 +1233,9 @@ class Schedule(object): log.error('Missing python-croniter. Ignoring job {0}'.format(job)) continue - if not data['_next_fire_time'] or \ - data['_next_fire_time'] < now: + if data['_next_fire_time'] is None: + # Get next time frame for a "cron" job if it has been never + # executed before or already executed in the past. try: data['_next_fire_time'] = int( croniter.croniter(data['cron'], now).get_next()) @@ -1218,6 +1243,14 @@ class Schedule(object): log.error('Invalid cron string. Ignoring') continue + # If next job run is scheduled more than 1 minute ahead and + # configured loop interval is longer than that, we should + # shorten it to get our job executed closer to the beginning + # of desired time. + interval = now - data['_next_fire_time'] + if interval >= 60 and interval < self.loop_interval: + self.loop_interval = interval + else: continue @@ -1231,6 +1264,11 @@ class Schedule(object): elif 'when' in data and data['_run']: data['_run'] = False run = True + elif 'cron' in data: + # Reset next scheduled time because it is in the past now, + # and we should trigger the job run, then wait for the next one. + data['_next_fire_time'] = None + run = True elif seconds == 0: run = True @@ -1308,7 +1346,7 @@ class Schedule(object): multiprocessing_enabled = self.opts.get('multiprocessing', True) - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): # Temporarily stash our function references. # You can't pickle function references, and pickling is # required when spawning new processes on Windows. @@ -1318,13 +1356,13 @@ class Schedule(object): self.returners = {} try: if multiprocessing_enabled: - thread_cls = SignalHandlingMultiprocessingProcess + thread_cls = salt.utils.process.SignalHandlingMultiprocessingProcess else: thread_cls = threading.Thread proc = thread_cls(target=self.handle_func, args=(multiprocessing_enabled, func, data)) if multiprocessing_enabled: - with default_signals(signal.SIGINT, signal.SIGTERM): + with salt.utils.process.default_signals(signal.SIGINT, signal.SIGTERM): # Reset current signals before starting the process in # order not to inherit the current signal handlers proc.start() @@ -1337,7 +1375,7 @@ class Schedule(object): if '_seconds' in data: data['_next_fire_time'] = now + data['_seconds'] data['_splay'] = None - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): # Restore our function references. self.functions = functions self.returners = returners @@ -1352,13 +1390,13 @@ def clean_proc_dir(opts): for basefilename in os.listdir(salt.minion.get_proc_dir(opts['cachedir'])): fn_ = os.path.join(salt.minion.get_proc_dir(opts['cachedir']), basefilename) - with salt.utils.fopen(fn_, 'rb') as fp_: + with salt.utils.files.fopen(fn_, 'rb') as fp_: job = None try: job = salt.payload.Serial(opts).load(fp_) except Exception: # It's corrupted # Windows cannot delete an open file - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): fp_.close() try: os.unlink(fn_) @@ -1373,7 +1411,7 @@ def clean_proc_dir(opts): 'pid {0} still exists.'.format(job['pid'])) else: # Windows cannot delete an open file - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): fp_.close() # Maybe the file is already gone try: diff --git a/salt/utils/schema.py b/salt/utils/schema.py index 6f1d824b3ac..7ce3371ee88 100644 --- a/salt/utils/schema.py +++ b/salt/utils/schema.py @@ -332,7 +332,7 @@ from salt.utils.odict import OrderedDict # Import 3rd-party libs #import yaml -import salt.ext.six as six +from salt.ext import six BASE_SCHEMA_URL = 'https://non-existing.saltstack.com/schemas' RENDER_COMMENT_YAML_MAX_LINE_LENGTH = 80 diff --git a/salt/utils/shlex.py b/salt/utils/shlex.py new file mode 100644 index 00000000000..5b45174d834 --- /dev/null +++ b/salt/utils/shlex.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +''' +Modified versions of functions from shlex module +''' +from __future__ import absolute_import + +# Import Python libs +import shlex + +# Import 3rd-party libs +from salt.ext import six + + +def split(s, **kwargs): + ''' + Only split if variable is a string + ''' + if isinstance(s, six.string_types): + return shlex.split(s, **kwargs) + else: + return s diff --git a/salt/utils/smb.py b/salt/utils/smb.py index bc4bdc27036..f5f21f1c8f8 100644 --- a/salt/utils/smb.py +++ b/salt/utils/smb.py @@ -8,6 +8,7 @@ Utility functions for SMB connections from __future__ import absolute_import # Import python libs +import salt.utils.files import logging log = logging.getLogger(__name__) @@ -96,3 +97,24 @@ def put_str(content, path, share='C$', conn=None, host=None, username=None, pass fh_ = StrHandle(content) conn.putFile(share, path, fh_.string) + + +def put_file(local_path, path, share='C$', conn=None, host=None, username=None, password=None): + ''' + Wrapper around impacket.smbconnection.putFile() that allows a file to be + uploaded + + Example usage: + + import salt.utils.smb + smb_conn = salt.utils.smb.get_conn('10.0.0.45', 'vagrant', 'vagrant') + salt.utils.smb.put_file('/root/test.pdf', 'temp\\myfiles\\test1.pdf', conn=smb_conn) + ''' + if conn is None: + conn = get_conn(host, username, password) + + if conn is False: + return False + + with salt.utils.files.fopen(local_path, 'rb') as fh_: + conn.putFile(share, path, fh_.read) diff --git a/salt/utils/state.py b/salt/utils/state.py new file mode 100644 index 00000000000..3251e6b3bdf --- /dev/null +++ b/salt/utils/state.py @@ -0,0 +1,213 @@ +# -*- coding: utf-8 -*- +''' +Utility functions for state functions + +.. versionadded:: Oxygen +''' + +# Import Python Libs +from __future__ import absolute_import + +# Import Salt libs +from salt.ext import six +import salt.state + +_empty = object() + + +def gen_tag(low): + ''' + Generate the running dict tag string from the low data structure + ''' + return '{0[state]}_|-{0[__id__]}_|-{0[name]}_|-{0[fun]}'.format(low) + + +def search_onfail_requisites(sid, highstate): + ''' + For a particular low chunk, search relevant onfail related states + ''' + onfails = [] + if '_|-' in sid: + st = salt.state.split_low_tag(sid) + else: + st = {'__id__': sid} + for fstate, fchunks in six.iteritems(highstate): + if fstate == st['__id__']: + continue + else: + for mod_, fchunk in six.iteritems(fchunks): + if ( + not isinstance(mod_, six.string_types) or + mod_.startswith('__') + ): + continue + else: + if not isinstance(fchunk, list): + continue + else: + # bydefault onfail will fail, but you can + # set onfail_stop: False to prevent the highstate + # to stop if you handle it + onfail_handled = False + for fdata in fchunk: + if not isinstance(fdata, dict): + continue + onfail_handled = (fdata.get('onfail_stop', True) + is False) + if onfail_handled: + break + if not onfail_handled: + continue + for fdata in fchunk: + if not isinstance(fdata, dict): + continue + for knob, fvalue in six.iteritems(fdata): + if knob != 'onfail': + continue + for freqs in fvalue: + for fmod, fid in six.iteritems(freqs): + if not ( + fid == st['__id__'] and + fmod == st.get('state', fmod) + ): + continue + onfails.append((fstate, mod_, fchunk)) + return onfails + + +def check_onfail_requisites(state_id, state_result, running, highstate): + ''' + When a state fail and is part of a highstate, check + if there is onfail requisites. + When we find onfail requisites, we will consider the state failed + only if at least one of those onfail requisites also failed + + Returns: + + True: if onfail handlers suceeded + False: if one on those handler failed + None: if the state does not have onfail requisites + + ''' + nret = None + if ( + state_id and state_result and + highstate and isinstance(highstate, dict) + ): + onfails = search_onfail_requisites(state_id, highstate) + if onfails: + for handler in onfails: + fstate, mod_, fchunk = handler + for rstateid, rstate in six.iteritems(running): + if '_|-' in rstateid: + st = salt.state.split_low_tag(rstateid) + # in case of simple state, try to guess + else: + id_ = rstate.get('__id__', rstateid) + if not id_: + raise ValueError('no state id') + st = {'__id__': id_, 'state': mod_} + if mod_ == st['state'] and fstate == st['__id__']: + ofresult = rstate.get('result', _empty) + if ofresult in [False, True]: + nret = ofresult + if ofresult is False: + # as soon as we find an errored onfail, we stop + break + # consider that if we parsed onfailes without changing + # the ret, that we have failed + if nret is None: + nret = False + return nret + + +def check_result(running, recurse=False, highstate=None): + ''' + Check the total return value of the run and determine if the running + dict has any issues + ''' + if not isinstance(running, dict): + return False + + if not running: + return False + + ret = True + for state_id, state_result in six.iteritems(running): + if not recurse and not isinstance(state_result, dict): + ret = False + if ret and isinstance(state_result, dict): + result = state_result.get('result', _empty) + if result is False: + ret = False + # only override return value if we are not already failed + elif result is _empty and isinstance(state_result, dict) and ret: + ret = check_result( + state_result, recurse=True, highstate=highstate) + # if we detect a fail, check for onfail requisites + if not ret: + # ret can be None in case of no onfail reqs, recast it to bool + ret = bool(check_onfail_requisites(state_id, state_result, + running, highstate)) + # return as soon as we got a failure + if not ret: + break + return ret + + +def merge_subreturn(original_return, sub_return, subkey=None): + ''' + Update an existing state return (`original_return`) in place + with another state return (`sub_return`), i.e. for a subresource. + + Returns: + dict: The updated state return. + + The existing state return does not need to have all the required fields, + as this is meant to be called from the internals of a state function, + but any existing data will be kept and respected. + + It is important after using this function to check the return value + to see if it is False, in which case the main state should return. + Prefer to check `_ret['result']` instead of `ret['result']`, + as the latter field may not yet be populated. + + Code Example: + + .. code-block:: python + def state_func(name, config, alarm=None): + ret = {'name': name, 'comment': '', 'changes': {}} + if alarm: + _ret = __states__['subresource.managed'](alarm) + __utils__['state.merge_subreturn'](ret, _ret) + if _ret['result'] is False: + return ret + ''' + if not subkey: + subkey = sub_return['name'] + + if sub_return['result'] is False: + # True or None stay the same + original_return['result'] = sub_return['result'] + + sub_comment = sub_return['comment'] + if not isinstance(sub_comment, list): + sub_comment = [sub_comment] + original_return.setdefault('comment', []) + if isinstance(original_return['comment'], list): + original_return['comment'].extend(sub_comment) + else: + if original_return['comment']: + # Skip for empty original comments + original_return['comment'] += u'\n' + original_return['comment'] += u'\n'.join(sub_comment) + + if sub_return['changes']: # changes always exists + original_return.setdefault('changes', {}) + original_return['changes'][subkey] = sub_return['changes'] + + if sub_return.get('pchanges'): # pchanges may or may not exist + original_return.setdefault('pchanges', {}) + original_return['pchanges'][subkey] = sub_return['pchanges'] + + return original_return diff --git a/salt/utils/stormpath.py b/salt/utils/stormpath.py deleted file mode 100644 index 516e2d57a72..00000000000 --- a/salt/utils/stormpath.py +++ /dev/null @@ -1,79 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Support for Stormpath - -.. versionadded:: 2015.8.0 -''' - -# Import python libs -from __future__ import absolute_import, print_function -import logging - -# Import salt libs -import salt.utils.http - -log = logging.getLogger(__name__) - - -def query(action=None, - command=None, - args=None, - method='GET', - header_dict=None, - data=None, - opts=None): - ''' - Make a web call to Stormpath - - .. versionadded:: 2015.8.0 - ''' - if opts is None: - opts = {} - - apiid = opts.get('stormpath', {}).get('apiid', None) - apikey = opts.get('stormpath', {}).get('apikey', None) - path = 'https://api.stormpath.com/v1/' - - if action: - path += action - - if command: - path += '/{0}'.format(command) - - log.debug('Stormpath URL: {0}'.format(path)) - - if not isinstance(args, dict): - args = {} - - if header_dict is None: - header_dict = {} - - if method != 'POST': - header_dict['Accept'] = 'application/json' - - decode = True - if method == 'DELETE': - decode = False - - return_content = None - result = salt.utils.http.query( - path, - method, - username=apiid, - password=apikey, - params=args, - data=data, - header_dict=header_dict, - decode=decode, - decode_type='json', - text=True, - status=True, - opts=opts, - ) - log.debug( - 'Stormpath Response Status Code: {0}'.format( - result['status'] - ) - ) - - return [result['status'], result.get('dict', {})] diff --git a/salt/utils/stringutils.py b/salt/utils/stringutils.py new file mode 100644 index 00000000000..89e91b0be7a --- /dev/null +++ b/salt/utils/stringutils.py @@ -0,0 +1,199 @@ +# -*- coding: utf-8 -*- + +# Import Python libs +from __future__ import absolute_import +import os +import string + +# Import Salt libs +from salt.utils.decorators.jinja import jinja_filter + +# Import 3rd-party libs +from salt.ext import six +from salt.ext.six.moves import range # pylint: disable=redefined-builtin + + +@jinja_filter('to_bytes') +def to_bytes(s, encoding=None): + ''' + Given bytes, bytearray, str, or unicode (python 2), return bytes (str for + python 2) + ''' + if six.PY3: + if isinstance(s, bytes): + return s + if isinstance(s, bytearray): + return bytes(s) + if isinstance(s, six.string_types): + return s.encode(encoding or __salt_system_encoding__) + raise TypeError('expected bytes, bytearray, or str') + else: + return to_str(s, encoding) + + +def to_str(s, encoding=None): + ''' + Given str, bytes, bytearray, or unicode (py2), return str + ''' + # This shouldn't be six.string_types because if we're on PY2 and we already + # have a string, we should just return it. + if isinstance(s, str): + return s + if six.PY3: + if isinstance(s, (bytes, bytearray)): + # https://docs.python.org/3/howto/unicode.html#the-unicode-type + # replace error with U+FFFD, REPLACEMENT CHARACTER + return s.decode(encoding or __salt_system_encoding__, "replace") + raise TypeError('expected str, bytes, or bytearray not {}'.format(type(s))) + else: + if isinstance(s, bytearray): + return str(s) + if isinstance(s, unicode): # pylint: disable=incompatible-py3-code,undefined-variable + return s.encode(encoding or __salt_system_encoding__) + raise TypeError('expected str, bytearray, or unicode') + + +def to_unicode(s, encoding=None): + ''' + Given str or unicode, return unicode (str for python 3) + ''' + if not isinstance(s, (bytes, bytearray, six.string_types)): + return s + if six.PY3: + if isinstance(s, (bytes, bytearray)): + return to_str(s, encoding) + else: + # This needs to be str and not six.string_types, since if the string is + # already a unicode type, it does not need to be decoded (and doing so + # will raise an exception). + if isinstance(s, str): + return s.decode(encoding or __salt_system_encoding__) + return s + + +@jinja_filter('str_to_num') # Remove this for Neon +@jinja_filter('to_num') +def to_num(text): + ''' + Convert a string to a number. + Returns an integer if the string represents an integer, a floating + point number if the string is a real number, or the string unchanged + otherwise. + ''' + try: + return int(text) + except ValueError: + try: + return float(text) + except ValueError: + return text + + +def to_none(text): + ''' + Convert a string to None if the string is empty or contains only spaces. + ''' + if str(text).strip(): + return text + return None + + +def is_quoted(value): + ''' + Return a single or double quote, if a string is wrapped in extra quotes. + Otherwise return an empty string. + ''' + ret = '' + if isinstance(value, six.string_types) \ + and value[0] == value[-1] \ + and value.startswith(('\'', '"')): + ret = value[0] + return ret + + +def dequote(value): + ''' + Remove extra quotes around a string. + ''' + if is_quoted(value): + return value[1:-1] + return value + + +@jinja_filter('is_hex') +def is_hex(value): + ''' + Returns True if value is a hexidecimal string, otherwise returns False + ''' + try: + int(value, 16) + return True + except (TypeError, ValueError): + return False + + +def is_binary(data): + ''' + Detects if the passed string of data is binary or text + ''' + if '\0' in data: + return True + if not data: + return False + + text_characters = ''.join([chr(x) for x in range(32, 127)] + list('\n\r\t\b')) + # Get the non-text characters (map each character to itself then use the + # 'remove' option to get rid of the text characters.) + if six.PY3: + trans = ''.maketrans('', '', text_characters) + nontext = data.translate(trans) + else: + trans = string.maketrans('', '') # pylint: disable=no-member + nontext = data.translate(trans, text_characters) + + # If more than 30% non-text characters, then + # this is considered binary data + if float(len(nontext)) / len(data) > 0.30: + return True + return False + + +@jinja_filter('random_str') +def random(size=32): + key = os.urandom(size) + return key.encode('base64').replace('\n', '')[:size] + + +@jinja_filter('contains_whitespace') +def contains_whitespace(text): + ''' + Returns True if there are any whitespace characters in the string + ''' + return any(x.isspace() for x in text) + + +def human_to_bytes(size): + ''' + Given a human-readable byte string (e.g. 2G, 30M), + return the number of bytes. Will return 0 if the argument has + unexpected form. + + .. versionadded:: Oxygen + ''' + sbytes = size[:-1] + unit = size[-1] + if sbytes.isdigit(): + sbytes = int(sbytes) + if unit == 'P': + sbytes *= 1125899906842624 + elif unit == 'T': + sbytes *= 1099511627776 + elif unit == 'G': + sbytes *= 1073741824 + elif unit == 'M': + sbytes *= 1048576 + else: + sbytes = 0 + else: + sbytes = 0 + return sbytes diff --git a/salt/utils/systemd.py b/salt/utils/systemd.py index 471d9bb241a..b41dce546bd 100644 --- a/salt/utils/systemd.py +++ b/salt/utils/systemd.py @@ -11,6 +11,7 @@ import subprocess # Import Salt libs from salt.exceptions import SaltInvocationError import salt.utils +import salt.utils.stringutils log = logging.getLogger(__name__) @@ -63,7 +64,7 @@ def version(context=None): ['systemctl', '--version'], close_fds=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate()[0] - outstr = salt.utils.to_str(stdout) + outstr = salt.utils.stringutils.to_str(stdout) try: ret = int(outstr.splitlines()[0].split()[-1]) except (IndexError, ValueError): diff --git a/salt/utils/templates.py b/salt/utils/templates.py index 04076a22add..bbb834bb412 100644 --- a/salt/utils/templates.py +++ b/salt/utils/templates.py @@ -5,24 +5,32 @@ Template render systems from __future__ import absolute_import -# Import python libs +# Import Python libs import codecs import os -import imp import logging import tempfile import traceback import sys -# Import third party libs +# Import 3rd-party libs import jinja2 import jinja2.ext -import salt.ext.six as six +from salt.ext import six -# Import salt libs +if sys.version_info[:2] >= (3, 5): + import importlib.machinery # pylint: disable=no-name-in-module,import-error + import importlib.util # pylint: disable=no-name-in-module,import-error + USE_IMPORTLIB = True +else: + import imp + USE_IMPORTLIB = False + +# Import Salt libs import salt.utils import salt.utils.http import salt.utils.files +import salt.utils.platform import salt.utils.yamlencoding import salt.utils.locales import salt.utils.hashutils @@ -32,7 +40,7 @@ from salt.exceptions import ( import salt.utils.jinja import salt.utils.network from salt.utils.odict import OrderedDict -from salt.utils.decorators import JinjaFilter, JinjaTest +from salt.utils.decorators.jinja import JinjaFilter, JinjaTest, JinjaGlobal from salt import __path__ as saltpath log = logging.getLogger(__name__) @@ -66,6 +74,9 @@ class AliasedLoader(object): def __getattr__(self, name): return getattr(self.wrapped, name) + def __contains__(self, name): + return name in self.wrapped + class AliasedModule(object): ''' @@ -158,7 +169,7 @@ def wrap_tmpl_func(render_str): output = render_str(tmplstr, context, tmplpath) if six.PY2: output = output.encode(SLS_ENCODING) - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): # Write out with Windows newlines output = os.linesep.join(output.splitlines()) @@ -262,7 +273,7 @@ def _get_jinja_error(trace, context=None): if add_log: if template_path: out = '\n{0}\n'.format(msg.splitlines()[0]) - with salt.utils.fopen(template_path) as fp_: + with salt.utils.files.fopen(template_path) as fp_: template_contents = fp_.read() out += salt.utils.get_context( template_contents, @@ -322,6 +333,7 @@ def render_jinja_tmpl(tmplstr, context, tmplpath=None): jinja_env.tests.update(JinjaTest.salt_jinja_tests) jinja_env.filters.update(JinjaFilter.salt_jinja_filters) + jinja_env.globals.update(JinjaGlobal.salt_jinja_globals) # globals jinja_env.globals['odict'] = OrderedDict @@ -346,9 +358,10 @@ def render_jinja_tmpl(tmplstr, context, tmplpath=None): line, out = _get_jinja_error(trace, context=decoded_context) if not line: tmplstr = '' - raise SaltRenderError('Jinja syntax error: {0}{1}'.format(exc, out), - line, - tmplstr) + raise SaltRenderError( + 'Jinja syntax error: {0}{1}'.format(exc, out), + line, + tmplstr) except jinja2.exceptions.UndefinedError as exc: trace = traceback.extract_tb(sys.exc_info()[2]) out = _get_jinja_error(trace, context=decoded_context)[1] @@ -483,10 +496,22 @@ def py(sfn, string=False, **kwargs): # pylint: disable=C0103 if not os.path.isfile(sfn): return {} - mod = imp.load_source( - os.path.basename(sfn).split('.')[0], - sfn - ) + base_fname = os.path.basename(sfn) + name = base_fname.split('.')[0] + + if USE_IMPORTLIB: + # pylint: disable=no-member + loader = importlib.machinery.SourceFileLoader(name, sfn) + spec = importlib.util.spec_from_file_location(name, sfn, loader=loader) + if spec is None: + raise ImportError() + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + # pylint: enable=no-member + sys.modules[name] = mod + else: + mod = imp.load_source(name, sfn) + # File templates need these set as __var__ if '__env__' not in kwargs and 'saltenv' in kwargs: setattr(mod, '__env__', kwargs['saltenv']) @@ -504,7 +529,7 @@ def py(sfn, string=False, **kwargs): # pylint: disable=C0103 return {'result': True, 'data': data} tgt = salt.utils.files.mkstemp() - with salt.utils.fopen(tgt, 'w+') as target: + with salt.utils.files.fopen(tgt, 'w+') as target: target.write(data) return {'result': True, 'data': tgt} diff --git a/salt/utils/thin.py b/salt/utils/thin.py index f18704aa62f..2dca2071405 100644 --- a/salt/utils/thin.py +++ b/salt/utils/thin.py @@ -71,6 +71,7 @@ except ImportError: # Import salt libs import salt import salt.utils +import salt.utils.files import salt.exceptions SALTCALL = ''' @@ -181,15 +182,15 @@ def gen_thin(cachedir, extra_mods='', overwrite=False, so_mods='', thinver = os.path.join(thindir, 'version') pythinver = os.path.join(thindir, '.thin-gen-py-version') salt_call = os.path.join(thindir, 'salt-call') - with salt.utils.fopen(salt_call, 'w+') as fp_: + with salt.utils.files.fopen(salt_call, 'w+') as fp_: fp_.write(SALTCALL) if os.path.isfile(thintar): if not overwrite: if os.path.isfile(thinver): - with salt.utils.fopen(thinver) as fh_: + with salt.utils.files.fopen(thinver) as fh_: overwrite = fh_.read() != salt.version.__version__ if overwrite is False and os.path.isfile(pythinver): - with salt.utils.fopen(pythinver) as fh_: + with salt.utils.files.fopen(pythinver) as fh_: overwrite = fh_.read() != str(sys.version_info[0]) else: overwrite = True @@ -318,9 +319,9 @@ def gen_thin(cachedir, extra_mods='', overwrite=False, so_mods='', tfp.add('salt-call') elif compress == 'zip': tfp.write('salt-call') - with salt.utils.fopen(thinver, 'w+') as fp_: + with salt.utils.files.fopen(thinver, 'w+') as fp_: fp_.write(salt.version.__version__) - with salt.utils.fopen(pythinver, 'w+') as fp_: + with salt.utils.files.fopen(pythinver, 'w+') as fp_: fp_.write(str(sys.version_info[0])) os.chdir(os.path.dirname(thinver)) if compress == 'gzip': @@ -366,15 +367,15 @@ def gen_min(cachedir, extra_mods='', overwrite=False, so_mods='', minver = os.path.join(mindir, 'version') pyminver = os.path.join(mindir, '.min-gen-py-version') salt_call = os.path.join(mindir, 'salt-call') - with salt.utils.fopen(salt_call, 'w+') as fp_: + with salt.utils.files.fopen(salt_call, 'w+') as fp_: fp_.write(SALTCALL) if os.path.isfile(mintar): if not overwrite: if os.path.isfile(minver): - with salt.utils.fopen(minver) as fh_: + with salt.utils.files.fopen(minver) as fh_: overwrite = fh_.read() != salt.version.__version__ if overwrite is False and os.path.isfile(pyminver): - with salt.utils.fopen(pyminver) as fh_: + with salt.utils.files.fopen(pyminver) as fh_: overwrite = fh_.read() != str(sys.version_info[0]) else: overwrite = True @@ -606,9 +607,9 @@ def gen_min(cachedir, extra_mods='', overwrite=False, so_mods='', os.chdir(mindir) tfp.add('salt-call') - with salt.utils.fopen(minver, 'w+') as fp_: + with salt.utils.files.fopen(minver, 'w+') as fp_: fp_.write(salt.version.__version__) - with salt.utils.fopen(pyminver, 'w+') as fp_: + with salt.utils.files.fopen(pyminver, 'w+') as fp_: fp_.write(str(sys.version_info[0])) os.chdir(os.path.dirname(minver)) tfp.add('version') diff --git a/salt/utils/url.py b/salt/utils/url.py index 0998ee5a0c7..ff02517f9da 100644 --- a/salt/utils/url.py +++ b/salt/utils/url.py @@ -11,6 +11,8 @@ import sys # Import salt libs from salt.ext.six.moves.urllib.parse import urlparse, urlunparse # pylint: disable=import-error,no-name-in-module import salt.utils +import salt.utils.platform +import salt.utils.versions from salt.utils.locales import sdecode @@ -25,19 +27,14 @@ def parse(url): resource = url.split('salt://', 1)[-1] if '?env=' in resource: - salt.utils.warn_until( - 'Oxygen', - 'Parameter \'env\' has been detected in the salt:// URL. This ' - 'parameter is no longer used and has been replaced by \'saltenv\' ' - 'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.' - ) + # "env" is not supported; Use "saltenv". path, saltenv = resource.split('?env=', 1)[0], None elif '?saltenv=' in resource: path, saltenv = resource.split('?saltenv=', 1) else: path, saltenv = resource, None - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): path = salt.utils.sanitize_win_path_string(path) return path, saltenv @@ -47,7 +44,7 @@ def create(path, saltenv=None): ''' join `path` and `saltenv` into a 'salt://' URL. ''' - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): path = salt.utils.sanitize_win_path_string(path) path = sdecode(path) @@ -60,15 +57,15 @@ def is_escaped(url): ''' test whether `url` is escaped with `|` ''' - if salt.utils.is_windows(): - return False - scheme = urlparse(url).scheme if not scheme: return url.startswith('|') elif scheme == 'salt': path, saltenv = parse(url) - return path.startswith('|') + if salt.utils.platform.is_windows() and '|' in url: + return path.startswith('_') + else: + return path.startswith('|') else: return False @@ -77,7 +74,7 @@ def escape(url): ''' add escape character `|` to `url` ''' - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): return url scheme = urlparse(url).scheme @@ -100,15 +97,15 @@ def unescape(url): ''' remove escape character `|` from `url` ''' - if salt.utils.is_windows(): - return url - scheme = urlparse(url).scheme if not scheme: return url.lstrip('|') elif scheme == 'salt': path, saltenv = parse(url) - return create(path.lstrip('|'), saltenv) + if salt.utils.platform.is_windows() and '|' in url: + return create(path.lstrip('_'), saltenv) + else: + return create(path.lstrip('|'), saltenv) else: return url diff --git a/salt/utils/validate/net.py b/salt/utils/validate/net.py index f96b7d0e04c..9250086d818 100644 --- a/salt/utils/validate/net.py +++ b/salt/utils/validate/net.py @@ -4,16 +4,16 @@ Various network validation utilities ''' from __future__ import absolute_import -# Import python libs +# Import Python libs import re import socket -# Import salt libs -from salt.ext.six import string_types -import salt.utils +# Import Salt libs +import salt.utils.platform -# Import third party libs -if salt.utils.is_windows(): +# Import 3rd-party libs +from salt.ext.six import string_types +if salt.utils.platform.is_windows(): from salt.ext import win_inet_pton # pylint: disable=unused-import diff --git a/salt/utils/validate/path.py b/salt/utils/validate/path.py index 1385b9bbce0..87f2d789c7b 100644 --- a/salt/utils/validate/path.py +++ b/salt/utils/validate/path.py @@ -64,3 +64,14 @@ def is_readable(path): # The path does not exist return False + + +def is_executable(path): + ''' + Check if a given path is executable by the current user. + + :param path: The path to check + :returns: True or False + ''' + + return os.access(path, os.X_OK) diff --git a/salt/utils/value.py b/salt/utils/value.py new file mode 100644 index 00000000000..222dd10813f --- /dev/null +++ b/salt/utils/value.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +''' +Utility functions used for values. + +.. versionadded:: Oxygen +''' + +# Import Python libs +from __future__ import absolute_import + + +def xor(*variables): + ''' + XOR definition for multiple variables + ''' + sum_ = False + for value in variables: + sum_ = sum_ ^ bool(value) + return sum_ diff --git a/salt/utils/vault.py b/salt/utils/vault.py index 30e66ab20e5..e34a41bb85d 100644 --- a/salt/utils/vault.py +++ b/salt/utils/vault.py @@ -16,6 +16,7 @@ import requests import salt.crypt import salt.exceptions +import salt.utils.versions log = logging.getLogger(__name__) logging.getLogger("requests").setLevel(logging.WARNING) @@ -136,7 +137,7 @@ def make_request_with_profile(method, resource, profile, **args): DEPRECATED! Make a request to Vault, with a profile including connection details. ''' - salt.utils.warn_until( + salt.utils.versions.warn_until( 'Fluorine', 'Specifying Vault connection data within a \'profile\' has been ' 'deprecated. Please see the documentation for details on the new ' diff --git a/salt/utils/verify.py b/salt/utils/verify.py index 5cac0fa651b..1559224e4d5 100644 --- a/salt/utils/verify.py +++ b/salt/utils/verify.py @@ -27,7 +27,9 @@ from salt.log.setup import LOG_LEVELS from salt.exceptions import SaltClientError, SaltSystemExit, \ CommandExecutionError import salt.defaults.exitcodes -import salt.utils +import salt.utils # Can be removed once get_jid_list and get_user are moved +import salt.utils.files +import salt.utils.platform log = logging.getLogger(__name__) @@ -145,7 +147,7 @@ def verify_files(files, user): ''' Verify that the named files exist and are owned by the named user ''' - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): return True import pwd # after confirming not running Windows try: @@ -167,7 +169,7 @@ def verify_files(files, user): if err.errno != errno.EEXIST: raise if not os.path.isfile(fn_): - with salt.utils.fopen(fn_, 'w+') as fp_: + with salt.utils.files.fopen(fn_, 'w+') as fp_: fp_.write('') except IOError as err: @@ -197,7 +199,7 @@ def verify_env(dirs, user, permissive=False, pki_dir='', skip_extra=False): Verify that the named directories are in place and that the environment can shake the salt ''' - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): return win_verify_env(dirs, permissive, pki_dir, skip_extra) import pwd # after confirming not running Windows try: @@ -298,7 +300,7 @@ def check_user(user): ''' Check user and assign process uid/gid. ''' - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): return True if user == salt.utils.get_user(): return True @@ -480,12 +482,21 @@ def clean_path(root, path, subdir=False): return '' +def clean_id(id_): + ''' + Returns if the passed id is clean. + ''' + if re.search(r'\.\.\{sep}'.format(sep=os.sep), id_): + return False + return True + + def valid_id(opts, id_): ''' Returns if the passed id is valid ''' try: - return bool(clean_path(opts['pki_dir'], id_)) + return bool(clean_path(opts['pki_dir'], id_)) and clean_id(id_) except (AttributeError, KeyError, TypeError) as e: return False diff --git a/salt/utils/versions.py b/salt/utils/versions.py index e4eb3524150..e51ae40b753 100644 --- a/salt/utils/versions.py +++ b/salt/utils/versions.py @@ -11,15 +11,24 @@ because on python 3 you can no longer compare strings against integers. ''' -# Import pytohn libs +# Import Python libs from __future__ import absolute_import +import logging +import numbers +import sys +import warnings # pylint: disable=blacklisted-module from distutils.version import StrictVersion as _StrictVersion from distutils.version import LooseVersion as _LooseVersion # pylint: enable=blacklisted-module +# Import Salt libs +import salt.version + # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six + +log = logging.getLogger(__name__) class StrictVersion(_StrictVersion): @@ -64,3 +73,221 @@ class LooseVersion(_LooseVersion): return -1 if self._str_version > other._str_version: return 1 + + +def warn_until(version, + message, + category=DeprecationWarning, + stacklevel=None, + _version_info_=None, + _dont_call_warnings=False): + ''' + Helper function to raise a warning, by default, a ``DeprecationWarning``, + until the provided ``version``, after which, a ``RuntimeError`` will + be raised to remind the developers to remove the warning because the + target version has been reached. + + :param version: The version info or name after which the warning becomes a + ``RuntimeError``. For example ``(0, 17)`` or ``Hydrogen`` + or an instance of :class:`salt.version.SaltStackVersion`. + :param message: The warning message to be displayed. + :param category: The warning class to be thrown, by default + ``DeprecationWarning`` + :param stacklevel: There should be no need to set the value of + ``stacklevel``. Salt should be able to do the right thing. + :param _version_info_: In order to reuse this function for other SaltStack + projects, they need to be able to provide the + version info to compare to. + :param _dont_call_warnings: This parameter is used just to get the + functionality until the actual error is to be + issued. When we're only after the salt version + checks to raise a ``RuntimeError``. + ''' + if not isinstance(version, (tuple, + six.string_types, + salt.version.SaltStackVersion)): + raise RuntimeError( + 'The \'version\' argument should be passed as a tuple, string or ' + 'an instance of \'salt.version.SaltStackVersion\'.' + ) + elif isinstance(version, tuple): + version = salt.version.SaltStackVersion(*version) + elif isinstance(version, six.string_types): + version = salt.version.SaltStackVersion.from_name(version) + + if stacklevel is None: + # Attribute the warning to the calling function, not to warn_until() + stacklevel = 2 + + if _version_info_ is None: + _version_info_ = salt.version.__version_info__ + + _version_ = salt.version.SaltStackVersion(*_version_info_) + + if _version_ >= version: + import inspect + caller = inspect.getframeinfo(sys._getframe(stacklevel - 1)) + raise RuntimeError( + 'The warning triggered on filename \'{filename}\', line number ' + '{lineno}, is supposed to be shown until version ' + '{until_version} is released. Current version is now ' + '{salt_version}. Please remove the warning.'.format( + filename=caller.filename, + lineno=caller.lineno, + until_version=version.formatted_version, + salt_version=_version_.formatted_version + ), + ) + + if _dont_call_warnings is False: + def _formatwarning(message, + category, + filename, + lineno, + line=None): # pylint: disable=W0613 + ''' + Replacement for warnings.formatwarning that disables the echoing of + the 'line' parameter. + ''' + return '{0}:{1}: {2}: {3}\n'.format( + filename, lineno, category.__name__, message + ) + saved = warnings.formatwarning + warnings.formatwarning = _formatwarning + warnings.warn( + message.format(version=version.formatted_version), + category, + stacklevel=stacklevel + ) + warnings.formatwarning = saved + + +def kwargs_warn_until(kwargs, + version, + category=DeprecationWarning, + stacklevel=None, + _version_info_=None, + _dont_call_warnings=False): + ''' + Helper function to raise a warning (by default, a ``DeprecationWarning``) + when unhandled keyword arguments are passed to function, until the + provided ``version_info``, after which, a ``RuntimeError`` will be raised + to remind the developers to remove the ``**kwargs`` because the target + version has been reached. + This function is used to help deprecate unused legacy ``**kwargs`` that + were added to function parameters lists to preserve backwards compatibility + when removing a parameter. See + :ref:`the deprecation development docs ` + for the modern strategy for deprecating a function parameter. + + :param kwargs: The caller's ``**kwargs`` argument value (a ``dict``). + :param version: The version info or name after which the warning becomes a + ``RuntimeError``. For example ``(0, 17)`` or ``Hydrogen`` + or an instance of :class:`salt.version.SaltStackVersion`. + :param category: The warning class to be thrown, by default + ``DeprecationWarning`` + :param stacklevel: There should be no need to set the value of + ``stacklevel``. Salt should be able to do the right thing. + :param _version_info_: In order to reuse this function for other SaltStack + projects, they need to be able to provide the + version info to compare to. + :param _dont_call_warnings: This parameter is used just to get the + functionality until the actual error is to be + issued. When we're only after the salt version + checks to raise a ``RuntimeError``. + ''' + if not isinstance(version, (tuple, + six.string_types, + salt.version.SaltStackVersion)): + raise RuntimeError( + 'The \'version\' argument should be passed as a tuple, string or ' + 'an instance of \'salt.version.SaltStackVersion\'.' + ) + elif isinstance(version, tuple): + version = salt.version.SaltStackVersion(*version) + elif isinstance(version, six.string_types): + version = salt.version.SaltStackVersion.from_name(version) + + if stacklevel is None: + # Attribute the warning to the calling function, + # not to kwargs_warn_until() or warn_until() + stacklevel = 3 + + if _version_info_ is None: + _version_info_ = salt.version.__version_info__ + + _version_ = salt.version.SaltStackVersion(*_version_info_) + + if kwargs or _version_.info >= version.info: + arg_names = ', '.join('\'{0}\''.format(key) for key in kwargs) + warn_until( + version, + message='The following parameter(s) have been deprecated and ' + 'will be removed in \'{0}\': {1}.'.format(version.string, + arg_names), + category=category, + stacklevel=stacklevel, + _version_info_=_version_.info, + _dont_call_warnings=_dont_call_warnings + ) + + +def version_cmp(pkg1, pkg2, ignore_epoch=False): + ''' + Compares two version strings using salt.utils.versions.LooseVersion. This + is a fallback for providers which don't have a version comparison utility + built into them. Return -1 if version1 < version2, 0 if version1 == + version2, and 1 if version1 > version2. Return None if there was a problem + making the comparison. + ''' + normalize = lambda x: str(x).split(':', 1)[-1] if ignore_epoch else str(x) + pkg1 = normalize(pkg1) + pkg2 = normalize(pkg2) + + try: + # pylint: disable=no-member + if LooseVersion(pkg1) < LooseVersion(pkg2): + return -1 + elif LooseVersion(pkg1) == LooseVersion(pkg2): + return 0 + elif LooseVersion(pkg1) > LooseVersion(pkg2): + return 1 + except Exception as exc: + log.exception(exc) + return None + + +def compare(ver1='', oper='==', ver2='', cmp_func=None, ignore_epoch=False): + ''' + Compares two version numbers. Accepts a custom function to perform the + cmp-style version comparison, otherwise uses version_cmp(). + ''' + cmp_map = {'<': (-1,), '<=': (-1, 0), '==': (0,), + '>=': (0, 1), '>': (1,)} + if oper not in ('!=',) and oper not in cmp_map: + log.error('Invalid operator \'%s\' for version comparison', oper) + return False + + if cmp_func is None: + cmp_func = version_cmp + + cmp_result = cmp_func(ver1, ver2, ignore_epoch=ignore_epoch) + if cmp_result is None: + return False + + # Check if integer/long + if not isinstance(cmp_result, numbers.Integral): + log.error('The version comparison function did not return an ' + 'integer/long.') + return False + + if oper == '!=': + return cmp_result not in cmp_map['=='] + else: + # Gracefully handle cmp_result not in (-1, 0, 1). + if cmp_result < -1: + cmp_result = -1 + elif cmp_result > 1: + cmp_result = 1 + + return cmp_result in cmp_map[oper] diff --git a/salt/utils/virt.py b/salt/utils/virt.py index 065181174cb..a7f10ec7be7 100644 --- a/salt/utils/virt.py +++ b/salt/utils/virt.py @@ -10,7 +10,7 @@ import time import logging # Import salt libs -import salt.utils +import salt.utils.files log = logging.getLogger(__name__) @@ -34,7 +34,7 @@ class VirtKey(object): Accept the provided key ''' try: - with salt.utils.fopen(self.path, 'r') as fp_: + with salt.utils.files.fopen(self.path, 'r') as fp_: expiry = int(fp_.read()) except (OSError, IOError): log.error( @@ -56,7 +56,7 @@ class VirtKey(object): pubfn = os.path.join(self.opts['pki_dir'], 'minions', self.id) - with salt.utils.fopen(pubfn, 'w+') as fp_: + with salt.utils.files.fopen(pubfn, 'w+') as fp_: fp_.write(pub) self.void() return True @@ -65,7 +65,7 @@ class VirtKey(object): ''' Prepare the master to expect a signing request ''' - with salt.utils.fopen(self.path, 'w+') as fp_: + with salt.utils.files.fopen(self.path, 'w+') as fp_: fp_.write(str(int(time.time()))) return True diff --git a/salt/utils/virtualbox.py b/salt/utils/virtualbox.py index 2798f1e6bee..c055a6ff8a9 100644 --- a/salt/utils/virtualbox.py +++ b/salt/utils/virtualbox.py @@ -19,7 +19,9 @@ from salt.utils.timeout import wait_for log = logging.getLogger(__name__) # Import 3rd-party libs +from salt.ext import six from salt.ext.six.moves import range + # Import virtualbox libs HAS_LIBS = False try: @@ -598,7 +600,7 @@ def vb_machinestate_to_tuple(machinestate): ''' if isinstance(machinestate, int): return MACHINE_STATES_ENUM.get(machinestate, UNKNOWN_MACHINE_STATE) - elif isinstance(machinestate, str): + elif isinstance(machinestate, six.string_types): return MACHINE_STATES.get(machinestate, UNKNOWN_MACHINE_STATE) else: return UNKNOWN_MACHINE_STATE @@ -625,9 +627,9 @@ def vb_machine_exists(name): vbox.findMachine(name) return True except Exception as e: - if isinstance(e.message, str): + if isinstance(e.message, six.string_types): message = e.message - elif hasattr(e, 'msg') and isinstance(getattr(e, 'msg'), str): + elif hasattr(e, 'msg') and isinstance(getattr(e, 'msg'), six.string_types): message = getattr(e, 'msg') else: message = '' diff --git a/salt/utils/vmware.py b/salt/utils/vmware.py index 19292f21283..6d2ff92e816 100644 --- a/salt/utils/vmware.py +++ b/salt/utils/vmware.py @@ -8,7 +8,7 @@ This is a base library used by a number of VMware services such as VMware ESX, ESXi, and vCenter servers. :codeauthor: Nitin Madhok -:codeauthor: Alexandru Bleotu +:codeauthor: Alexandru Bleotu Dependencies ~~~~~~~~~~~~ @@ -83,11 +83,13 @@ import time # Import Salt Libs import salt.exceptions import salt.modules.cmdmod -import salt.utils +import salt.utils.path +import salt.utils.platform +import salt.utils.stringutils # Import Third Party Libs -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves.http_client import BadStatusLine # pylint: disable=E0611 try: from pyVim.connect import GetSi, SmartConnect, Disconnect, GetStub @@ -134,7 +136,7 @@ def esxcli(host, user, pwd, cmd, protocol=None, port=None, esxi_host=None, creds :return: Dictionary ''' - esx_cmd = salt.utils.which('esxcli') + esx_cmd = salt.utils.path.which('esxcli') if not esx_cmd: log.error('Missing dependency: The salt.utils.vmware.esxcli function requires ESXCLI.') return False @@ -238,12 +240,12 @@ def _get_service_instance(host, username, password, protocol, pwd=password, protocol=protocol, port=port, - sslContext=ssl._create_unverified_context(), + sslContext=getattr(ssl, '_create_unverified_context', getattr(ssl, '_create_stdlib_context'))(), b64token=token, mechanism=mechanism) else: + log.exception(exc) err_msg = exc.msg if hasattr(exc, 'msg') else default_msg - log.trace(exc) raise salt.exceptions.VMwareConnectionError(err_msg) except Exception as exc: if 'certificate verify failed' in str(exc): @@ -262,8 +264,8 @@ def _get_service_instance(host, username, password, protocol, mechanism=mechanism ) except Exception as exc: + log.exception(exc) err_msg = exc.msg if hasattr(exc, 'msg') else str(exc) - log.trace(err_msg) raise salt.exceptions.VMwareConnectionError( 'Could not connect to host \'{0}\': ' '{1}'.format(host, err_msg)) @@ -353,7 +355,7 @@ def get_service_instance(host, username=None, password=None, protocol=None, service_instance = GetSi() if service_instance: stub = GetStub() - if salt.utils.is_proxy() or (hasattr(stub, 'host') and stub.host != ':'.join([host, str(port)])): + if salt.utils.platform.is_proxy() or (hasattr(stub, 'host') and stub.host != ':'.join([host, str(port)])): # Proxies will fork and mess up the cached service instance. # If this is a proxy or we are connecting to a different host # invalidate the service instance to avoid a potential memory leak @@ -388,9 +390,16 @@ def get_service_instance(host, username=None, password=None, protocol=None, mechanism, principal, domain) + except vim.fault.NoPermission as exc: + log.exception(exc) + raise salt.exceptions.VMwareApiError( + 'Not enough permissions. Required privilege: ' + '{}'.format(exc.privilegeId)) except vim.fault.VimFault as exc: + log.exception(exc) raise salt.exceptions.VMwareApiError(exc.msg) except vmodl.RuntimeFault as exc: + log.exception(exc) raise salt.exceptions.VMwareRuntimeError(exc.msg) return service_instance @@ -425,9 +434,16 @@ def disconnect(service_instance): log.trace('Disconnecting') try: Disconnect(service_instance) + except vim.fault.NoPermission as exc: + log.exception(exc) + raise salt.exceptions.VMwareApiError( + 'Not enough permissions. Required privilege: ' + '{}'.format(exc.privilegeId)) except vim.fault.VimFault as exc: + log.exception(exc) raise salt.exceptions.VMwareApiError(exc.msg) except vmodl.RuntimeFault as exc: + log.exception(exc) raise salt.exceptions.VMwareRuntimeError(exc.msg) @@ -441,9 +457,16 @@ def is_connection_to_a_vcenter(service_instance): ''' try: api_type = service_instance.content.about.apiType + except vim.fault.NoPermission as exc: + log.exception(exc) + raise salt.exceptions.VMwareApiError( + 'Not enough permissions. Required privilege: ' + '{}'.format(exc.privilegeId)) except vim.fault.VimFault as exc: + log.exception(exc) raise salt.exceptions.VMwareApiError(exc.msg) except vmodl.RuntimeFault as exc: + log.exception(exc) raise salt.exceptions.VMwareRuntimeError(exc.msg) log.trace('api_type = {0}'.format(api_type)) if api_type == 'VirtualCenter': @@ -456,6 +479,28 @@ def is_connection_to_a_vcenter(service_instance): '\'VirtualCenter/HostAgent\''.format(api_type)) +def get_service_info(service_instance): + ''' + Returns information of the vCenter or ESXi host + + service_instance + The Service Instance from which to obtain managed object references. + ''' + try: + return service_instance.content.about + except vim.fault.NoPermission as exc: + log.exception(exc) + raise salt.exceptions.VMwareApiError( + 'Not enough permissions. Required privilege: ' + '{0}'.format(exc.privilegeId)) + except vim.fault.VimFault as exc: + log.exception(exc) + raise salt.exceptions.VMwareApiError(exc.msg) + except vmodl.RuntimeFault as exc: + log.exception(exc) + raise salt.exceptions.VMwareRuntimeError(exc.msg) + + def _get_dvs(service_instance, dvs_name): ''' Return a reference to a Distributed Virtual Switch object. @@ -552,7 +597,7 @@ def get_gssapi_token(principal, host, domain): if out_token: if six.PY2: return base64.b64encode(out_token) - return base64.b64encode(salt.utils.to_bytes(out_token)) + return base64.b64encode(salt.utils.stringutils.to_bytes(out_token)) if ctx.established: break if not in_token: @@ -648,9 +693,16 @@ def get_root_folder(service_instance): try: log.trace('Retrieving root folder') return service_instance.RetrieveContent().rootFolder + except vim.fault.NoPermission as exc: + log.exception(exc) + raise salt.exceptions.VMwareApiError( + 'Not enough permissions. Required privilege: ' + '{}'.format(exc.privilegeId)) except vim.fault.VimFault as exc: + log.exception(exc) raise salt.exceptions.VMwareApiError(exc.msg) except vmodl.RuntimeFault as exc: + log.exception(exc) raise salt.exceptions.VMwareRuntimeError(exc.msg) @@ -700,9 +752,16 @@ def get_content(service_instance, obj_type, property_list=None, try: obj_ref = service_instance.content.viewManager.CreateContainerView( container_ref, [obj_type], True) + except vim.fault.NoPermission as exc: + log.exception(exc) + raise salt.exceptions.VMwareApiError( + 'Not enough permissions. Required privilege: ' + '{}'.format(exc.privilegeId)) except vim.fault.VimFault as exc: + log.exception(exc) raise salt.exceptions.VMwareApiError(exc.msg) except vmodl.RuntimeFault as exc: + log.exception(exc) raise salt.exceptions.VMwareRuntimeError(exc.msg) # Create 'Traverse All' traversal spec to determine the path for @@ -738,18 +797,32 @@ def get_content(service_instance, obj_type, property_list=None, # Retrieve the contents try: content = service_instance.content.propertyCollector.RetrieveContents([filter_spec]) + except vim.fault.NoPermission as exc: + log.exception(exc) + raise salt.exceptions.VMwareApiError( + 'Not enough permissions. Required privilege: ' + '{}'.format(exc.privilegeId)) except vim.fault.VimFault as exc: + log.exception(exc) raise salt.exceptions.VMwareApiError(exc.msg) except vmodl.RuntimeFault as exc: + log.exception(exc) raise salt.exceptions.VMwareRuntimeError(exc.msg) # Destroy the object view if local_traversal_spec: try: obj_ref.Destroy() + except vim.fault.NoPermission as exc: + log.exception(exc) + raise salt.exceptions.VMwareApiError( + 'Not enough permissions. Required privilege: ' + '{}'.format(exc.privilegeId)) except vim.fault.VimFault as exc: + log.exception(exc) raise salt.exceptions.VMwareApiError(exc.msg) except vmodl.RuntimeFault as exc: + log.exception(exc) raise salt.exceptions.VMwareRuntimeError(exc.msg) return content @@ -1000,9 +1073,16 @@ def create_datacenter(service_instance, datacenter_name): log.trace('Creating datacenter \'{0}\''.format(datacenter_name)) try: dc_obj = root_folder.CreateDatacenter(datacenter_name) + except vim.fault.NoPermission as exc: + log.exception(exc) + raise salt.exceptions.VMwareApiError( + 'Not enough permissions. Required privilege: ' + '{}'.format(exc.privilegeId)) except vim.fault.VimFault as exc: + log.exception(exc) raise salt.exceptions.VMwareApiError(exc.msg) except vmodl.RuntimeFault as exc: + log.exception(exc) raise salt.exceptions.VMwareRuntimeError(exc.msg) return dc_obj @@ -1062,9 +1142,16 @@ def create_cluster(dc_ref, cluster_name, cluster_spec): ''.format(cluster_name, dc_name)) try: dc_ref.hostFolder.CreateClusterEx(cluster_name, cluster_spec) + except vim.fault.NoPermission as exc: + log.exception(exc) + raise salt.exceptions.VMwareApiError( + 'Not enough permissions. Required privilege: ' + '{}'.format(exc.privilegeId)) except vim.fault.VimFault as exc: + log.exception(exc) raise salt.exceptions.VMwareApiError(exc.msg) except vmodl.RuntimeFault as exc: + log.exception(exc) raise salt.exceptions.VMwareRuntimeError(exc.msg) @@ -1084,9 +1171,16 @@ def update_cluster(cluster_ref, cluster_spec): try: task = cluster_ref.ReconfigureComputeResource_Task(cluster_spec, modify=True) + except vim.fault.NoPermission as exc: + log.exception(exc) + raise salt.exceptions.VMwareApiError( + 'Not enough permissions. Required privilege: ' + '{}'.format(exc.privilegeId)) except vim.fault.VimFault as exc: + log.exception(exc) raise salt.exceptions.VMwareApiError(exc.msg) except vmodl.RuntimeFault as exc: + log.exception(exc) raise salt.exceptions.VMwareRuntimeError(exc.msg) wait_for_task(task, cluster_name, 'ClusterUpdateTask') @@ -1291,9 +1385,16 @@ def wait_for_task(task, instance_name, task_type, sleep_seconds=1, log_level='de task.__class__.__name__)) try: task_info = task.info + except vim.fault.NoPermission as exc: + log.exception(exc) + raise salt.exceptions.VMwareApiError( + 'Not enough permissions. Required privilege: ' + '{}'.format(exc.privilegeId)) except vim.fault.VimFault as exc: + log.exception(exc) raise salt.exceptions.VMwareApiError(exc.msg) except vmodl.RuntimeFault as exc: + log.exception(exc) raise salt.exceptions.VMwareRuntimeError(exc.msg) while task_info.state == 'running' or task_info.state == 'queued': if time_counter % sleep_seconds == 0: @@ -1307,9 +1408,16 @@ def wait_for_task(task, instance_name, task_type, sleep_seconds=1, log_level='de time_counter += 1 try: task_info = task.info + except vim.fault.NoPermission as exc: + log.exception(exc) + raise salt.exceptions.VMwareApiError( + 'Not enough permissions. Required privilege: ' + '{}'.format(exc.privilegeId)) except vim.fault.VimFault as exc: + log.exception(exc) raise salt.exceptions.VMwareApiError(exc.msg) except vmodl.RuntimeFault as exc: + log.exception(exc) raise salt.exceptions.VMwareRuntimeError(exc.msg) if task_info.state == 'success': msg = '[ {0} ] Successfully completed {1} task in {2} seconds'.format( @@ -1324,11 +1432,19 @@ def wait_for_task(task, instance_name, task_type, sleep_seconds=1, log_level='de # task is in an error state try: raise task_info.error + except vim.fault.NoPermission as exc: + log.exception(exc) + raise salt.exceptions.VMwareApiError( + 'Not enough permissions. Required privilege: ' + '{}'.format(exc.privilegeId)) except vim.fault.VimFault as exc: + log.exception(exc) raise salt.exceptions.VMwareApiError(exc.msg) except vmodl.fault.SystemError as exc: + log.exception(exc) raise salt.exceptions.VMwareSystemError(exc.msg) except vmodl.fault.InvalidArgument as exc: + log.exception(exc) exc_message = exc.msg if exc.faultMessage: exc_message = '{0} ({1})'.format(exc_message, diff --git a/salt/utils/vsan.py b/salt/utils/vsan.py new file mode 100644 index 00000000000..1b9f796efe5 --- /dev/null +++ b/salt/utils/vsan.py @@ -0,0 +1,158 @@ +# -*- coding: utf-8 -*- +''' +Connection library for VMware vSAN endpoint + +This library used the vSAN extension of the VMware SDK +used to manage vSAN related objects + +:codeauthor: Alexandru Bleotu + +Dependencies +~~~~~~~~~~~~ + +- pyVmomi Python Module + +pyVmomi +------- + +PyVmomi can be installed via pip: + +.. code-block:: bash + + pip install pyVmomi + +.. note:: + + versions of Python. If using version 6.0 of pyVmomi, Python 2.6, + Python 2.7.9, or newer must be present. This is due to an upstream dependency + in pyVmomi 6.0 that is not supported in Python versions 2.7 to 2.7.8. If the + version of Python is not in the supported range, you will need to install an + earlier version of pyVmomi. See `Issue #29537`_ for more information. + +.. _Issue #29537: https://github.com/saltstack/salt/issues/29537 + +Based on the note above, to install an earlier version of pyVmomi than the +version currently listed in PyPi, run the following: + +.. code-block:: bash + + pip install pyVmomi==5.5.0.2014.1.1 + +The 5.5.0.2014.1.1 is a known stable version that this original VMware utils file +was developed against. +''' + +# Import Python Libs +from __future__ import absolute_import +import sys +import logging +import ssl + +# Import Salt Libs +from salt.exceptions import VMwareApiError, VMwareRuntimeError +import salt.utils.vmware + +try: + from pyVmomi import vim, vmodl + HAS_PYVMOMI = True +except ImportError: + HAS_PYVMOMI = False + + +try: + from salt.ext.vsan import vsanapiutils + HAS_PYVSAN = True +except ImportError: + HAS_PYVSAN = False + +# Get Logging Started +log = logging.getLogger(__name__) + + +def __virtual__(): + ''' + Only load if PyVmomi is installed. + ''' + if HAS_PYVSAN and HAS_PYVMOMI: + return True + else: + return False, 'Missing dependency: The salt.utils.vsan module ' \ + 'requires pyvmomi and the pyvsan extension library' + + +def vsan_supported(service_instance): + ''' + Returns whether vsan is supported on the vCenter: + api version needs to be 6 or higher + + service_instance + Service instance to the host or vCenter + ''' + try: + api_version = service_instance.content.about.apiVersion + except vim.fault.NoPermission as exc: + log.exception(exc) + raise VMwareApiError('Not enough permissions. Required privilege: ' + '{0}'.format(exc.privilegeId)) + except vim.fault.VimFault as exc: + log.exception(exc) + raise VMwareApiError(exc.msg) + except vmodl.RuntimeFault as exc: + log.exception(exc) + raise VMwareRuntimeError(exc.msg) + if int(api_version.split('.')[0]) < 6: + return False + return True + + +def get_vsan_cluster_config_system(service_instance): + ''' + Returns a vim.cluster.VsanVcClusterConfigSystem object + + service_instance + Service instance to the host or vCenter + ''' + + #TODO Replace when better connection mechanism is available + + #For python 2.7.9 and later, the defaul SSL conext has more strict + #connection handshaking rule. We may need turn of the hostname checking + #and client side cert verification + context = None + if sys.version_info[:3] > (2, 7, 8): + context = ssl.create_default_context() + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + + stub = service_instance._stub + vc_mos = vsanapiutils.GetVsanVcMos(stub, context=context) + return vc_mos['vsan-cluster-config-system'] + + +def get_cluster_vsan_info(cluster_ref): + ''' + Returns the extended cluster vsan configuration object + (vim.VsanConfigInfoEx). + + cluster_ref + Reference to the cluster + ''' + + cluster_name = salt.utils.vmware.get_managed_object_name(cluster_ref) + log.trace('Retrieving cluster vsan info of cluster ' + '\'{0}\''.format(cluster_name)) + si = salt.utils.vmware.get_service_instance_from_managed_object( + cluster_ref) + vsan_cl_conf_sys = get_vsan_cluster_config_system(si) + try: + return vsan_cl_conf_sys.VsanClusterGetConfig(cluster_ref) + except vim.fault.NoPermission as exc: + log.exception(exc) + raise VMwareApiError('Not enough permissions. Required privilege: ' + '{0}'.format(exc.privilegeId)) + except vim.fault.VimFault as exc: + log.exception(exc) + raise VMwareApiError(exc.msg) + except vmodl.RuntimeFault as exc: + log.exception(exc) + raise VMwareRuntimeError(exc.msg) diff --git a/salt/utils/vt.py b/salt/utils/vt.py index 4cde141dac5..b3ffb6d2394 100644 --- a/salt/utils/vt.py +++ b/salt/utils/vt.py @@ -53,6 +53,7 @@ except ImportError: # Import salt libs import salt.utils +import salt.utils.stringutils from salt.ext.six import string_types from salt.log.setup import LOG_LEVELS @@ -643,7 +644,7 @@ class Terminal(object): if self.child_fde in rlist: try: stderr = self._translate_newlines( - salt.utils.to_str( + salt.utils.stringutils.to_str( os.read(self.child_fde, maxsize) ) ) @@ -676,7 +677,7 @@ class Terminal(object): if self.child_fd in rlist: try: stdout = self._translate_newlines( - salt.utils.to_str( + salt.utils.stringutils.to_str( os.read(self.child_fd, maxsize) ) ) diff --git a/salt/utils/win_dacl.py b/salt/utils/win_dacl.py index 749cb1665d3..4b41429a6e1 100644 --- a/salt/utils/win_dacl.py +++ b/salt/utils/win_dacl.py @@ -134,7 +134,7 @@ import logging # Import Salt libs from salt.exceptions import CommandExecutionError, SaltInvocationError from salt.ext.six.moves import range -import salt.ext.six as six +from salt.ext import six # Import 3rd-party libs HAS_WIN32 = False diff --git a/salt/utils/win_functions.py b/salt/utils/win_functions.py index 23ee3edf04a..4e3ec9663c0 100644 --- a/salt/utils/win_functions.py +++ b/salt/utils/win_functions.py @@ -4,6 +4,9 @@ Various functions to be used by windows during start up and to monkey patch missing functions in other modules ''' from __future__ import absolute_import +import platform + +# Import Salt Libs from salt.exceptions import CommandExecutionError # Import 3rd Party Libs @@ -138,3 +141,19 @@ def get_current_user(): return False return user_name + + +def get_sam_name(username): + ''' + Gets the SAM name for a user. It basically prefixes a username without a + backslash with the computer name. If the username contains a backslash, it + is returned as is. + + Everything is returned lower case + + i.e. salt.utils.fix_local_user('Administrator') would return 'computername\administrator' + ''' + if '\\' not in username: + username = '{0}\\{1}'.format(platform.node(), username) + + return username.lower() diff --git a/salt/utils/win_update.py b/salt/utils/win_update.py index bc0e1e7648c..fac45d92760 100644 --- a/salt/utils/win_update.py +++ b/salt/utils/win_update.py @@ -9,6 +9,7 @@ import subprocess # Import Salt libs import salt.utils +import salt.utils.args from salt.ext import six from salt.ext.six.moves import range from salt.exceptions import CommandExecutionError @@ -963,7 +964,7 @@ class WindowsUpdateAgent(object): ''' if isinstance(cmd, six.string_types): - cmd = salt.utils.shlex_split(cmd) + cmd = salt.utils.args.shlex_split(cmd) try: log.debug(cmd) diff --git a/salt/utils/xmlutil.py b/salt/utils/xmlutil.py index 559d2eeae81..bb07921aaa8 100644 --- a/salt/utils/xmlutil.py +++ b/salt/utils/xmlutil.py @@ -14,7 +14,7 @@ def to_dict(xmltree): ''' # If this object has no children, the for..loop below will return nothing # for it, so just return a single dict representing it. - if len(xmltree.getchildren()) < 1: + if len(xmltree.getchildren()) < 1 and len(xmltree.attrib.items()) < 1: name = xmltree.tag if '}' in name: comps = name.split('}') @@ -33,6 +33,8 @@ def to_dict(xmltree): if name not in xmldict: if len(item.getchildren()) > 0: xmldict[name] = to_dict(item) + elif len(item.attrib.items()) > 0: + xmldict[name] = to_dict(item) else: xmldict[name] = item.text else: @@ -42,4 +44,12 @@ def to_dict(xmltree): if not isinstance(xmldict[name], list): xmldict[name] = [xmldict[name]] xmldict[name].append(to_dict(item)) + + for attrName, attrValue in xmltree.attrib.items(): + if attrName not in xmldict: + xmldict[attrName] = attrValue + else: + # Attempt to ensure that items are not overwritten by attributes. + xmldict["attr{0}".format(attrName)] = attrValue + return xmldict diff --git a/salt/utils/yamldumper.py b/salt/utils/yamldumper.py index 8f8742cf14b..514aae9613c 100644 --- a/salt/utils/yamldumper.py +++ b/salt/utils/yamldumper.py @@ -15,6 +15,9 @@ except ImportError: from yaml import Dumper from yaml import SafeDumper +import yaml +import collections + from salt.utils.odict import OrderedDict try: @@ -25,6 +28,17 @@ except ImportError: HAS_IOFLO = False +class IndentMixin(Dumper): + ''' + Mixin that improves YAML dumped list readability + by indenting them by two spaces, + instead of being flush with the key they are under. + ''' + + def increase_indent(self, flow=False, indentless=False): + return super(IndentMixin, self).increase_indent(flow, False) + + class OrderedDumper(Dumper): ''' A YAML dumper that represents python OrderedDict as simple YAML map. @@ -37,6 +51,14 @@ class SafeOrderedDumper(SafeDumper): ''' +class IndentedSafeOrderedDumper(IndentMixin, SafeOrderedDumper): + ''' + A YAML safe dumper that represents python OrderedDict as simple YAML map, + and also indents lists by two spaces. + ''' + pass + + def represent_ordereddict(dumper, data): return dumper.represent_dict(list(data.items())) @@ -44,6 +66,31 @@ def represent_ordereddict(dumper, data): OrderedDumper.add_representer(OrderedDict, represent_ordereddict) SafeOrderedDumper.add_representer(OrderedDict, represent_ordereddict) +OrderedDumper.add_representer( + collections.defaultdict, + yaml.representer.SafeRepresenter.represent_dict +) +SafeOrderedDumper.add_representer( + collections.defaultdict, + yaml.representer.SafeRepresenter.represent_dict +) + if HAS_IOFLO: OrderedDumper.add_representer(odict, represent_ordereddict) SafeOrderedDumper.add_representer(odict, represent_ordereddict) + + +def get_dumper(dumper_name): + return { + 'OrderedDumper': OrderedDumper, + 'SafeOrderedDumper': SafeOrderedDumper, + 'IndentedSafeOrderedDumper': IndentedSafeOrderedDumper, + }.get(dumper_name) + + +def safe_dump(data, stream=None, **kwargs): + ''' + Use a custom dumper to ensure that defaultdict and OrderedDict are + represented properly + ''' + return yaml.dump(data, stream, Dumper=SafeOrderedDumper, **kwargs) diff --git a/salt/utils/yamlencoding.py b/salt/utils/yamlencoding.py index 325b207722c..15e35b83da7 100644 --- a/salt/utils/yamlencoding.py +++ b/salt/utils/yamlencoding.py @@ -6,10 +6,10 @@ import io # Import 3rd-party libs import yaml -import salt.ext.six as six +from salt.ext import six # Import salt libs -from salt.utils.decorators import jinja_filter +from salt.utils.decorators.jinja import jinja_filter @jinja_filter() diff --git a/salt/utils/yamlloader.py b/salt/utils/yamlloader.py index a51d1f1f25b..6fe3a3fa1b8 100644 --- a/salt/utils/yamlloader.py +++ b/salt/utils/yamlloader.py @@ -4,6 +4,7 @@ from __future__ import absolute_import import warnings # Import third party libs +import re import yaml from yaml.nodes import MappingNode, SequenceNode from yaml.constructor import ConstructorError @@ -36,7 +37,7 @@ class SaltYamlSafeLoader(yaml.SafeLoader, object): to make things like sls file more intuitive. ''' def __init__(self, stream, dictclass=dict): - yaml.SafeLoader.__init__(self, stream) + super(SaltYamlSafeLoader, self).__init__(stream) if dictclass is not dict: # then assume ordered dict and use it for both !map and !omap self.add_constructor( @@ -45,9 +46,9 @@ class SaltYamlSafeLoader(yaml.SafeLoader, object): self.add_constructor( u'tag:yaml.org,2002:omap', type(self).construct_yaml_map) - self.add_constructor( - u'tag:yaml.org,2002:python/unicode', - type(self).construct_unicode) + self.add_constructor( + u'tag:yaml.org,2002:python/unicode', + type(self).construct_unicode) self.dictclass = dictclass def construct_yaml_map(self, node): @@ -101,6 +102,11 @@ class SaltYamlSafeLoader(yaml.SafeLoader, object): # an empty string. Change it to '0'. if node.value == '': node.value = '0' + elif node.tag == 'tag:yaml.org,2002:str': + # If any string comes in as a quoted unicode literal, eval it into + # the proper unicode string type. + if re.match(r'^u([\'"]).+\1$', node.value, flags=re.IGNORECASE): + node.value = eval(node.value, {}, {}) # pylint: disable=W0123 return super(SaltYamlSafeLoader, self).construct_scalar(node) def flatten_mapping(self, node): diff --git a/salt/utils/yast.py b/salt/utils/yast.py index 6dc9e1a8611..481e1dcf07a 100644 --- a/salt/utils/yast.py +++ b/salt/utils/yast.py @@ -7,7 +7,7 @@ Utilities for managing YAST from __future__ import absolute_import from salt._compat import ElementTree as ET import salt.utils.xmlutil as xml -import salt.utils +import salt.utils.files import yaml @@ -15,11 +15,11 @@ def mksls(src, dst=None): ''' Convert an AutoYAST file to an SLS file ''' - with salt.utils.fopen(src, 'r') as fh_: + with salt.utils.files.fopen(src, 'r') as fh_: ps_opts = xml.to_dict(ET.fromstring(fh_.read())) if dst is not None: - with salt.utils.fopen(dst, 'w') as fh_: + with salt.utils.files.fopen(dst, 'w') as fh_: fh_.write(yaml.safe_dump(ps_opts, default_flow_style=False)) else: return yaml.safe_dump(ps_opts, default_flow_style=False) diff --git a/salt/version.py b/salt/version.py index 89df51200a0..9a32a876151 100644 --- a/salt/version.py +++ b/salt/version.py @@ -7,6 +7,7 @@ Set up the version of Salt from __future__ import absolute_import, print_function import re import sys +import locale import platform # linux_distribution depreacted in py3.7 @@ -59,8 +60,9 @@ class SaltStackVersion(object): and also supports version comparison. ''' - __slots__ = ('name', 'major', 'minor', 'bugfix', 'mbugfix', 'pre_type', 'pre_num', 'noc', 'sha') + __slots__ = (u'name', u'major', u'minor', u'bugfix', u'mbugfix', u'pre_type', u'pre_num', u'noc', u'sha') + # future lint: disable=non-unicode-string git_describe_regex = re.compile( r'(?:[^\d]+)?(?P[\d]{1,4})' r'\.(?P[\d]{1,2})' @@ -69,7 +71,11 @@ class SaltStackVersion(object): r'(?:(?Prc|a|b|alpha|beta|nb)(?P[\d]{1}))?' r'(?:(?:.*)-(?P(?:[\d]+|n/a))-(?P[a-z0-9]{8}))?' ) - git_sha_regex = re.compile(r'(?P[a-z0-9]{7})') + git_sha_regex = r'(?P[a-z0-9]{7})' + if six.PY2: + git_sha_regex = git_sha_regex.decode(__salt_system_encoding__) + git_sha_regex = re.compile(git_sha_regex) + # future lint: enable=non-unicode-string # Salt versions after 0.17.0 will be numbered like: # <4-digit-year>.. @@ -87,16 +93,17 @@ class SaltStackVersion(object): # ----- Please refrain from fixing PEP-8 E203 and E265 -----> # The idea is to keep this readable. # ----------------------------------------------------------- - 'Hydrogen' : (2014, 1), - 'Helium' : (2014, 7), - 'Lithium' : (2015, 5), - 'Beryllium' : (2015, 8), - 'Boron' : (2016, 3), - 'Carbon' : (2016, 11), - 'Nitrogen' : (2017, 7), - 'Oxygen' : (MAX_SIZE - 101, 0), - 'Fluorine' : (MAX_SIZE - 100, 0), - 'Neon' : (MAX_SIZE - 99, 0), + u'Hydrogen' : (2014, 1), + u'Helium' : (2014, 7), + u'Lithium' : (2015, 5), + u'Beryllium' : (2015, 8), + u'Boron' : (2016, 3), + u'Carbon' : (2016, 11), + u'Nitrogen' : (2017, 7), + u'Oxygen' : (MAX_SIZE - 101, 0), + u'Fluorine' : (MAX_SIZE - 100, 0), + u'Neon' : (MAX_SIZE - 99, 0), + u'Sodium' : (MAX_SIZE - 98, 0), # pylint: disable=E8265 #'Sodium' : (MAX_SIZE - 98, 0), #'Magnesium' : (MAX_SIZE - 97, 0), @@ -232,7 +239,7 @@ class SaltStackVersion(object): mbugfix = int(mbugfix) if pre_type is None: - pre_type = '' + pre_type = u'' if pre_num is None: pre_num = 0 elif isinstance(pre_num, string_types): @@ -240,7 +247,7 @@ class SaltStackVersion(object): if noc is None: noc = 0 - elif isinstance(noc, string_types) and noc == 'n/a': + elif isinstance(noc, string_types) and noc == u'n/a': noc = -1 elif isinstance(noc, string_types): noc = int(noc) @@ -263,7 +270,7 @@ class SaltStackVersion(object): match = cls.git_describe_regex.match(vstr) if not match: raise ValueError( - 'Unable to parse version string: \'{0}\''.format(version_string) + u'Unable to parse version string: \'{0}\''.format(version_string) ) return cls(*match.groups()) @@ -271,7 +278,7 @@ class SaltStackVersion(object): def from_name(cls, name): if name.lower() not in cls.LNAMES: raise ValueError( - 'Named version \'{0}\' is not known'.format(name) + u'Named version \'{0}\' is not known'.format(name) ) return cls(*cls.LNAMES[name.lower()]) @@ -309,16 +316,6 @@ class SaltStackVersion(object): self.mbugfix ) - @property - def rc_info(self): - import salt.utils - salt.utils.warn_until( - 'Oxygen', - 'Please stop using the \'rc_info\' attribute and instead use ' - '\'pre_info\'. \'rc_info\' will be supported until Salt {version}.' - ) - return self.pre_info - @property def pre_info(self): return ( @@ -357,20 +354,20 @@ class SaltStackVersion(object): @property def string(self): - version_string = '{0}.{1}.{2}'.format( + version_string = u'{0}.{1}.{2}'.format( self.major, self.minor, self.bugfix ) if self.mbugfix: - version_string += '.{0}'.format(self.mbugfix) + version_string += u'.{0}'.format(self.mbugfix) if self.pre_type: - version_string += '{0}{1}'.format(self.pre_type, self.pre_num) + version_string += u'{0}{1}'.format(self.pre_type, self.pre_num) if self.noc and self.sha: noc = self.noc if noc < 0: - noc = 'n/a' - version_string += '-{0}-{1}'.format(noc, self.sha) + noc = u'n/a' + version_string += u'-{0}-{1}'.format(noc, self.sha) return version_string @property @@ -378,14 +375,14 @@ class SaltStackVersion(object): if self.name and self.major > 10000: version_string = self.name if self.sse: - version_string += ' Enterprise' - version_string += ' (Unreleased)' + version_string += u' Enterprise' + version_string += u' (Unreleased)' return version_string version_string = self.string if self.sse: - version_string += ' Enterprise' + version_string += u' Enterprise' if (self.major, self.minor) in self.RMATCH: - version_string += ' ({0})'.format(self.RMATCH[(self.major, self.minor)]) + version_string += u' ({0})'.format(self.RMATCH[(self.major, self.minor)]) return version_string def __str__(self): @@ -399,7 +396,7 @@ class SaltStackVersion(object): other = SaltStackVersion(*other) else: raise ValueError( - 'Cannot instantiate Version from type \'{0}\''.format( + u'Cannot instantiate Version from type \'{0}\''.format( type(other) ) ) @@ -411,13 +408,13 @@ class SaltStackVersion(object): if self.pre_type and not other.pre_type: # We have pre-release information, the other side doesn't other_noc_info = list(other.noc_info) - other_noc_info[4] = 'zzzzz' + other_noc_info[4] = u'zzzzz' return method(self.noc_info, tuple(other_noc_info)) if not self.pre_type and other.pre_type: # The other side has pre-release informatio, we don't noc_info = list(self.noc_info) - noc_info[4] = 'zzzzz' + noc_info[4] = u'zzzzz' return method(tuple(noc_info), other.noc_info) def __lt__(self, other): @@ -441,25 +438,25 @@ class SaltStackVersion(object): def __repr__(self): parts = [] if self.name: - parts.append('name=\'{0}\''.format(self.name)) + parts.append(u'name=\'{0}\''.format(self.name)) parts.extend([ - 'major={0}'.format(self.major), - 'minor={0}'.format(self.minor), - 'bugfix={0}'.format(self.bugfix) + u'major={0}'.format(self.major), + u'minor={0}'.format(self.minor), + u'bugfix={0}'.format(self.bugfix) ]) if self.mbugfix: - parts.append('minor-bugfix={0}'.format(self.mbugfix)) + parts.append(u'minor-bugfix={0}'.format(self.mbugfix)) if self.pre_type: - parts.append('{0}={1}'.format(self.pre_type, self.pre_num)) + parts.append(u'{0}={1}'.format(self.pre_type, self.pre_num)) noc = self.noc if noc == -1: - noc = 'n/a' + noc = u'n/a' if noc and self.sha: parts.extend([ - 'noc={0}'.format(noc), - 'sha={0}'.format(self.sha) + u'noc={0}'.format(noc), + u'sha={0}'.format(self.sha) ]) - return '<{0} {1}>'.format(self.__class__.__name__, ' '.join(parts)) + return u'<{0} {1}>'.format(self.__class__.__name__, u' '.join(parts)) # ----- Hardcoded Salt Codename Version Information -----------------------------------------------------------------> @@ -477,15 +474,15 @@ def __discover_version(saltstack_version): import os import subprocess - if 'SETUP_DIRNAME' in globals(): + if u'SETUP_DIRNAME' in globals(): # This is from the exec() call in Salt's setup.py cwd = SETUP_DIRNAME # pylint: disable=E0602 - if not os.path.exists(os.path.join(cwd, '.git')): + if not os.path.exists(os.path.join(cwd, u'.git')): # This is not a Salt git checkout!!! Don't even try to parse... return saltstack_version else: cwd = os.path.abspath(os.path.dirname(__file__)) - if not os.path.exists(os.path.join(os.path.dirname(cwd), '.git')): + if not os.path.exists(os.path.join(os.path.dirname(cwd), u'.git')): # This is not a Salt git checkout!!! Don't even try to parse... return saltstack_version @@ -496,12 +493,12 @@ def __discover_version(saltstack_version): cwd=cwd ) - if not sys.platform.startswith('win'): + if not sys.platform.startswith(u'win'): # Let's not import `salt.utils` for the above check - kwargs['close_fds'] = True + kwargs[u'close_fds'] = True process = subprocess.Popen( - ['git', 'describe', '--tags', '--first-parent', '--match', 'v[0-9]*', '--always'], **kwargs) + [u'git', u'describe', u'--tags', u'--first-parent', u'--match', u'v[0-9]*', u'--always'], **kwargs) out, err = process.communicate() @@ -509,7 +506,7 @@ def __discover_version(saltstack_version): # The git version running this might not support --first-parent # Revert to old command process = subprocess.Popen( - ['git', 'describe', '--tags', '--match', 'v[0-9]*', '--always'], **kwargs) + [u'git', u'describe', u'--tags', u'--match', u'v[0-9]*', u'--always'], **kwargs) out, err = process.communicate() out = out.strip() err = err.strip() @@ -566,7 +563,7 @@ def salt_information(): ''' Report version of salt. ''' - yield 'Salt', __version__ + yield u'Salt', __version__ def dependency_information(include_salt_cloud=False): @@ -574,39 +571,39 @@ def dependency_information(include_salt_cloud=False): Report versions of library dependencies. ''' libs = [ - ('Python', None, sys.version.rsplit('\n')[0].strip()), - ('Jinja2', 'jinja2', '__version__'), - ('M2Crypto', 'M2Crypto', 'version'), - ('msgpack-python', 'msgpack', 'version'), - ('msgpack-pure', 'msgpack_pure', 'version'), - ('pycrypto', 'Crypto', '__version__'), - ('pycryptodome', 'Cryptodome', 'version_info'), - ('libnacl', 'libnacl', '__version__'), - ('PyYAML', 'yaml', '__version__'), - ('ioflo', 'ioflo', '__version__'), - ('PyZMQ', 'zmq', '__version__'), - ('RAET', 'raet', '__version__'), - ('ZMQ', 'zmq', 'zmq_version'), - ('Mako', 'mako', '__version__'), - ('Tornado', 'tornado', 'version'), - ('timelib', 'timelib', 'version'), - ('dateutil', 'dateutil', '__version__'), - ('pygit2', 'pygit2', '__version__'), - ('libgit2', 'pygit2', 'LIBGIT2_VERSION'), - ('smmap', 'smmap', '__version__'), - ('cffi', 'cffi', '__version__'), - ('pycparser', 'pycparser', '__version__'), - ('gitdb', 'gitdb', '__version__'), - ('gitpython', 'git', '__version__'), - ('python-gnupg', 'gnupg', '__version__'), - ('mysql-python', 'MySQLdb', '__version__'), - ('cherrypy', 'cherrypy', '__version__'), - ('docker-py', 'docker', '__version__'), + (u'Python', None, sys.version.rsplit(u'\n')[0].strip()), + (u'Jinja2', u'jinja2', u'__version__'), + (u'M2Crypto', u'M2Crypto', u'version'), + (u'msgpack-python', u'msgpack', u'version'), + (u'msgpack-pure', u'msgpack_pure', u'version'), + (u'pycrypto', u'Crypto', u'__version__'), + (u'pycryptodome', u'Cryptodome', u'version_info'), + (u'libnacl', u'libnacl', u'__version__'), + (u'PyYAML', u'yaml', u'__version__'), + (u'ioflo', u'ioflo', u'__version__'), + (u'PyZMQ', u'zmq', u'__version__'), + (u'RAET', u'raet', u'__version__'), + (u'ZMQ', u'zmq', u'zmq_version'), + (u'Mako', u'mako', u'__version__'), + (u'Tornado', u'tornado', u'version'), + (u'timelib', u'timelib', u'version'), + (u'dateutil', u'dateutil', u'__version__'), + (u'pygit2', u'pygit2', u'__version__'), + (u'libgit2', u'pygit2', u'LIBGIT2_VERSION'), + (u'smmap', u'smmap', u'__version__'), + (u'cffi', u'cffi', u'__version__'), + (u'pycparser', u'pycparser', u'__version__'), + (u'gitdb', u'gitdb', u'__version__'), + (u'gitpython', u'git', u'__version__'), + (u'python-gnupg', u'gnupg', u'__version__'), + (u'mysql-python', u'MySQLdb', u'__version__'), + (u'cherrypy', u'cherrypy', u'__version__'), + (u'docker-py', u'docker', u'__version__'), ] if include_salt_cloud: libs.append( - ('Apache Libcloud', 'libcloud', '__version__'), + (u'Apache Libcloud', u'libcloud', u'__version__'), ) for name, imp, attr in libs: @@ -619,7 +616,7 @@ def dependency_information(include_salt_cloud=False): if callable(version): version = version() if isinstance(version, (tuple, list)): - version = '.'.join(map(str, version)) + version = u'.'.join(map(str, version)) yield name, version except Exception: yield name, None @@ -638,40 +635,45 @@ def system_information(): win_ver = platform.win32_ver() if lin_ver[0]: - return ' '.join(lin_ver) + return u' '.join(lin_ver) elif mac_ver[0]: - if isinstance(mac_ver[1], (tuple, list)) and ''.join(mac_ver[1]): - return ' '.join([mac_ver[0], '.'.join(mac_ver[1]), mac_ver[2]]) + if isinstance(mac_ver[1], (tuple, list)) and u''.join(mac_ver[1]): + return u' '.join([mac_ver[0], u'.'.join(mac_ver[1]), mac_ver[2]]) else: - return ' '.join([mac_ver[0], mac_ver[2]]) + return u' '.join([mac_ver[0], mac_ver[2]]) elif win_ver[0]: - return ' '.join(win_ver) + return u' '.join(win_ver) else: - return '' + return u'' version = system_version() release = platform.release() if platform.win32_ver()[0]: import win32api # pylint: disable=3rd-party-module-not-gated - if ((sys.version_info.major == 2 and sys.version_info >= (2, 7, 12)) or - (sys.version_info.major == 3 and sys.version_info >= (3, 5, 2))): - if win32api.GetVersionEx(1)[8] > 1: - server = {'Vista': '2008Server', - '7': '2008ServerR2', - '8': '2012Server', - '8.1': '2012ServerR2', - '10': '2016Server'} - release = server.get(platform.release(), - 'UNKServer') - _, ver, sp, extra = platform.win32_ver() - version = ' '.join([release, ver, sp, extra]) + server = {u'Vista': u'2008Server', + u'7': u'2008ServerR2', + u'8': u'2012Server', + u'8.1': u'2012ServerR2', + u'10': u'2016Server'} + # Starting with Python 2.7.12 and 3.5.2 the `platform.uname()` function + # started reporting the Desktop version instead of the Server version on + # Server versions of Windows, so we need to look those up + # So, if you find a Server Platform that's a key in the server + # dictionary, then lookup the actual Server Release. + # If this is a Server Platform then `GetVersionEx` will return a number + # greater than 1. + if win32api.GetVersionEx(1)[8] > 1 and release in server: + release = server[release] + _, ver, sp, extra = platform.win32_ver() + version = ' '.join([release, ver, sp, extra]) system = [ - ('system', platform.system()), - ('dist', ' '.join(linux_distribution(full_distribution_name=False))), - ('release', release), - ('machine', platform.machine()), - ('version', version), + (u'system', platform.system()), + (u'dist', u' '.join(linux_distribution(full_distribution_name=False))), + (u'release', release), + (u'machine', platform.machine()), + (u'version', version), + (u'locale', locale.getpreferredencoding()), ] for name, attr in system: @@ -687,9 +689,9 @@ def versions_information(include_salt_cloud=False): lib_info = list(dependency_information(include_salt_cloud)) sys_info = list(system_information()) - return {'Salt Version': dict(salt_info), - 'Dependency Versions': dict(lib_info), - 'System Versions': dict(sys_info)} + return {u'Salt Version': dict(salt_info), + u'Dependency Versions': dict(lib_info), + u'System Versions': dict(sys_info)} def versions_report(include_salt_cloud=False): @@ -698,21 +700,21 @@ def versions_report(include_salt_cloud=False): ''' ver_info = versions_information(include_salt_cloud) - lib_pad = max(len(name) for name in ver_info['Dependency Versions']) - sys_pad = max(len(name) for name in ver_info['System Versions']) + lib_pad = max(len(name) for name in ver_info[u'Dependency Versions']) + sys_pad = max(len(name) for name in ver_info[u'System Versions']) padding = max(lib_pad, sys_pad) + 1 - fmt = '{0:>{pad}}: {1}' + fmt = u'{0:>{pad}}: {1}' info = [] - for ver_type in ('Salt Version', 'Dependency Versions', 'System Versions'): - info.append('{0}:'.format(ver_type)) + for ver_type in (u'Salt Version', u'Dependency Versions', u'System Versions'): + info.append(u'{0}:'.format(ver_type)) # List dependencies in alphabetical, case insensitive order for name in sorted(ver_info[ver_type], key=lambda x: x.lower()): ver = fmt.format(name, - ver_info[ver_type][name] or 'Not Installed', + ver_info[ver_type][name] or u'Not Installed', pad=padding) info.append(ver) - info.append(' ') + info.append(u' ') for line in info: yield line @@ -734,10 +736,10 @@ def msi_conformant_version(): month = __saltstack_version__.minor minor = __saltstack_version__.bugfix commi = __saltstack_version__.noc - return '{0}.{1}.{2}.{3}'.format(year2, month, minor, commi) + return u'{0}.{1}.{2}.{3}'.format(year2, month, minor, commi) -if __name__ == '__main__': - if len(sys.argv) == 2 and sys.argv[1] == 'msi': +if __name__ == u'__main__': + if len(sys.argv) == 2 and sys.argv[1] == u'msi': # Building the msi requires an msi-conformant version print(msi_conformant_version()) else: diff --git a/salt/wheel/config.py b/salt/wheel/config.py index 9fcbd19e8b2..146604c7742 100644 --- a/salt/wheel/config.py +++ b/salt/wheel/config.py @@ -13,6 +13,7 @@ import yaml # Import salt libs import salt.config +import salt.utils.files from salt.utils.yamldumper import SafeOrderedDumper log = logging.getLogger(__name__) @@ -39,7 +40,7 @@ def apply(key, value): path = os.path.join(path, 'master') data = values() data[key] = value - with salt.utils.fopen(path, 'w+') as fp_: + with salt.utils.files.fopen(path, 'w+') as fp_: fp_.write( yaml.dump( data, @@ -87,7 +88,7 @@ def update_config(file_name, yaml_contents): os.makedirs(dir_path, 0o755) file_path = os.path.join(dir_path, file_name) - with salt.utils.fopen(file_path, 'w') as fp_: + with salt.utils.files.fopen(file_path, 'w') as fp_: fp_.write(yaml_out) return 'Wrote {0}'.format(file_name) diff --git a/salt/wheel/file_roots.py b/salt/wheel/file_roots.py index ff13d3971bc..8df2eee4b89 100644 --- a/salt/wheel/file_roots.py +++ b/salt/wheel/file_roots.py @@ -8,10 +8,10 @@ from __future__ import absolute_import import os # Import salt libs -import salt.utils +import salt.utils.files # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six def find(path, saltenv='base'): @@ -26,8 +26,8 @@ def find(path, saltenv='base'): full = os.path.join(root, path) if os.path.isfile(full): # Add it to the dict - with salt.utils.fopen(full, 'rb') as fp_: - if salt.utils.istextfile(fp_): + with salt.utils.files.fopen(full, 'rb') as fp_: + if salt.utils.files.is_text_file(fp_): ret.append({full: 'txt'}) else: ret.append({full: 'bin'}) @@ -86,7 +86,7 @@ def read(path, saltenv='base'): full = next(six.iterkeys(fn_)) form = fn_[full] if form == 'txt': - with salt.utils.fopen(full, 'rb') as fp_: + with salt.utils.files.fopen(full, 'rb') as fp_: ret.append({full: fp_.read()}) return ret @@ -108,6 +108,6 @@ def write(data, path, saltenv='base', index=0): dest_dir = os.path.dirname(dest) if not os.path.isdir(dest_dir): os.makedirs(dest_dir) - with salt.utils.fopen(dest, 'w+') as fp_: + with salt.utils.files.fopen(dest, 'w+') as fp_: fp_.write(data) return 'Wrote data to file {0}'.format(dest) diff --git a/salt/wheel/key.py b/salt/wheel/key.py index b1e008c3fd5..9f1a17bfa7b 100644 --- a/salt/wheel/key.py +++ b/salt/wheel/key.py @@ -36,7 +36,9 @@ import logging # Import salt libs from salt.key import get_key import salt.crypt -import salt.utils +import salt.utils # Can be removed once pem_finger is moved +import salt.utils.files +import salt.utils.platform from salt.utils.sanitizers import clean @@ -353,14 +355,14 @@ def gen(id_=None, keysize=2048): 'pub': ''} priv = salt.crypt.gen_keys(__opts__['pki_dir'], id_, keysize) pub = '{0}.pub'.format(priv[:priv.rindex('.')]) - with salt.utils.fopen(priv) as fp_: + with salt.utils.files.fopen(priv) as fp_: ret['priv'] = fp_.read() - with salt.utils.fopen(pub) as fp_: + with salt.utils.files.fopen(pub) as fp_: ret['pub'] = fp_.read() # The priv key is given the Read-Only attribute. The causes `os.remove` to # fail in Windows. - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): os.chmod(priv, 128) os.remove(priv) @@ -413,7 +415,7 @@ def gen_accept(id_, keysize=2048, force=False): acc_path = os.path.join(__opts__['pki_dir'], 'minions', id_) if os.path.isfile(acc_path) and not force: return {} - with salt.utils.fopen(acc_path, 'w+') as fp_: + with salt.utils.files.fopen(acc_path, 'w+') as fp_: fp_.write(ret['pub']) return ret diff --git a/salt/wheel/pillar_roots.py b/salt/wheel/pillar_roots.py index a86270d9ce4..65790e17d95 100644 --- a/salt/wheel/pillar_roots.py +++ b/salt/wheel/pillar_roots.py @@ -9,10 +9,10 @@ from __future__ import absolute_import import os # Import salt libs -import salt.utils +import salt.utils.files # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six def find(path, saltenv='base'): @@ -27,8 +27,8 @@ def find(path, saltenv='base'): full = os.path.join(root, path) if os.path.isfile(full): # Add it to the dict - with salt.utils.fopen(full, 'rb') as fp_: - if salt.utils.istextfile(fp_): + with salt.utils.files.fopen(full, 'rb') as fp_: + if salt.utils.files.is_text_file(fp_): ret.append({full: 'txt'}) else: ret.append({full: 'bin'}) @@ -87,7 +87,7 @@ def read(path, saltenv='base'): full = next(six.iterkeys(fn_)) form = fn_[full] if form == 'txt': - with salt.utils.fopen(full, 'rb') as fp_: + with salt.utils.files.fopen(full, 'rb') as fp_: ret.append({full: fp_.read()}) return ret @@ -109,6 +109,6 @@ def write(data, path, saltenv='base', index=0): dest_dir = os.path.dirname(dest) if not os.path.isdir(dest_dir): os.makedirs(dest_dir) - with salt.utils.fopen(dest, 'w+') as fp_: + with salt.utils.files.fopen(dest, 'w+') as fp_: fp_.write(data) return 'Wrote data to file {0}'.format(dest) diff --git a/scripts/salt-master b/scripts/salt-master index eeaae97c507..b4323bd5326 100755 --- a/scripts/salt-master +++ b/scripts/salt-master @@ -3,12 +3,12 @@ Start the salt-master ''' +import salt.utils.platform from salt.scripts import salt_master -from salt.utils import is_windows if __name__ == '__main__': - if is_windows(): + if salt.utils.platform.is_windows(): # Since this file does not have a '.py' extension, when running on # Windows, spawning any addional processes will fail due to Python # not being able to load this 'module' in the new process. diff --git a/scripts/salt-minion b/scripts/salt-minion index e787fd8f229..74a9fa1d676 100755 --- a/scripts/salt-minion +++ b/scripts/salt-minion @@ -3,13 +3,13 @@ This script is used to kick off a salt minion daemon ''' +import salt.utils.platform from salt.scripts import salt_minion -from salt.utils import is_windows from multiprocessing import freeze_support if __name__ == '__main__': - if is_windows(): + if salt.utils.platform.is_windows(): # Since this file does not have a '.py' extension, when running on # Windows, spawning any addional processes will fail due to Python # not being able to load this 'module' in the new process. diff --git a/scripts/salt-proxy b/scripts/salt-proxy index 9d2a3b2f430..e921e6590bd 100755 --- a/scripts/salt-proxy +++ b/scripts/salt-proxy @@ -5,13 +5,13 @@ This script is used to kick off a salt proxy minion daemon ''' from __future__ import absolute_import +import salt.utils.platform from salt.scripts import salt_proxy -from salt.utils import is_windows from multiprocessing import freeze_support if __name__ == '__main__': - if is_windows(): + if salt.utils.platform.is_windows(): # Since this file does not have a '.py' extension, when running on # Windows, spawning any addional processes will fail due to Python # not being able to load this 'module' in the new process. diff --git a/scripts/salt-unity b/scripts/salt-unity index c81ed60b669..8180b68a240 100644 --- a/scripts/salt-unity +++ b/scripts/salt-unity @@ -5,7 +5,7 @@ import sys # Import salt libs import salt.scripts -from salt.utils import is_windows +import salt.utils.platform def get_avail(): @@ -43,7 +43,7 @@ def redirect(): if __name__ == '__main__': - if is_windows(): + if salt.utils.platform.is_windows(): # Since this file does not have a '.py' extension, when running on # Windows, spawning any addional processes will fail due to Python # not being able to load this 'module' in the new process. diff --git a/scripts/suse/watchdog/salt-daemon-watcher b/scripts/suse/watchdog/salt-daemon-watcher new file mode 100755 index 00000000000..1025b20e38b --- /dev/null +++ b/scripts/suse/watchdog/salt-daemon-watcher @@ -0,0 +1,26 @@ +#!/bin/bash +# +# Author: Bo Maryniuk +# Requires: yum install propcps +# +# Runs every minute from crontab, +# checks salt-minion every 10 seconds. +# +# Use this with a following crontab: +# * * * * * /path/to/this/script + +if [ "$1" != "--with-init" ]; then + echo "This command is not used directly." + exit 1; +fi + +SHELL=/bin/sh +PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin + +for iter in {1..5}; do + if [[ $(pgrep salt-minion) == "" ]]; then + service salt-minion restart + fi + sleep 10; +done +true diff --git a/scripts/suse/yum/plugins/README.md b/scripts/suse/yum/plugins/README.md new file mode 100644 index 00000000000..cb3abd22602 --- /dev/null +++ b/scripts/suse/yum/plugins/README.md @@ -0,0 +1,20 @@ +## What it is + +Plugin which provides a notification mechanism to Salt, if Yum is +used outside of it. + +## Installation + +Configuration files are going to: + + `/etc/yum/pluginconf.d/[name].conf` + +Plugin itself goes to: + + `/usr/share/yum-plugins/[name].conf` + +## Permissions + +User: root +Group: root +Mode: 644 diff --git a/scripts/suse/yum/plugins/yumnotify.conf b/scripts/suse/yum/plugins/yumnotify.conf new file mode 100644 index 00000000000..8e4d76c728b --- /dev/null +++ b/scripts/suse/yum/plugins/yumnotify.conf @@ -0,0 +1,2 @@ +[main] +enabled=1 diff --git a/scripts/suse/yum/plugins/yumnotify.py b/scripts/suse/yum/plugins/yumnotify.py new file mode 100644 index 00000000000..268e1e9531d --- /dev/null +++ b/scripts/suse/yum/plugins/yumnotify.py @@ -0,0 +1,55 @@ +# Copyright (c) 2016 SUSE Linux LLC +# All Rights Reserved. +# +# Author: Bo Maryniuk + +from yum.plugins import TYPE_CORE +from yum import config +import os +import hashlib + +CK_PATH = "/var/cache/salt/minion/rpmdb.cookie" +RPM_PATH = "/var/lib/rpm/Packages" + +requires_api_version = '2.5' +plugin_type = TYPE_CORE + + +def _get_mtime(): + """ + Get the modified time of the RPM Database. + + Returns: + Unix ticks + """ + return os.path.exists(RPM_PATH) and int(os.path.getmtime(RPM_PATH)) or 0 + + +def _get_checksum(): + """ + Get the checksum of the RPM Database. + + Returns: + hexdigest + """ + digest = hashlib.md5() + with open(RPM_PATH, "rb") as rpm_db_fh: + while True: + buff = rpm_db_fh.read(0x1000) + if not buff: + break + digest.update(buff) + return digest.hexdigest() + + +def posttrans_hook(conduit): + """ + Hook after the package installation transaction. + + :param conduit: + :return: + """ + # Integrate Yum with Salt + if 'SALT_RUNNING' not in os.environ: + with open(CK_PATH, 'w') as ck_fh: + ck_fh.write('{chksum} {mtime}\n'.format(chksum=_get_checksum(), mtime=_get_mtime())) diff --git a/scripts/suse/zypper/plugins/commit/README.md b/scripts/suse/zypper/plugins/commit/README.md new file mode 100644 index 00000000000..01c8917c8e0 --- /dev/null +++ b/scripts/suse/zypper/plugins/commit/README.md @@ -0,0 +1,3 @@ +# Zypper plugins + +Plugins here are required to interact with SUSE Manager in conjunction of SaltStack and Zypper. diff --git a/scripts/suse/zypper/plugins/commit/zyppnotify b/scripts/suse/zypper/plugins/commit/zyppnotify new file mode 100755 index 00000000000..268298b1081 --- /dev/null +++ b/scripts/suse/zypper/plugins/commit/zyppnotify @@ -0,0 +1,59 @@ +#!/usr/bin/python +# +# Copyright (c) 2016 SUSE Linux LLC +# All Rights Reserved. +# +# Author: Bo Maryniuk + +import sys +import os +import hashlib + +from zypp_plugin import Plugin + + +class DriftDetector(Plugin): + """ + Return diff of the installed packages outside the Salt. + """ + def __init__(self): + Plugin.__init__(self) + self.ck_path = "/var/cache/salt/minion/rpmdb.cookie" + self.rpm_path = "/var/lib/rpm/Packages" + + def _get_mtime(self): + ''' + Get the modified time of the RPM Database. + Returns: + Unix ticks + ''' + return os.path.exists(self.rpm_path) and int(os.path.getmtime(self.rpm_path)) or 0 + + def _get_checksum(self): + ''' + Get the checksum of the RPM Database. + Returns: + hexdigest + ''' + digest = hashlib.md5() + with open(self.rpm_path, "rb") as rpm_db_fh: + while True: + buff = rpm_db_fh.read(0x1000) + if not buff: + break + digest.update(buff) + + return digest.hexdigest() + + def PLUGINEND(self, headers, body): + """ + Hook when plugin closes Zypper's transaction. + """ + if 'SALT_RUNNING' not in os.environ: + with open(self.ck_path, 'w') as ck_fh: + ck_fh.write('{chksum} {mtime}\n'.format(chksum=self._get_checksum(), mtime=self._get_mtime())) + + self.ack() + + +DriftDetector().main() diff --git a/setup.py b/setup.py index b67efa04ee2..aacebb95a1c 100755 --- a/setup.py +++ b/setup.py @@ -234,6 +234,7 @@ class GenerateSaltSyspaths(Command): spm_formula_path=self.distribution.salt_spm_formula_dir, spm_pillar_path=self.distribution.salt_spm_pillar_dir, spm_reactor_path=self.distribution.salt_spm_reactor_dir, + home_dir=self.distribution.salt_home_dir, ) ) @@ -697,8 +698,7 @@ class Clean(clean): INSTALL_VERSION_TEMPLATE = '''\ -# This file was auto-generated by salt's setup on \ -{date:%A, %d %B %Y @ %H:%m:%S UTC}. +# This file was auto-generated by salt's setup from salt.version import SaltStackVersion @@ -725,6 +725,7 @@ PIDFILE_DIR = {pidfile_dir!r} SPM_FORMULA_PATH = {spm_formula_path!r} SPM_PILLAR_PATH = {spm_pillar_path!r} SPM_REACTOR_PATH = {spm_reactor_path!r} +HOME_DIR = {home_dir!r} ''' @@ -869,6 +870,8 @@ class SaltDistribution(distutils.dist.Distribution): 'Salt\'s pre-configured SPM pillar directory'), ('salt-spm-reactor-dir=', None, 'Salt\'s pre-configured SPM reactor directory'), + ('salt-home-dir=', None, + 'Salt\'s pre-configured user home directory'), ] def __init__(self, attrs=None): @@ -893,6 +896,7 @@ class SaltDistribution(distutils.dist.Distribution): self.salt_spm_formula_dir = None self.salt_spm_pillar_dir = None self.salt_spm_reactor_dir = None + self.salt_home_dir = None self.name = 'salt-ssh' if PACKAGED_FOR_SALT_SSH else 'salt' self.salt_version = __version__ # pylint: disable=undefined-variable diff --git a/tests/conftest.py b/tests/conftest.py index 4414287c463..7591cd46dfd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -46,10 +46,11 @@ from _pytest.terminal import TerminalReporter # Import 3rd-party libs import psutil -import salt.ext.six as six +from salt.ext import six # Import salt libs -import salt.utils +import salt.utils.files +import salt.utils.path import salt.log.setup from salt.utils.odict import OrderedDict @@ -281,14 +282,14 @@ def pytest_runtest_setup(item): message = skip_if_binaries_missing_marker.kwargs.get('message', None) if check_all: for binary in binaries: - if salt.utils.which(binary) is None: + if salt.utils.path.which(binary) is None: pytest.skip( '{0}The "{1}" binary was not found'.format( message and '{0}. '.format(message) or '', binary ) ) - elif salt.utils.which_bin(binaries) is None: + elif salt.utils.path.which_bin(binaries) is None: pytest.skip( '{0}None of the following binaries was found: {1}'.format( message and '{0}. '.format(message) or '', @@ -592,7 +593,7 @@ def cli_bin_dir(tempdir, if not os.path.isfile(script_path): log.info('Generating {0}'.format(script_path)) - with salt.utils.fopen(script_path, 'w') as sfh: + with salt.utils.files.fopen(script_path, 'w') as sfh: script_template = script_templates.get(original_script_name, None) if script_template is None: script_template = script_templates.get('common', None) diff --git a/tests/consist.py b/tests/consist.py index 6df7f153675..ad4a0b6403f 100644 --- a/tests/consist.py +++ b/tests/consist.py @@ -9,13 +9,13 @@ import pprint import optparse # Import Salt libs -from salt.utils import get_colors +import salt.utils.color # Import 3rd-party libs import yaml -import salt.ext.six as six +from salt.ext import six -colors = get_colors() +colors = salt.utils.color.get_colors() def parse(): diff --git a/tests/eventlisten.py b/tests/eventlisten.py index d9f9a267278..65bbeed94d1 100644 --- a/tests/eventlisten.py +++ b/tests/eventlisten.py @@ -18,7 +18,7 @@ import os import salt.utils.event # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six def parse(): diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 12ce122c011..92dcd8fedf0 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -49,10 +49,14 @@ import salt.minion import salt.runner import salt.output import salt.version -import salt.utils +import salt.utils # Can be removed once appendproctitle is moved +import salt.utils.color +import salt.utils.files +import salt.utils.path +import salt.utils.platform import salt.utils.process +import salt.utils.stringutils import salt.log.setup as salt_log_setup -from salt.ext import six from salt.utils.verify import verify_env from salt.utils.immutabletypes import freeze from salt.utils.nb_popen import NonBlockingPopen @@ -67,7 +71,7 @@ except ImportError: # Import 3rd-party libs import yaml import msgpack -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import cStringIO try: @@ -185,8 +189,8 @@ class TestDaemon(object): def __init__(self, parser): self.parser = parser - self.colors = salt.utils.get_colors(self.parser.options.no_colors is False) - if salt.utils.is_windows(): + self.colors = salt.utils.color.get_colors(self.parser.options.no_colors is False) + if salt.utils.platform.is_windows(): # There's no shell color support on windows... for key in self.colors: self.colors[key] = '' @@ -527,8 +531,8 @@ class TestDaemon(object): **self.colors ) ) - keygen = salt.utils.which('ssh-keygen') - sshd = salt.utils.which('sshd') + keygen = salt.utils.path.which('ssh-keygen') + sshd = salt.utils.path.which('sshd') if not (keygen and sshd): print('WARNING: Could not initialize SSH subsystem. Tests for salt-ssh may break!') @@ -561,7 +565,7 @@ class TestDaemon(object): ) _, keygen_err = keygen_process.communicate() if keygen_err: - print('ssh-keygen had errors: {0}'.format(salt.utils.to_str(keygen_err))) + print('ssh-keygen had errors: {0}'.format(salt.utils.stringutils.to_str(keygen_err))) sshd_config_path = os.path.join(FILES, 'conf/_ssh/sshd_config') shutil.copy(sshd_config_path, RUNTIME_VARS.TMP_CONF_DIR) auth_key_file = os.path.join(RUNTIME_VARS.TMP_CONF_DIR, 'key_test.pub') @@ -604,7 +608,7 @@ class TestDaemon(object): ) _, keygen_dsa_err = keygen_process_dsa.communicate() if keygen_dsa_err: - print('ssh-keygen had errors: {0}'.format(salt.utils.to_str(keygen_dsa_err))) + print('ssh-keygen had errors: {0}'.format(salt.utils.stringutils.to_str(keygen_dsa_err))) keygen_process_ecdsa = subprocess.Popen( [keygen, '-t', @@ -624,7 +628,7 @@ class TestDaemon(object): ) _, keygen_escda_err = keygen_process_ecdsa.communicate() if keygen_escda_err: - print('ssh-keygen had errors: {0}'.format(salt.utils.to_str(keygen_escda_err))) + print('ssh-keygen had errors: {0}'.format(salt.utils.stringutils.to_str(keygen_escda_err))) keygen_process_ed25519 = subprocess.Popen( [keygen, '-t', @@ -644,9 +648,9 @@ class TestDaemon(object): ) _, keygen_ed25519_err = keygen_process_ed25519.communicate() if keygen_ed25519_err: - print('ssh-keygen had errors: {0}'.format(salt.utils.to_str(keygen_ed25519_err))) + print('ssh-keygen had errors: {0}'.format(salt.utils.stringutils.to_str(keygen_ed25519_err))) - with salt.utils.fopen(os.path.join(RUNTIME_VARS.TMP_CONF_DIR, 'sshd_config'), 'a') as ssh_config: + with salt.utils.files.fopen(os.path.join(RUNTIME_VARS.TMP_CONF_DIR, 'sshd_config'), 'a') as ssh_config: ssh_config.write('AuthorizedKeysFile {0}\n'.format(auth_key_file)) if not keygen_dsa_err: ssh_config.write('HostKey {0}\n'.format(server_dsa_priv_key_file)) @@ -665,12 +669,12 @@ class TestDaemon(object): ) _, sshd_err = self.sshd_process.communicate() if sshd_err: - print('sshd had errors on startup: {0}'.format(salt.utils.to_str(sshd_err))) + print('sshd had errors on startup: {0}'.format(salt.utils.stringutils.to_str(sshd_err))) else: os.environ['SSH_DAEMON_RUNNING'] = 'True' roster_path = os.path.join(FILES, 'conf/_ssh/roster') shutil.copy(roster_path, RUNTIME_VARS.TMP_CONF_DIR) - with salt.utils.fopen(os.path.join(RUNTIME_VARS.TMP_CONF_DIR, 'roster'), 'a') as roster: + with salt.utils.files.fopen(os.path.join(RUNTIME_VARS.TMP_CONF_DIR, 'roster'), 'a') as roster: roster.write(' user: {0}\n'.format(RUNTIME_VARS.RUNNING_TESTS_USER)) roster.write(' priv: {0}/{1}'.format(RUNTIME_VARS.TMP_CONF_DIR, 'key_test')) sys.stdout.write( @@ -724,7 +728,7 @@ class TestDaemon(object): os.makedirs(RUNTIME_VARS.TMP_SYNDIC_MINION_CONF_DIR) print(' * Transplanting configuration files to \'{0}\''.format(RUNTIME_VARS.TMP_CONF_DIR)) tests_known_hosts_file = os.path.join(RUNTIME_VARS.TMP_CONF_DIR, 'salt_ssh_known_hosts') - with salt.utils.fopen(tests_known_hosts_file, 'w') as known_hosts: + with salt.utils.files.fopen(tests_known_hosts_file, 'w') as known_hosts: known_hosts.write('') # This master connects to syndic_master via a syndic @@ -825,7 +829,7 @@ class TestDaemon(object): for opts_dict in (master_opts, syndic_master_opts): if 'ext_pillar' not in opts_dict: opts_dict['ext_pillar'] = [] - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): opts_dict['ext_pillar'].append( {'cmd_yaml': 'type {0}'.format(os.path.join(FILES, 'ext.yaml'))}) else: @@ -898,22 +902,22 @@ class TestDaemon(object): for entry in ('master', 'minion', 'sub_minion', 'syndic', 'syndic_master', 'proxy'): computed_config = copy.deepcopy(locals()['{0}_opts'.format(entry)]) - with salt.utils.fopen(os.path.join(RUNTIME_VARS.TMP_CONF_DIR, entry), 'w') as fp_: + with salt.utils.files.fopen(os.path.join(RUNTIME_VARS.TMP_CONF_DIR, entry), 'w') as fp_: fp_.write(yaml.dump(computed_config, default_flow_style=False)) sub_minion_computed_config = copy.deepcopy(sub_minion_opts) - with salt.utils.fopen(os.path.join(RUNTIME_VARS.TMP_SUB_MINION_CONF_DIR, 'minion'), 'w') as wfh: + with salt.utils.files.fopen(os.path.join(RUNTIME_VARS.TMP_SUB_MINION_CONF_DIR, 'minion'), 'w') as wfh: wfh.write( yaml.dump(sub_minion_computed_config, default_flow_style=False) ) shutil.copyfile(os.path.join(RUNTIME_VARS.TMP_CONF_DIR, 'master'), os.path.join(RUNTIME_VARS.TMP_SUB_MINION_CONF_DIR, 'master')) syndic_master_computed_config = copy.deepcopy(syndic_master_opts) - with salt.utils.fopen(os.path.join(RUNTIME_VARS.TMP_SYNDIC_MASTER_CONF_DIR, 'master'), 'w') as wfh: + with salt.utils.files.fopen(os.path.join(RUNTIME_VARS.TMP_SYNDIC_MASTER_CONF_DIR, 'master'), 'w') as wfh: wfh.write( yaml.dump(syndic_master_computed_config, default_flow_style=False) ) syndic_computed_config = copy.deepcopy(syndic_opts) - with salt.utils.fopen(os.path.join(RUNTIME_VARS.TMP_SYNDIC_MINION_CONF_DIR, 'minion'), 'w') as wfh: + with salt.utils.files.fopen(os.path.join(RUNTIME_VARS.TMP_SYNDIC_MINION_CONF_DIR, 'minion'), 'w') as wfh: wfh.write( yaml.dump(syndic_computed_config, default_flow_style=False) ) @@ -1061,7 +1065,7 @@ class TestDaemon(object): except OSError as exc: if exc.errno != 3: raise - with salt.utils.fopen(self.sshd_pidfile) as fhr: + with salt.utils.files.fopen(self.sshd_pidfile) as fhr: try: os.kill(int(fhr.read()), signal.SIGKILL) except OSError as exc: diff --git a/tests/integration/cli/test_grains.py b/tests/integration/cli/test_grains.py index 7c34ca85cf3..3ef0d352177 100644 --- a/tests/integration/cli/test_grains.py +++ b/tests/integration/cli/test_grains.py @@ -17,7 +17,7 @@ from __future__ import absolute_import import os # Import Salt Libs -import salt.utils +import salt.utils.files # Import Salt Testing Libs from tests.support.case import ShellCase, SSHCase @@ -61,9 +61,11 @@ class GrainsTargetingTest(ShellCase): # Create a minion key, but do not start the "fake" minion. This mimics a # disconnected minion. key_file = os.path.join(self.master_opts['pki_dir'], 'minions', 'disconnected') - with salt.utils.fopen(key_file, 'a'): + with salt.utils.files.fopen(key_file, 'a'): pass + import logging + log = logging.getLogger(__name__) # ping disconnected minion and ensure it times out and returns with correct message try: ret = '' diff --git a/tests/integration/client/test_kwarg.py b/tests/integration/client/test_kwarg.py index bb5b369de04..ac4d69a0e73 100644 --- a/tests/integration/client/test_kwarg.py +++ b/tests/integration/client/test_kwarg.py @@ -94,3 +94,25 @@ class StdTest(ModuleCase): ret = self.client.cmd('minion', 'test.ping', full_return=True) for mid, data in ret.items(): self.assertIn('retcode', data) + + def test_cmd_arg_kwarg_parsing(self): + ret = self.client.cmd('minion', 'test.arg_clean', + arg=[ + 'foo', + 'bar=off', + 'baz={qux: 123}' + ], + kwarg={ + 'quux': 'Quux', + }) + + self.assertEqual(ret['minion'], { + 'args': ['foo'], + 'kwargs': { + 'bar': False, + 'baz': { + 'qux': 123, + }, + 'quux': 'Quux', + }, + }) diff --git a/tests/integration/client/test_runner.py b/tests/integration/client/test_runner.py index 01f6b8aaca1..989bcb69a82 100644 --- a/tests/integration/client/test_runner.py +++ b/tests/integration/client/test_runner.py @@ -133,3 +133,14 @@ class RunnerModuleTest(TestCase, AdaptedConfigurationTestCaseMixin): 'quuz': 'on', }, }) + + def test_invalid_kwargs_are_ignored(self): + low = { + 'client': 'runner', + 'fun': 'test.metasyntactic', + 'thiskwargisbad': 'justpretendimnothere', + } + low.update(self.eauth_creds) + + ret = self.runner.cmd_sync(low) + self.assertEqual(ret[0], 'foo') diff --git a/tests/integration/client/test_standard.py b/tests/integration/client/test_standard.py index 629b18d8236..b0e8809da10 100644 --- a/tests/integration/client/test_standard.py +++ b/tests/integration/client/test_standard.py @@ -8,7 +8,7 @@ import os from tests.support.case import ModuleCase # Import salt libs -import salt.utils +import salt.utils.files class StdTest(ModuleCase): @@ -43,7 +43,7 @@ class StdTest(ModuleCase): # create fake minion key_file = os.path.join(self.master_opts['pki_dir'], 'minions', 'footest') # touch the file - with salt.utils.fopen(key_file, 'a'): + with salt.utils.files.fopen(key_file, 'a'): pass # ping that minion and ensure it times out try: @@ -126,7 +126,7 @@ class StdTest(ModuleCase): # Create a minion key, but do not start the "fake" minion. This mimics # a disconnected minion. key_file = os.path.join(self.master_opts['pki_dir'], 'minions', 'disconnected') - with salt.utils.fopen(key_file, 'a'): + with salt.utils.files.fopen(key_file, 'a'): pass # ping disconnected minion and ensure it times out and returns with correct message @@ -147,3 +147,33 @@ class StdTest(ModuleCase): finally: os.unlink(key_file) + + def test_missing_minion_list(self): + ''' + test cmd with missing minion in nodegroup + ''' + ret = self.client.cmd( + 'minion,ghostminion', + 'test.ping', + tgt_type='list' + ) + self.assertIn('minion', ret) + self.assertIn('ghostminion', ret) + self.assertEqual(True, ret['minion']) + self.assertEqual(u'Minion did not return. [No response]', + ret['ghostminion']) + + def test_missing_minion_nodegroup(self): + ''' + test cmd with missing minion in nodegroup + ''' + ret = self.client.cmd( + 'missing_minion', + 'test.ping', + tgt_type='nodegroup' + ) + self.assertIn('minion', ret) + self.assertIn('ghostminion', ret) + self.assertEqual(True, ret['minion']) + self.assertEqual(u'Minion did not return. [No response]', + ret['ghostminion']) diff --git a/tests/integration/cloud/helpers/virtualbox.py b/tests/integration/cloud/helpers/virtualbox.py index 2f3cc6f86d6..c7eb9897ce1 100644 --- a/tests/integration/cloud/helpers/virtualbox.py +++ b/tests/integration/cloud/helpers/virtualbox.py @@ -13,7 +13,7 @@ from tests.support.unit import TestCase, skipIf from tests.support.paths import FILES # Import Salt libs -import salt.ext.six as six +from salt.ext import six import salt.utils.virtualbox # Create the cloud instance name to be used throughout the tests diff --git a/tests/integration/cloud/providers/test_digital_ocean.py b/tests/integration/cloud/providers/test_digitalocean.py similarity index 91% rename from tests/integration/cloud/providers/test_digital_ocean.py rename to tests/integration/cloud/providers/test_digitalocean.py index 63a541bcb46..adbf76f43e4 100644 --- a/tests/integration/cloud/providers/test_digital_ocean.py +++ b/tests/integration/cloud/providers/test_digitalocean.py @@ -6,33 +6,18 @@ Integration tests for DigitalOcean APIv2 # Import Python Libs from __future__ import absolute_import import os -import random -import string # Import Salt Testing Libs from tests.support.case import ShellCase from tests.support.paths import FILES -from tests.support.helpers import expensiveTest +from tests.support.helpers import expensiveTest, generate_random_name # Import Salt Libs from salt.config import cloud_providers_config -# Import 3rd-party libs -from salt.ext.six.moves import range # pylint: disable=import-error,redefined-builtin - - -def __random_name(size=6): - ''' - Generates a random cloud instance name - ''' - return 'CLOUD-TEST-' + ''.join( - random.choice(string.ascii_uppercase + string.digits) - for x in range(size) - ) - # Create the cloud instance name to be used throughout the tests -INSTANCE_NAME = __random_name() -PROVIDER_NAME = 'digital_ocean' +INSTANCE_NAME = generate_random_name('CLOUD-TEST-') +PROVIDER_NAME = 'digitalocean' class DigitalOceanTest(ShellCase): @@ -81,7 +66,7 @@ class DigitalOceanTest(ShellCase): def test_list_images(self): ''' - Tests the return of running the --list-images command for digital ocean + Tests the return of running the --list-images command for digitalocean ''' image_list = self.run_cloud('--list-images {0}'.format(PROVIDER_NAME)) self.assertIn( @@ -91,7 +76,7 @@ class DigitalOceanTest(ShellCase): def test_list_locations(self): ''' - Tests the return of running the --list-locations command for digital ocean + Tests the return of running the --list-locations command for digitalocean ''' _list_locations = self.run_cloud('--list-locations {0}'.format(PROVIDER_NAME)) self.assertIn( @@ -101,7 +86,7 @@ class DigitalOceanTest(ShellCase): def test_list_sizes(self): ''' - Tests the return of running the --list-sizes command for digital ocean + Tests the return of running the --list-sizes command for digitalocean ''' _list_sizes = self.run_cloud('--list-sizes {0}'.format(PROVIDER_NAME)) self.assertIn( diff --git a/tests/integration/cloud/providers/test_ec2.py b/tests/integration/cloud/providers/test_ec2.py index aa51ac3b883..58b5c243893 100644 --- a/tests/integration/cloud/providers/test_ec2.py +++ b/tests/integration/cloud/providers/test_ec2.py @@ -6,8 +6,6 @@ # Import Python Libs from __future__ import absolute_import import os -import random -import string # Import Salt Libs from salt.config import cloud_providers_config @@ -15,23 +13,10 @@ from salt.config import cloud_providers_config # Import Salt Testing Libs from tests.support.case import ShellCase from tests.support.paths import FILES -from tests.support.helpers import expensiveTest - -# Import Third-Party Libs -from salt.ext.six.moves import range # pylint: disable=import-error,redefined-builtin - - -def __random_name(size=6): - ''' - Generates a radom cloud instance name - ''' - return 'CLOUD-TEST-' + ''.join( - random.choice(string.ascii_uppercase + string.digits) - for x in range(size) - ) +from tests.support.helpers import expensiveTest, generate_random_name # Create the cloud instance name to be used throughout the tests -INSTANCE_NAME = __random_name() +INSTANCE_NAME = generate_random_name('CLOUD-TEST-') PROVIDER_NAME = 'ec2' diff --git a/tests/integration/cloud/providers/test_gce.py b/tests/integration/cloud/providers/test_gce.py index 44a1fd4f819..2fb09f83cf9 100644 --- a/tests/integration/cloud/providers/test_gce.py +++ b/tests/integration/cloud/providers/test_gce.py @@ -7,8 +7,6 @@ # Import Python Libs from __future__ import absolute_import import os -import random -import string # Import Salt Libs from salt.config import cloud_providers_config @@ -16,24 +14,11 @@ from salt.config import cloud_providers_config # Import Salt Testing Libs from tests.support.case import ShellCase from tests.support.paths import FILES -from tests.support.helpers import expensiveTest - -# Import Third-Party Libs -from salt.ext.six.moves import range # pylint: disable=import-error,redefined-builtin +from tests.support.helpers import expensiveTest, generate_random_name TIMEOUT = 500 -def _random_name(size=6): - ''' - Generates a radom cloud instance name - ''' - return 'cloud-test-' + ''.join( - random.choice(string.ascii_lowercase + string.digits) - for x in range(size) - ) - - class GCETest(ShellCase): ''' Integration tests for the GCE cloud provider in Salt-Cloud @@ -51,7 +36,7 @@ class GCETest(ShellCase): provider = 'gce' providers = self.run_cloud('--list-providers') # Create the cloud instance name to be used throughout the tests - self.INSTANCE_NAME = _random_name() + self.INSTANCE_NAME = generate_random_name('CLOUD-TEST-') if profile_str not in providers: self.skipTest( diff --git a/tests/integration/cloud/providers/test_gogrid.py b/tests/integration/cloud/providers/test_gogrid.py index e33173e99bb..9fdea5452bf 100644 --- a/tests/integration/cloud/providers/test_gogrid.py +++ b/tests/integration/cloud/providers/test_gogrid.py @@ -6,31 +6,18 @@ # Import Python Libs from __future__ import absolute_import import os -import random -import string # Import Salt Testing Libs from tests.support.case import ShellCase from tests.support.paths import FILES -from tests.support.helpers import expensiveTest +from tests.support.helpers import expensiveTest, generate_random_name from tests.support.unit import skipIf # Import Salt Libs from salt.config import cloud_providers_config -from salt.ext.six.moves import range - - -def __random_name(size=6): - ''' - Generates a random cloud instance name - ''' - return 'CLOUD-TEST-' + ''.join( - random.choice(string.ascii_uppercase + string.digits) - for x in range(size) - ) # Create the cloud instance name to be used throughout the tests -INSTANCE_NAME = __random_name() +INSTANCE_NAME = generate_random_name('CLOUD-TEST-') PROVIDER_NAME = 'gogrid' diff --git a/tests/integration/cloud/providers/test_joyent.py b/tests/integration/cloud/providers/test_joyent.py index af3f832338e..70f802640d2 100644 --- a/tests/integration/cloud/providers/test_joyent.py +++ b/tests/integration/cloud/providers/test_joyent.py @@ -6,30 +6,18 @@ # Import Python Libs from __future__ import absolute_import import os -import random -import string # Import Salt Testing Libs from tests.support.case import ShellCase from tests.support.paths import FILES -from tests.support.helpers import expensiveTest +from tests.support.helpers import expensiveTest, generate_random_name # Import Salt Libs from salt.config import cloud_providers_config -from salt.ext.six.moves import range # pylint: disable=redefined-builtin -def __random_name(size=6): - ''' - Generates a random cloud instance name - ''' - return 'CLOUD-TEST-' + ''.join( - random.choice(string.ascii_uppercase + string.digits) - for x in range(size) - ) - # Create the cloud instance name to be used throughout the tests -INSTANCE_NAME = __random_name() +INSTANCE_NAME = generate_random_name('CLOUD-TEST-') PROVIDER_NAME = 'joyent' diff --git a/tests/integration/cloud/providers/test_linode.py b/tests/integration/cloud/providers/test_linode.py index 554beb7b8c8..98e30ba4ca5 100644 --- a/tests/integration/cloud/providers/test_linode.py +++ b/tests/integration/cloud/providers/test_linode.py @@ -6,30 +6,17 @@ # Import Python Libs from __future__ import absolute_import import os -import random -import string # Import Salt Testing Libs from tests.support.case import ShellCase from tests.support.paths import FILES -from tests.support.helpers import expensiveTest +from tests.support.helpers import expensiveTest, generate_random_name # Import Salt Libs from salt.config import cloud_providers_config -from salt.ext.six.moves import range - - -def __random_name(size=6): - ''' - Generates a random cloud instance name - ''' - return 'CLOUD-TEST-' + ''.join( - random.choice(string.ascii_uppercase + string.digits) - for x in range(size) - ) # Create the cloud instance name to be used throughout the tests -INSTANCE_NAME = __random_name() +INSTANCE_NAME = generate_random_name('CLOUD-TEST-') PROVIDER_NAME = 'linode' diff --git a/tests/integration/cloud/providers/test_msazure.py b/tests/integration/cloud/providers/test_msazure.py index 0fcbdbb6091..59f844124af 100644 --- a/tests/integration/cloud/providers/test_msazure.py +++ b/tests/integration/cloud/providers/test_msazure.py @@ -6,22 +6,17 @@ # Import Python Libs from __future__ import absolute_import import os -import random -import string # Import Salt Testing Libs from tests.support.case import ShellCase from tests.support.paths import FILES from tests.support.unit import skipIf -from tests.support.helpers import expensiveTest +from tests.support.helpers import expensiveTest, generate_random_name # Import Salt Libs from salt.config import cloud_providers_config from salt.utils.versions import LooseVersion -# Import Third-Party Libs -from salt.ext.six.moves import range - TIMEOUT = 500 try: @@ -33,18 +28,8 @@ except ImportError: if HAS_AZURE and not hasattr(azure, '__version__'): import azure.common - -def __random_name(size=6): - ''' - Generates a random cloud instance name - ''' - return 'CLOUD-TEST-' + ''.join( - random.choice(string.ascii_uppercase + string.digits) - for x in range(size) - ) - # Create the cloud instance name to be used throughout the tests -INSTANCE_NAME = __random_name() +INSTANCE_NAME = generate_random_name('CLOUD-TEST-') PROVIDER_NAME = 'azure' PROFILE_NAME = 'azure-test' REQUIRED_AZURE = '0.11.1' diff --git a/tests/integration/cloud/providers/test_oneandone.py b/tests/integration/cloud/providers/test_oneandone.py new file mode 100644 index 00000000000..8e26134f18e --- /dev/null +++ b/tests/integration/cloud/providers/test_oneandone.py @@ -0,0 +1,120 @@ +# -*- coding: utf-8 -*- +''' + :codeauthor: :email:`Amel Ajdinovic ` +''' + +# Import Python Libs +from __future__ import absolute_import +import os + +# Import Salt Testing Libs +from tests.support.case import ShellCase +from tests.support.paths import FILES +from tests.support.unit import skipIf +from tests.support.helpers import expensiveTest, generate_random_name + +# Import Salt Libs +from salt.config import cloud_providers_config + +# Import Third-Party Libs +try: + from oneandone.client import OneAndOneService # pylint: disable=unused-import + HAS_ONEANDONE = True +except ImportError: + HAS_ONEANDONE = False + +# Create the cloud instance name to be used throughout the tests +INSTANCE_NAME = generate_random_name('CLOUD-TEST-') +PROVIDER_NAME = 'oneandone' +DRIVER_NAME = 'oneandone' + + +@skipIf(HAS_ONEANDONE is False, 'salt-cloud requires >= 1and1 1.2.0') +class OneAndOneTest(ShellCase): + ''' + Integration tests for the 1and1 cloud provider + ''' + + @expensiveTest + def setUp(self): + ''' + Sets up the test requirements + ''' + super(OneAndOneTest, self).setUp() + + # check if appropriate cloud provider and profile files are present + profile_str = 'oneandone-config' + providers = self.run_cloud('--list-providers') + if profile_str + ':' not in providers: + self.skipTest( + 'Configuration file for {0} was not found. Check {0}.conf ' + 'files in tests/integration/files/conf/cloud.*.d/ to run ' + 'these tests.'.format(PROVIDER_NAME) + ) + + # check if api_token present + config = cloud_providers_config( + os.path.join( + FILES, + 'conf', + 'cloud.providers.d', + PROVIDER_NAME + '.conf' + ) + ) + + api_token = config[profile_str][DRIVER_NAME]['api_token'] + if api_token == '': + self.skipTest( + 'api_token must be provided to ' + 'run these tests. Check ' + 'tests/integration/files/conf/cloud.providers.d/{0}.conf' + .format(PROVIDER_NAME) + ) + + def test_list_images(self): + ''' + Tests the return of running the --list-images command for 1and1 + ''' + image_list = self.run_cloud('--list-images {0}'.format(PROVIDER_NAME)) + self.assertIn( + 'coreOSimage', + [i.strip() for i in image_list] + ) + + def test_instance(self): + ''' + Test creating an instance on 1and1 + ''' + # check if instance with salt installed returned + try: + self.assertIn( + INSTANCE_NAME, + [i.strip() for i in self.run_cloud( + '-p oneandone-test {0}'.format(INSTANCE_NAME), timeout=500 + )] + ) + except AssertionError: + self.run_cloud('-d {0} --assume-yes'.format(INSTANCE_NAME), timeout=500) + raise + + # delete the instance + try: + self.assertIn( + INSTANCE_NAME + ':', + [i.strip() for i in self.run_cloud( + '-d {0} --assume-yes'.format(INSTANCE_NAME), timeout=500 + )] + ) + except AssertionError: + raise + + def tearDown(self): + ''' + Clean up after tests + ''' + query = self.run_cloud('--query') + ret = ' {0}:'.format(INSTANCE_NAME) + + # if test instance is still present, delete it + if ret in query: + self.run_cloud('-d {0} --assume-yes'.format(INSTANCE_NAME), timeout=500) diff --git a/tests/integration/cloud/providers/test_profitbricks.py b/tests/integration/cloud/providers/test_profitbricks.py index d51fd9a0e38..bb4f30ad60a 100644 --- a/tests/integration/cloud/providers/test_profitbricks.py +++ b/tests/integration/cloud/providers/test_profitbricks.py @@ -6,18 +6,15 @@ # Import Python Libs from __future__ import absolute_import import os -import random -import string # Import Salt Testing Libs from tests.support.case import ShellCase from tests.support.paths import FILES from tests.support.unit import skipIf -from tests.support.helpers import expensiveTest +from tests.support.helpers import expensiveTest, generate_random_name # Import Salt Libs from salt.config import cloud_providers_config -from salt.ext.six.moves import range # Import Third-Party Libs try: @@ -26,18 +23,8 @@ try: except ImportError: HAS_PROFITBRICKS = False - -def __random_name(size=6): - ''' - Generates a random cloud instance name - ''' - return 'CLOUD-TEST-' + ''.join( - random.choice(string.ascii_uppercase + string.digits) - for x in range(size) - ) - # Create the cloud instance name to be used throughout the tests -INSTANCE_NAME = __random_name() +INSTANCE_NAME = generate_random_name('CLOUD-TEST-') PROVIDER_NAME = 'profitbricks' DRIVER_NAME = 'profitbricks' diff --git a/tests/integration/cloud/providers/test_rackspace.py b/tests/integration/cloud/providers/test_rackspace.py index 4c0231938dc..7496f452db8 100644 --- a/tests/integration/cloud/providers/test_rackspace.py +++ b/tests/integration/cloud/providers/test_rackspace.py @@ -6,18 +6,15 @@ # Import Python Libs from __future__ import absolute_import import os -import random -import string # Import Salt Testing Libs from tests.support.case import ShellCase from tests.support.paths import FILES from tests.support.unit import skipIf -from tests.support.helpers import expensiveTest +from tests.support.helpers import expensiveTest, generate_random_name # Import Salt Libs from salt.config import cloud_providers_config -from salt.ext.six.moves import range # Import Third-Party Libs try: @@ -26,18 +23,8 @@ try: except ImportError: HAS_LIBCLOUD = False - -def __random_name(size=6): - ''' - Generates a random cloud instance name - ''' - return 'CLOUD-TEST-' + ''.join( - random.choice(string.ascii_uppercase + string.digits) - for x in range(size) - ) - # Create the cloud instance name to be used throughout the tests -INSTANCE_NAME = __random_name() +INSTANCE_NAME = generate_random_name('CLOUD-TEST-') PROVIDER_NAME = 'rackspace' DRIVER_NAME = 'openstack' diff --git a/tests/integration/cloud/providers/test_virtualbox.py b/tests/integration/cloud/providers/test_virtualbox.py index 2d31b45bd0a..73325014b63 100644 --- a/tests/integration/cloud/providers/test_virtualbox.py +++ b/tests/integration/cloud/providers/test_virtualbox.py @@ -22,7 +22,7 @@ from tests.integration.cloud.helpers.virtualbox import (VirtualboxTestCase, DEPLOY_PROFILE_NAME) # Import Salt Libs -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import range from salt.config import cloud_providers_config, vm_profiles_config from salt.utils.virtualbox import (vb_xpcom_to_attribute_dict, diff --git a/tests/integration/cloud/providers/test_vmware.py b/tests/integration/cloud/providers/test_vmware.py index d776510ed40..5364ec04753 100644 --- a/tests/integration/cloud/providers/test_vmware.py +++ b/tests/integration/cloud/providers/test_vmware.py @@ -6,30 +6,17 @@ # Import Python Libs from __future__ import absolute_import import os -import random -import string # Import Salt Libs -from salt.config import cloud_providers_config +from salt.config import cloud_providers_config, cloud_config # Import Salt Testing LIbs from tests.support.case import ShellCase from tests.support.paths import FILES -from tests.support.helpers import expensiveTest -from salt.ext.six.moves import range - - -def __random_name(size=6): - ''' - Generates a radom cloud instance name - ''' - return 'CLOUD-TEST-' + ''.join( - random.choice(string.ascii_uppercase + string.digits) - for x in range(size) - ) +from tests.support.helpers import expensiveTest, generate_random_name # Create the cloud instance name to be used throughout the tests -INSTANCE_NAME = __random_name() +INSTANCE_NAME = generate_random_name('CLOUD-TEST-') PROVIDER_NAME = 'vmware' TIMEOUT = 500 @@ -90,12 +77,25 @@ class VMWareTest(ShellCase): Tests creating and deleting an instance on vmware and installing salt ''' # create the instance + profile = os.path.join( + FILES, + 'conf', + 'cloud.profiles.d', + PROVIDER_NAME + '.conf' + ) + + profile_config = cloud_config(profile) + disk_datastore = profile_config['vmware-test']['devices']['disk']['Hard disk 2']['datastore'] + instance = self.run_cloud('-p vmware-test {0}'.format(INSTANCE_NAME), timeout=TIMEOUT) ret_str = '{0}:'.format(INSTANCE_NAME) + disk_datastore_str = ' [{0}] {1}/Hard disk 2-flat.vmdk'.format(disk_datastore, INSTANCE_NAME) # check if instance returned with salt installed try: self.assertIn(ret_str, instance) + self.assertIn(disk_datastore_str, instance, + msg='Hard Disk 2 did not use the Datastore {0} '.format(disk_datastore)) except AssertionError: self.run_cloud('-d {0} --assume-yes'.format(INSTANCE_NAME), timeout=TIMEOUT) raise diff --git a/tests/integration/cloud/providers/test_vultr.py b/tests/integration/cloud/providers/test_vultr.py index 6d3d1687abe..1720dcef5ca 100644 --- a/tests/integration/cloud/providers/test_vultr.py +++ b/tests/integration/cloud/providers/test_vultr.py @@ -6,33 +6,18 @@ Integration tests for Vultr # Import Python Libs from __future__ import absolute_import import os -import random -import string import time # Import Salt Testing Libs from tests.support.case import ShellCase from tests.support.paths import FILES -from tests.support.helpers import expensiveTest +from tests.support.helpers import expensiveTest, generate_random_name # Import Salt Libs from salt.config import cloud_providers_config -# Import 3rd-party libs -from salt.ext.six.moves import range # pylint: disable=import-error,redefined-builtin - - -def __random_name(size=6): - ''' - Generates a random cloud instance name - ''' - return 'CLOUD-TEST-' + ''.join( - random.choice(string.ascii_uppercase + string.digits) - for x in range(size) - ) - # Create the cloud instance name to be used throughout the tests -INSTANCE_NAME = __random_name() +INSTANCE_NAME = generate_random_name('CLOUD-TEST-') PROVIDER_NAME = 'vultr' diff --git a/tests/integration/cloud/test_cloud.py b/tests/integration/cloud/test_cloud.py new file mode 100644 index 00000000000..983f34fdeda --- /dev/null +++ b/tests/integration/cloud/test_cloud.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- +''' +Integration tests for functions located in the salt.cloud.__init__.py file. +''' + +# Import Python Libs +from __future__ import absolute_import +import os +import random +import string + +# Import Salt Testing libs +from tests.support.case import ShellCase +from tests.support.helpers import expensiveTest +from tests.support.runtests import RUNTIME_VARS + +# Import Salt libs +import salt.cloud +from salt.ext.six.moves import range + + +def __random_name(size=6): + ''' + Generates a random cloud instance name + ''' + return 'CLOUD-TEST-' + ''.join( + random.choice(string.ascii_uppercase + string.digits) + for x in range(size) + ) + +# Create the cloud instance name to be used throughout the tests +INSTANCE_NAME = __random_name() + + +class CloudClientTestCase(ShellCase): + ''' + Integration tests for the CloudClient class. Uses DigitalOcean as a salt-cloud provider. + ''' + + @expensiveTest + def setUp(self): + self.config_file = os.path.join(RUNTIME_VARS.TMP_CONF_CLOUD_PROVIDER_INCLUDES, + 'digitalocean.conf') + self.provider_name = 'digitalocean-config' + self.image_name = '14.04.5 x64' + + # Use a --list-images salt-cloud call to see if the DigitalOcean provider is + # configured correctly before running any tests. + images = self.run_cloud('--list-images {0}'.format(self.provider_name)) + + if self.image_name not in [i.strip() for i in images]: + self.skipTest( + 'Image \'{0}\' was not found in image search. Is the {1} provider ' + 'configured correctly for this test?'.format( + self.provider_name, + self.image_name + ) + ) + + def test_cloud_client_create_and_delete(self): + ''' + Tests that a VM is created successfully when calling salt.cloud.CloudClient.create(), + which does not require a profile configuration. + + Also checks that salt.cloud.CloudClient.destroy() works correctly since this test needs + to remove the VM after creating it. + + This test was created as a regression check against Issue #41971. + ''' + cloud_client = salt.cloud.CloudClient(self.config_file) + + # Create the VM using salt.cloud.CloudClient.create() instead of calling salt-cloud + created = cloud_client.create( + provider=self.provider_name, + names=[INSTANCE_NAME], + image=self.image_name, + location='sfo1', + size='512mb', + vm_size='512mb' + ) + + # Check that the VM was created correctly + self.assertIn(INSTANCE_NAME, created) + + # Clean up after ourselves and delete the VM + deleted = cloud_client.destroy(names=[INSTANCE_NAME]) + + # Check that the VM was deleted correctly + self.assertIn(INSTANCE_NAME, deleted) diff --git a/tests/integration/files/conf/_ssh/sshd_config b/tests/integration/files/conf/_ssh/sshd_config index 3c6cb93f62e..36247d77544 100644 --- a/tests/integration/files/conf/_ssh/sshd_config +++ b/tests/integration/files/conf/_ssh/sshd_config @@ -4,7 +4,7 @@ Port 2827 ListenAddress 127.0.0.1 Protocol 2 -UsePrivilegeSeparation yes +#UsePrivilegeSeparation yes # Turn strict modes off so that we can operate in /tmp StrictModes no diff --git a/tests/integration/files/conf/cloud.profiles.d/oneandone.conf b/tests/integration/files/conf/cloud.profiles.d/oneandone.conf new file mode 100644 index 00000000000..e2cf63129d4 --- /dev/null +++ b/tests/integration/files/conf/cloud.profiles.d/oneandone.conf @@ -0,0 +1,15 @@ +oneandone-test: + provider: oneandone-config + description: Testing salt-cloud create operation + vcore: 2 + cores_per_processor: 1 + ram: 2 + password: P4$$w0rD + appliance_id: 8E3BAA98E3DFD37857810E0288DD8FBA + hdds: + - + is_main: true + size: 20 + - + is_main: false + size: 20 diff --git a/tests/integration/files/conf/cloud.profiles.d/vmware.conf b/tests/integration/files/conf/cloud.profiles.d/vmware.conf index 398100b3956..2e5d54f4b5b 100644 --- a/tests/integration/files/conf/cloud.profiles.d/vmware.conf +++ b/tests/integration/files/conf/cloud.profiles.d/vmware.conf @@ -7,6 +7,8 @@ vmware-test: disk: Hard disk 1: size: 30 + Hard disk 2: + size: 5 datastore: '' resourcepool: '' datastore: '' diff --git a/tests/integration/files/conf/cloud.providers.d/digital_ocean.conf b/tests/integration/files/conf/cloud.providers.d/digital_ocean.conf index 44f89b558f5..b0248442a93 100644 --- a/tests/integration/files/conf/cloud.providers.d/digital_ocean.conf +++ b/tests/integration/files/conf/cloud.providers.d/digital_ocean.conf @@ -1,5 +1,5 @@ digitalocean-config: - driver: digital_ocean + driver: digitalocean personal_access_token: '' ssh_key_file: '' ssh_key_name: '' diff --git a/tests/integration/files/conf/cloud.providers.d/oneandone.conf b/tests/integration/files/conf/cloud.providers.d/oneandone.conf new file mode 100644 index 00000000000..bc4b5f48dfd --- /dev/null +++ b/tests/integration/files/conf/cloud.providers.d/oneandone.conf @@ -0,0 +1,5 @@ +oneandone-config: + driver: oneandone + api_token: '' + ssh_private_key: ~/.ssh/id_rsa + ssh_public_key: ~/.ssh/id_rsa.pub diff --git a/tests/integration/files/conf/master b/tests/integration/files/conf/master index b78ecd91800..10c2eec07b5 100644 --- a/tests/integration/files/conf/master +++ b/tests/integration/files/conf/master @@ -78,6 +78,7 @@ nodegroups: redundant_minions: N@min or N@mins nodegroup_loop_a: N@nodegroup_loop_b nodegroup_loop_b: N@nodegroup_loop_a + missing_minion: L@minion,ghostminion mysql.host: localhost diff --git a/tests/integration/files/file/base/_grains/matcher_grain.py b/tests/integration/files/file/base/_grains/matcher_grain.py index 95db7fd9ef6..590ae41d16e 100644 --- a/tests/integration/files/file/base/_grains/matcher_grain.py +++ b/tests/integration/files/file/base/_grains/matcher_grain.py @@ -1,4 +1,5 @@ -#!/usr/bin/env python +# -*- coding: utf-8 -*- + def myfunction(): grains = {} grains['match'] = 'maker' diff --git a/tests/integration/files/file/base/_grains/test_custom_grain1.py b/tests/integration/files/file/base/_grains/test_custom_grain1.py index 983ca95ee31..99d0d419762 100644 --- a/tests/integration/files/file/base/_grains/test_custom_grain1.py +++ b/tests/integration/files/file/base/_grains/test_custom_grain1.py @@ -1,4 +1,5 @@ -#!/usr/bin/env python +# -*- coding: utf-8 -*- + def myfunction(): grains = {} grains['a_custom'] = {'k1': 'v1'} diff --git a/tests/integration/files/file/base/_grains/test_custom_grain2.py b/tests/integration/files/file/base/_grains/test_custom_grain2.py index 239b8f3cb7e..5783a5f130c 100644 --- a/tests/integration/files/file/base/_grains/test_custom_grain2.py +++ b/tests/integration/files/file/base/_grains/test_custom_grain2.py @@ -1,4 +1,5 @@ -#!/usr/bin/env python +# -*- coding: utf-8 -*- + def myfunction(): grains = {} grains['a_custom'] = {'k2': 'v2'} diff --git a/tests/integration/files/file/base/_modules/runtests_decorators.py b/tests/integration/files/file/base/_modules/runtests_decorators.py index b43168c9993..512d0b7014c 100644 --- a/tests/integration/files/file/base/_modules/runtests_decorators.py +++ b/tests/integration/files/file/base/_modules/runtests_decorators.py @@ -9,6 +9,11 @@ import salt.utils.decorators def _fallbackfunc(): + ''' + CLI Example: + + .. code-block:: bash + ''' return False, 'fallback' @@ -33,11 +38,21 @@ def booldependsTrue(): @salt.utils.decorators.depends(False) def booldependsFalse(): + ''' + CLI Example: + + .. code-block:: bash + ''' return True @salt.utils.decorators.depends('time') def depends(): + ''' + CLI Example: + + .. code-block:: bash + ''' ret = {'ret': True, 'time': time.time()} return ret @@ -45,6 +60,11 @@ def depends(): @salt.utils.decorators.depends('time123') def missing_depends(): + ''' + CLI Example: + + .. code-block:: bash + ''' return True @@ -62,6 +82,11 @@ def depends_will_not_fallback(): @salt.utils.decorators.depends('time123', fallback_function=_fallbackfunc) def missing_depends_will_fallback(): + ''' + CLI Example: + + .. code-block:: bash + ''' ret = {'ret': True, 'time': time.time()} return ret diff --git a/tests/integration/files/file/base/_modules/runtests_helpers.py b/tests/integration/files/file/base/_modules/runtests_helpers.py index 0aa5df742ad..f3d49095372 100644 --- a/tests/integration/files/file/base/_modules/runtests_helpers.py +++ b/tests/integration/files/file/base/_modules/runtests_helpers.py @@ -16,16 +16,16 @@ import fnmatch import tempfile # Import salt libs -import salt.utils +import salt.utils.platform # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six SYS_TMP_DIR = os.path.realpath( # Avoid ${TMPDIR} and gettempdir() on MacOS as they yield a base path too long # for unix sockets: ``error: AF_UNIX path too long`` # Gentoo Portage prefers ebuild tests are rooted in ${TMPDIR} - os.environ.get('TMPDIR', tempfile.gettempdir()) if not salt.utils.is_darwin() else '/tmp' + os.environ.get('TMPDIR', tempfile.gettempdir()) if not salt.utils.platform.is_darwin() else '/tmp' ) # This tempdir path is defined on tests.integration.__init__ TMP = os.path.join(SYS_TMP_DIR, 'salt-tests-tmpdir') @@ -51,9 +51,13 @@ def get_invalid_docs(): allow_failure = ( 'cmd.win_runas', 'cp.recv', + 'cp.recv_chunked', 'glance.warn_until', 'ipset.long_range', + 'libcloud_compute.get_driver', 'libcloud_dns.get_driver', + 'libcloud_loadbalancer.get_driver', + 'libcloud_storage.get_driver', 'log.critical', 'log.debug', 'log.error', @@ -73,6 +77,7 @@ def get_invalid_docs(): 'state.apply', 'status.list2cmdline', 'swift.head', + 'test.rand_str', 'travisci.parse_qs', 'vsphere.clean_kwargs', 'vsphere.disconnect', diff --git a/tests/integration/files/file/base/issue-42116-cli-pillar-override.sls b/tests/integration/files/file/base/issue-42116-cli-pillar-override.sls new file mode 100644 index 00000000000..d3b2b239ecc --- /dev/null +++ b/tests/integration/files/file/base/issue-42116-cli-pillar-override.sls @@ -0,0 +1,2 @@ +ping -c 2 {{ pillar['myhost'] }}: + cmd.run diff --git a/tests/integration/files/file/base/jinja_salt_contains_function.sls b/tests/integration/files/file/base/jinja_salt_contains_function.sls new file mode 100644 index 00000000000..24978d1799c --- /dev/null +++ b/tests/integration/files/file/base/jinja_salt_contains_function.sls @@ -0,0 +1,10 @@ +{% set salt_foo_bar_exist = 'foo.bar' in salt %} +{% set salt_test_ping_exist = 'test.ping' in salt %} + +test-ping-exist: + test.succeed_without_changes: + - name: salt_test_ping_exist_{{ salt_test_ping_exist }} + +foo-bar-not-exist: + test.succeed_without_changes: + - name: salt_foo_bar_exist_{{ salt_foo_bar_exist }} diff --git a/tests/integration/files/file/base/script.py b/tests/integration/files/file/base/script.py index dc347ff1ff2..ea1c887150d 100644 --- a/tests/integration/files/file/base/script.py +++ b/tests/integration/files/file/base/script.py @@ -1,4 +1,7 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- + +from __future__ import absolute_import import sys print(' '.join(sys.argv[1:])) diff --git a/tests/integration/files/log_handlers/runtests_log_handler.py b/tests/integration/files/log_handlers/runtests_log_handler.py index 63b2546c357..82e1509f72f 100644 --- a/tests/integration/files/log_handlers/runtests_log_handler.py +++ b/tests/integration/files/log_handlers/runtests_log_handler.py @@ -23,6 +23,7 @@ from multiprocessing import Queue import msgpack # Import Salt libs +from salt.ext import six import salt.log.setup log = logging.getLogger(__name__) @@ -33,6 +34,8 @@ __virtualname__ = 'runtests_log_handler' def __virtual__(): if 'runtests_log_port' not in __opts__: return False, "'runtests_log_port' not in options" + if six.PY3: + return False, "runtests external logging handler is temporarily disabled for Python 3 tests" return True diff --git a/tests/integration/grains/test_core.py b/tests/integration/grains/test_core.py index a219b50f3d4..25ac1370963 100644 --- a/tests/integration/grains/test_core.py +++ b/tests/integration/grains/test_core.py @@ -10,9 +10,9 @@ from __future__ import absolute_import from tests.support.case import ModuleCase from tests.support.unit import skipIf -# Import salt libs -import salt.utils -if salt.utils.is_windows(): +# Import Salt libs +import salt.utils.platform +if salt.utils.platform.is_windows(): try: import salt.modules.reg except ImportError: @@ -23,7 +23,7 @@ class TestGrainsCore(ModuleCase): ''' Test the core grains grains ''' - @skipIf(not salt.utils.is_windows(), 'Only run on Windows') + @skipIf(not salt.utils.platform.is_windows(), 'Only run on Windows') def test_win_cpu_model(self): ''' test grains['cpu_model'] diff --git a/tests/integration/minion/test_blackout.py b/tests/integration/minion/test_blackout.py index e91701cba78..2a7764c637b 100644 --- a/tests/integration/minion/test_blackout.py +++ b/tests/integration/minion/test_blackout.py @@ -15,7 +15,7 @@ from tests.support.paths import PILLAR_DIR from tests.support.helpers import destructiveTest # Import Salt libs -import salt.utils +import salt.utils.files BLACKOUT_PILLAR = os.path.join(PILLAR_DIR, 'base', 'blackout.sls') @@ -30,7 +30,7 @@ class MinionBlackoutTestCase(ModuleCase): ''' setup minion blackout mode ''' - with salt.utils.fopen(BLACKOUT_PILLAR, 'w') as wfh: + with salt.utils.files.fopen(BLACKOUT_PILLAR, 'w') as wfh: wfh.write(blackout_data) self.run_function('saltutil.refresh_pillar') sleep(5) # wait for minion to enter blackout mode @@ -39,7 +39,7 @@ class MinionBlackoutTestCase(ModuleCase): ''' takedown minion blackout mode ''' - with salt.utils.fopen(BLACKOUT_PILLAR, 'w') as blackout_pillar: + with salt.utils.files.fopen(BLACKOUT_PILLAR, 'w') as blackout_pillar: blackout_pillar.write(textwrap.dedent('''\ minion_blackout: False ''')) diff --git a/tests/integration/minion/test_pillar.py b/tests/integration/minion/test_pillar.py index 1b13a0eba79..a2fa5879140 100644 --- a/tests/integration/minion/test_pillar.py +++ b/tests/integration/minion/test_pillar.py @@ -21,10 +21,11 @@ from tests.support.helpers import requires_system_grains # Import 3rd-party libs import yaml -import salt.ext.six as six +from salt.ext import six # Import salt libs -import salt.utils +import salt.utils.files +import salt.utils.path import salt.pillar as pillar log = logging.getLogger(__name__) @@ -190,7 +191,7 @@ GPG_PILLAR_DECRYPTED = { } -@skipIf(not salt.utils.which('gpg'), 'GPG is not installed') +@skipIf(not salt.utils.path.which('gpg'), 'GPG is not installed') class DecryptGPGPillarTest(ModuleCase): ''' Tests for pillar decryption @@ -226,13 +227,13 @@ class DecryptGPGPillarTest(ModuleCase): log.debug('Result:\n%s', output) os.makedirs(PILLAR_BASE) - with salt.utils.fopen(TOP_SLS, 'w') as fp_: + with salt.utils.files.fopen(TOP_SLS, 'w') as fp_: fp_.write(textwrap.dedent('''\ base: '*': - gpg ''')) - with salt.utils.fopen(GPG_SLS, 'w') as fp_: + with salt.utils.files.fopen(GPG_SLS, 'w') as fp_: fp_.write(GPG_PILLAR_YAML) @classmethod diff --git a/tests/integration/modules/test_archive.py b/tests/integration/modules/test_archive.py index c579e1b4167..8898c0ddb26 100644 --- a/tests/integration/modules/test_archive.py +++ b/tests/integration/modules/test_archive.py @@ -15,7 +15,8 @@ from tests.support.paths import TMP from tests.support.helpers import destructiveTest # Import salt libs -import salt.utils +import salt.utils.files +import salt.utils.path # Import 3rd party libs try: @@ -59,23 +60,23 @@ class ArchiveTest(ModuleCase): # Create source os.makedirs(self.src) - with salt.utils.fopen(os.path.join(self.src, 'file'), 'w') as theorem: + with salt.utils.files.fopen(os.path.join(self.src, 'file'), 'w') as theorem: theorem.write(textwrap.dedent(r'''\ Compression theorem of computational complexity theory: Given a Gödel numbering $φ$ of the computable functions and a Blum complexity measure $Φ$ where a complexity class for a boundary function $f$ is defined as - + $\mathrm C(f) := \{φ_i ∈ \mathbb R^{(1)} | (∀^∞ x) Φ_i(x) ≤ f(x)\}$. - + Then there exists a total computable function $f$ so that for all $i$ - + $\mathrm{Dom}(φ_i) = \mathrm{Dom}(φ_{f(i)})$ - + and - + $\mathrm C(φ_i) ⊊ \mathrm{C}(φ_{f(i)})$. ''')) @@ -118,7 +119,7 @@ class ArchiveTest(ModuleCase): self.assertTrue(dir_in_ret) self.assertTrue(file_in_ret) - @skipIf(not salt.utils.which('tar'), 'Cannot find tar executable') + @skipIf(not salt.utils.path.which('tar'), 'Cannot find tar executable') def test_tar_pack(self): ''' Validate using the tar function to create archives @@ -132,7 +133,7 @@ class ArchiveTest(ModuleCase): self._tear_down() - @skipIf(not salt.utils.which('tar'), 'Cannot find tar executable') + @skipIf(not salt.utils.path.which('tar'), 'Cannot find tar executable') def test_tar_unpack(self): ''' Validate using the tar function to extract archives @@ -147,7 +148,7 @@ class ArchiveTest(ModuleCase): self._tear_down() - @skipIf(not salt.utils.which('gzip'), 'Cannot find gzip executable') + @skipIf(not salt.utils.path.which('gzip'), 'Cannot find gzip executable') def test_gzip(self): ''' Validate using the gzip function @@ -161,8 +162,8 @@ class ArchiveTest(ModuleCase): self._tear_down() - @skipIf(not salt.utils.which('gzip'), 'Cannot find gzip executable') - @skipIf(not salt.utils.which('gunzip'), 'Cannot find gunzip executable') + @skipIf(not salt.utils.path.which('gzip'), 'Cannot find gzip executable') + @skipIf(not salt.utils.path.which('gunzip'), 'Cannot find gunzip executable') def test_gunzip(self): ''' Validate using the gunzip function @@ -177,7 +178,7 @@ class ArchiveTest(ModuleCase): self._tear_down() - @skipIf(not salt.utils.which('zip'), 'Cannot find zip executable') + @skipIf(not salt.utils.path.which('zip'), 'Cannot find zip executable') def test_cmd_zip(self): ''' Validate using the cmd_zip function @@ -191,8 +192,8 @@ class ArchiveTest(ModuleCase): self._tear_down() - @skipIf(not salt.utils.which('zip'), 'Cannot find zip executable') - @skipIf(not salt.utils.which('unzip'), 'Cannot find unzip executable') + @skipIf(not salt.utils.path.which('zip'), 'Cannot find zip executable') + @skipIf(not salt.utils.path.which('unzip'), 'Cannot find unzip executable') def test_cmd_unzip(self): ''' Validate using the cmd_unzip function @@ -236,7 +237,7 @@ class ArchiveTest(ModuleCase): self._tear_down() - @skipIf(not salt.utils.which('rar'), 'Cannot find rar executable') + @skipIf(not salt.utils.path.which('rar'), 'Cannot find rar executable') def test_rar(self): ''' Validate using the rar function @@ -250,8 +251,8 @@ class ArchiveTest(ModuleCase): self._tear_down() - @skipIf(not salt.utils.which('rar'), 'Cannot find rar executable') - @skipIf(not salt.utils.which('unrar'), 'Cannot find unrar executable') + @skipIf(not salt.utils.path.which('rar'), 'Cannot find rar executable') + @skipIf(not salt.utils.path.which('unrar'), 'Cannot find unrar executable') def test_unrar(self): ''' Validate using the unrar function diff --git a/tests/integration/modules/test_beacons.py b/tests/integration/modules/test_beacons.py index aee7ba8fdb6..7c906caeb49 100644 --- a/tests/integration/modules/test_beacons.py +++ b/tests/integration/modules/test_beacons.py @@ -35,7 +35,7 @@ class BeaconsAddDeleteTest(ModuleCase): ''' Test adding and deleting a beacon ''' - _add = self.run_function('beacons.add', ['ps', [{'apache2': 'stopped'}]]) + _add = self.run_function('beacons.add', ['ps', [{'processes': {'apache2': 'stopped'}}]]) self.assertTrue(_add['result']) # save added beacon @@ -58,7 +58,7 @@ class BeaconsTest(ModuleCase): @classmethod def tearDownClass(cls): - if os.path.isfile(cls.beacons_config_file_path): + if cls.beacons_config_file_path and os.path.isfile(cls.beacons_config_file_path): os.unlink(cls.beacons_config_file_path) def setUp(self): @@ -71,7 +71,7 @@ class BeaconsTest(ModuleCase): self.__class__.beacons_config_file_path = os.path.join(self.minion_conf_d_dir, 'beacons.conf') try: # Add beacon to disable - self.run_function('beacons.add', ['ps', [{'apache2': 'stopped'}]]) + self.run_function('beacons.add', ['ps', [{'processes': {'apache2': 'stopped'}}]]) self.run_function('beacons.save') except CommandExecutionError: self.skipTest('Unable to add beacon') @@ -143,6 +143,6 @@ class BeaconsTest(ModuleCase): # list beacons ret = self.run_function('beacons.list', return_yaml=False) if 'enabled' in ret: - self.assertEqual(ret, {'ps': [{'apache2': 'stopped'}], 'enabled': True}) + self.assertEqual(ret, {'ps': [{'processes': {'apache2': 'stopped'}}], 'enabled': True}) else: - self.assertEqual(ret, {'ps': {'apache': 'stopped'}}) + self.assertEqual(ret, {'ps': [{'processes': {'apache2': 'stopped'}}]}) diff --git a/tests/integration/modules/test_cmdmod.py b/tests/integration/modules/test_cmdmod.py index a1ecde08e72..be306cbb483 100644 --- a/tests/integration/modules/test_cmdmod.py +++ b/tests/integration/modules/test_cmdmod.py @@ -15,13 +15,13 @@ from tests.support.helpers import ( ) # Import salt libs -import salt.utils +import salt.utils.path # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six -AVAILABLE_PYTHON_EXECUTABLE = salt.utils.which_bin([ +AVAILABLE_PYTHON_EXECUTABLE = salt.utils.path.which_bin([ 'python', 'python2', 'python2.6', @@ -187,6 +187,34 @@ class CMDModuleTest(ModuleCase): code]).rstrip(), 'cheese') + def test_exec_code_with_single_arg(self): + ''' + cmd.exec_code + ''' + code = textwrap.dedent('''\ + import sys + sys.stdout.write(sys.argv[1])''') + arg = 'cheese' + self.assertEqual(self.run_function('cmd.exec_code', + [AVAILABLE_PYTHON_EXECUTABLE, + code], + args=arg).rstrip(), + arg) + + def test_exec_code_with_multiple_args(self): + ''' + cmd.exec_code + ''' + code = textwrap.dedent('''\ + import sys + sys.stdout.write(sys.argv[1])''') + arg = 'cheese' + self.assertEqual(self.run_function('cmd.exec_code', + [AVAILABLE_PYTHON_EXECUTABLE, + code], + args=[arg, 'test']).rstrip(), + arg) + def test_quotes(self): ''' cmd.run with quoted command diff --git a/tests/integration/modules/test_cp.py b/tests/integration/modules/test_cp.py index 40720029a86..d8a5dd8a62f 100644 --- a/tests/integration/modules/test_cp.py +++ b/tests/integration/modules/test_cp.py @@ -4,7 +4,6 @@ from __future__ import absolute_import import os import uuid -import shutil import hashlib import logging import psutil @@ -20,8 +19,13 @@ from tests.support.unit import skipIf import tests.support.paths as paths # Import salt libs -import salt.ext.six as six -import salt.utils +import salt.utils.files +import salt.utils.path +import salt.utils.platform +import salt.utils.stringutils + +# Import 3rd-party libs +from salt.ext import six log = logging.getLogger(__name__) @@ -41,7 +45,23 @@ class CPModuleTest(ModuleCase): 'salt://grail/scene33', tgt, ]) - with salt.utils.fopen(tgt, 'r') as scene: + with salt.utils.files.fopen(tgt, 'r') as scene: + data = scene.read() + self.assertIn('KNIGHT: They\'re nervous, sire.', data) + self.assertNotIn('bacon', data) + + def test_get_file_to_dir(self): + ''' + cp.get_file + ''' + tgt = os.path.join(paths.TMP, '') + self.run_function( + 'cp.get_file', + [ + 'salt://grail/scene33', + tgt, + ]) + with salt.utils.files.fopen(tgt + 'scene33', 'r') as scene: data = scene.read() self.assertIn('KNIGHT: They\'re nervous, sire.', data) self.assertNotIn('bacon', data) @@ -59,7 +79,7 @@ class CPModuleTest(ModuleCase): ], template='jinja' ) - with salt.utils.fopen(tgt, 'r') as cheese: + with salt.utils.files.fopen(tgt, 'r') as cheese: data = cheese.read() self.assertIn('Gromit', data) self.assertNotIn('bacon', data) @@ -70,10 +90,10 @@ class CPModuleTest(ModuleCase): ''' tgt = os.path.join(paths.TMP, 'file.big') src = os.path.join(paths.FILES, 'file', 'base', 'file.big') - with salt.utils.fopen(src, 'r') as fp_: + with salt.utils.files.fopen(src, 'r') as fp_: data = fp_.read() if six.PY3: - data = salt.utils.to_bytes(data) + data = salt.utils.stringutils.to_bytes(data) hash_str = hashlib.md5(data).hexdigest() self.run_function( @@ -84,12 +104,12 @@ class CPModuleTest(ModuleCase): ], gzip=5 ) - with salt.utils.fopen(tgt, 'r') as scene: + with salt.utils.files.fopen(tgt, 'r') as scene: data = scene.read() self.assertIn('KNIGHT: They\'re nervous, sire.', data) self.assertNotIn('bacon', data) if six.PY3: - data = salt.utils.to_bytes(data) + data = salt.utils.stringutils.to_bytes(data) self.assertEqual(hash_str, hashlib.md5(data).hexdigest()) def test_get_file_makedirs(self): @@ -106,7 +126,7 @@ class CPModuleTest(ModuleCase): makedirs=True ) self.addCleanup(shutil.rmtree, os.path.join(paths.TMP, 'make'), ignore_errors=True) - with salt.utils.fopen(tgt, 'r') as scene: + with salt.utils.files.fopen(tgt, 'r') as scene: data = scene.read() self.assertIn('KNIGHT: They\'re nervous, sire.', data) self.assertNotIn('bacon', data) @@ -120,7 +140,7 @@ class CPModuleTest(ModuleCase): 'cp.get_template', ['salt://grail/scene33', tgt], spam='bacon') - with salt.utils.fopen(tgt, 'r') as scene: + with salt.utils.files.fopen(tgt, 'r') as scene: data = scene.read() self.assertIn('bacon', data) self.assertNotIn('spam', data) @@ -171,7 +191,7 @@ class CPModuleTest(ModuleCase): 'salt://grail/scene33', tgt, ]) - with salt.utils.fopen(tgt, 'r') as scene: + with salt.utils.files.fopen(tgt, 'r') as scene: data = scene.read() self.assertIn('KNIGHT: They\'re nervous, sire.', data) self.assertNotIn('bacon', data) @@ -190,7 +210,7 @@ class CPModuleTest(ModuleCase): makedirs=True ) self.addCleanup(shutil.rmtree, os.path.join(paths.TMP, 'make'), ignore_errors=True) - with salt.utils.fopen(tgt, 'r') as scene: + with salt.utils.files.fopen(tgt, 'r') as scene: data = scene.read() self.assertIn('KNIGHT: They\'re nervous, sire.', data) self.assertNotIn('bacon', data) @@ -204,7 +224,7 @@ class CPModuleTest(ModuleCase): [ 'salt://grail/scene33', ]) - with salt.utils.fopen(ret, 'r') as scene: + with salt.utils.files.fopen(ret, 'r') as scene: data = scene.read() self.assertIn('KNIGHT: They\'re nervous, sire.', data) self.assertNotIn('bacon', data) @@ -235,6 +255,22 @@ class CPModuleTest(ModuleCase): ]) self.assertEqual(ret, False) + def test_get_url_to_dir(self): + ''' + cp.get_url with salt:// source + ''' + tgt = os.path.join(paths.TMP, '') + self.run_function( + 'cp.get_url', + [ + 'salt://grail/scene33', + tgt, + ]) + with salt.utils.files.fopen(tgt + 'scene33', 'r') as scene: + data = scene.read() + self.assertIn('KNIGHT: They\'re nervous, sire.', data) + self.assertNotIn('bacon', data) + def test_get_url_https(self): ''' cp.get_url with https:// source given @@ -246,7 +282,7 @@ class CPModuleTest(ModuleCase): 'https://repo.saltstack.com/index.html', tgt, ]) - with salt.utils.fopen(tgt, 'r') as instructions: + with salt.utils.files.fopen(tgt, 'r') as instructions: data = instructions.read() self.assertIn('Bootstrap', data) self.assertIn('Debian', data) @@ -262,7 +298,7 @@ class CPModuleTest(ModuleCase): [ 'https://repo.saltstack.com/index.html', ]) - with salt.utils.fopen(ret, 'r') as instructions: + with salt.utils.files.fopen(ret, 'r') as instructions: data = instructions.read() self.assertIn('Bootstrap', data) self.assertIn('Debian', data) @@ -297,7 +333,7 @@ class CPModuleTest(ModuleCase): src, tgt, ]) - with salt.utils.fopen(ret, 'r') as scene: + with salt.utils.files.fopen(ret, 'r') as scene: data = scene.read() self.assertIn('KNIGHT: They\'re nervous, sire.', data) self.assertNotIn('bacon', data) @@ -382,7 +418,7 @@ class CPModuleTest(ModuleCase): [ 'salt://grail/scene33', ]) - with salt.utils.fopen(ret, 'r') as scene: + with salt.utils.files.fopen(ret, 'r') as scene: data = scene.read() self.assertIn('KNIGHT: They\'re nervous, sire.', data) self.assertNotIn('bacon', data) @@ -397,7 +433,7 @@ class CPModuleTest(ModuleCase): ['salt://grail/scene33', 'salt://grail/36/scene'], ]) for path in ret: - with salt.utils.fopen(path, 'r') as scene: + with salt.utils.files.fopen(path, 'r') as scene: data = scene.read() self.assertIn('ARTHUR:', data) self.assertNotIn('bacon', data) @@ -417,15 +453,15 @@ class CPModuleTest(ModuleCase): cp.cache_local_file ''' src = os.path.join(paths.TMP, 'random') - with salt.utils.fopen(src, 'w+') as fn_: + with salt.utils.files.fopen(src, 'w+') as fn_: fn_.write('foo') ret = self.run_function( 'cp.cache_local_file', [src]) - with salt.utils.fopen(ret, 'r') as cp_: + with salt.utils.files.fopen(ret, 'r') as cp_: self.assertEqual(cp_.read(), 'foo') - @skipIf(not salt.utils.which('nginx'), 'nginx not installed') + @skipIf(not salt.utils.path.which('nginx'), 'nginx not installed') @skip_if_not_root def test_cache_remote_file(self): ''' @@ -445,11 +481,11 @@ class CPModuleTest(ModuleCase): os.mkdir(dirname) # Write the temp file - with salt.utils.fopen(os.path.join(nginx_root_dir, 'actual_file'), 'w') as fp_: + with salt.utils.files.fopen(os.path.join(nginx_root_dir, 'actual_file'), 'w') as fp_: fp_.write(file_contents) # Write the nginx config - with salt.utils.fopen(nginx_conf, 'w') as fp_: + with salt.utils.files.fopen(nginx_conf, 'w') as fp_: fp_.write(textwrap.dedent( '''\ user root; @@ -489,7 +525,7 @@ class CPModuleTest(ModuleCase): [['nginx', '-c', nginx_conf]], python_shell=False ) - with salt.utils.fopen(nginx_pidfile) as fp_: + with salt.utils.files.fopen(nginx_pidfile) as fp_: nginx_pid = int(fp_.read().strip()) nginx_proc = psutil.Process(pid=nginx_pid) self.addCleanup(nginx_proc.send_signal, signal.SIGQUIT) @@ -498,7 +534,7 @@ class CPModuleTest(ModuleCase): url = url_prefix + (code or 'actual_file') log.debug('attempting to cache %s', url) ret = self.run_function('cp.cache_file', [url]) - with salt.utils.fopen(ret) as fp_: + with salt.utils.files.fopen(ret) as fp_: cached_contents = fp_.read() self.assertEqual(cached_contents, file_contents) @@ -524,7 +560,7 @@ class CPModuleTest(ModuleCase): ret = self.run_function('cp.list_minion') found = False search = 'grail/scene33' - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): search = r'grail\scene33' for path in ret: if search in path: @@ -568,10 +604,10 @@ class CPModuleTest(ModuleCase): [ 'salt://grail/scene33', ]) - with salt.utils.fopen(path, 'r') as fn_: + with salt.utils.files.fopen(path, 'r') as fn_: data = fn_.read() if six.PY3: - data = salt.utils.to_bytes(data) + data = salt.utils.stringutils.to_bytes(data) self.assertEqual( sha256_hash['hsum'], hashlib.sha256(data).hexdigest()) @@ -582,7 +618,7 @@ class CPModuleTest(ModuleCase): tgt = os.path.join(paths.TMP, 'cheese') try: self.run_function('cp.get_file', ['salt://cheese', tgt]) - with salt.utils.fopen(tgt, 'r') as cheese: + with salt.utils.files.fopen(tgt, 'r') as cheese: data = cheese.read() self.assertIn('Gromit', data) self.assertNotIn('Comte', data) @@ -593,7 +629,7 @@ class CPModuleTest(ModuleCase): tgt = os.path.join(paths.TMP, 'cheese') try: self.run_function('cp.get_file', ['salt://cheese?saltenv=prod', tgt]) - with salt.utils.fopen(tgt, 'r') as cheese: + with salt.utils.files.fopen(tgt, 'r') as cheese: data = cheese.read() self.assertIn('Gromit', data) self.assertIn('Comte', data) diff --git a/tests/integration/modules/test_darwin_sysctl.py b/tests/integration/modules/test_darwin_sysctl.py index f25e96e6cbb..411dbcc034c 100644 --- a/tests/integration/modules/test_darwin_sysctl.py +++ b/tests/integration/modules/test_darwin_sysctl.py @@ -9,7 +9,6 @@ import os import random # Import Salt Libs -import salt.utils import salt.utils.files from salt.exceptions import CommandExecutionError @@ -142,8 +141,8 @@ class DarwinSysctlModuleTest(ModuleCase): ''' # Create new temporary file path and open needed files temp_path = salt.utils.files.mkstemp() - with salt.utils.fopen(CONFIG, 'r') as org_conf: - with salt.utils.fopen(temp_path, 'w') as temp_sysconf: + with salt.utils.files.fopen(CONFIG, 'r') as org_conf: + with salt.utils.files.fopen(temp_path, 'w') as temp_sysconf: # write sysctl lines to temp file for line in org_conf: temp_sysconf.write(line) @@ -158,8 +157,8 @@ class DarwinSysctlModuleTest(ModuleCase): os.remove(CONFIG) # write temp lines to sysctl file to restore - with salt.utils.fopen(self.conf, 'r') as temp_sysctl: - with salt.utils.fopen(CONFIG, 'w') as sysctl: + with salt.utils.files.fopen(self.conf, 'r') as temp_sysctl: + with salt.utils.files.fopen(CONFIG, 'w') as sysctl: for line in temp_sysctl: sysctl.write(line) @@ -170,7 +169,7 @@ class DarwinSysctlModuleTest(ModuleCase): ''' Returns True if given line is present in file ''' - with salt.utils.fopen(conf_file, 'r') as f_in: + with salt.utils.files.fopen(conf_file, 'r') as f_in: for line in f_in: if to_find in line: return True diff --git a/tests/integration/modules/test_decorators.py b/tests/integration/modules/test_decorators.py index a6f919678dc..dc896c343ef 100644 --- a/tests/integration/modules/test_decorators.py +++ b/tests/integration/modules/test_decorators.py @@ -22,8 +22,9 @@ class DecoratorTest(ModuleCase): self.assertTrue(isinstance(ret['time'], float)) def test_missing_depends(self): - self.assertIn( - 'is not available', + self.assertEqual( + {'runtests_decorators.missing_depends_will_fallback': '\n CLI Example:\n\n ', + 'runtests_decorators.missing_depends': "'runtests_decorators.missing_depends' is not available."}, self.run_function('runtests_decorators.missing_depends' ) ) diff --git a/tests/integration/modules/test_disk.py b/tests/integration/modules/test_disk.py index d3672d6df8b..66923573bc1 100644 --- a/tests/integration/modules/test_disk.py +++ b/tests/integration/modules/test_disk.py @@ -10,16 +10,16 @@ from tests.support.case import ModuleCase from tests.support.unit import skipIf from tests.support.helpers import destructiveTest -# Import salt libs -import salt.utils +# Import Salt libs +import salt.utils.platform # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six @destructiveTest -@skipIf(salt.utils.is_windows(), 'No mtab on Windows') -@skipIf(salt.utils.is_darwin(), 'No mtab on Darwin') +@skipIf(salt.utils.platform.is_windows(), 'No mtab on Windows') +@skipIf(salt.utils.platform.is_darwin(), 'No mtab on Darwin') class DiskModuleVirtualizationTest(ModuleCase): ''' Test to make sure we return a clean result under Docker. Refs #8976 @@ -52,7 +52,7 @@ class DiskModuleTest(ModuleCase): self.assertTrue(isinstance(ret, dict)) if not isinstance(ret, dict): return - if salt.utils.is_darwin(): + if salt.utils.platform.is_darwin(): for key, val in six.iteritems(ret): self.assertTrue('filesystem' in val) self.assertTrue('512-blocks' in val) @@ -70,7 +70,7 @@ class DiskModuleTest(ModuleCase): self.assertTrue('available' in val) self.assertTrue('capacity' in val) - @skipIf(salt.utils.is_windows(), 'inode info not available on Windows') + @skipIf(salt.utils.platform.is_windows(), 'inode info not available on Windows') def test_inodeusage(self): ''' disk.inodeusage diff --git a/tests/integration/modules/test_dockermod.py b/tests/integration/modules/test_dockermod.py index 64f981aae00..f3bc50fa1e1 100644 --- a/tests/integration/modules/test_dockermod.py +++ b/tests/integration/modules/test_dockermod.py @@ -18,7 +18,7 @@ from tests.support.helpers import destructiveTest from tests.support.mixins import SaltReturnAssertsMixin # Import Salt Libs -import salt.utils +import salt.utils.path # Import 3rd-party libs from salt.ext.six.moves import range # pylint: disable=import-error,redefined-builtin @@ -43,7 +43,7 @@ def with_random_name(func): @destructiveTest -@skipIf(not salt.utils.which('dockerd'), 'Docker not installed') +@skipIf(not salt.utils.path.which('dockerd'), 'Docker not installed') class DockerCallTestCase(ModuleCase, SaltReturnAssertsMixin): ''' Test docker_container states diff --git a/tests/integration/modules/test_event.py b/tests/integration/modules/test_event.py index 080b7b8e02e..a7d672051bd 100644 --- a/tests/integration/modules/test_event.py +++ b/tests/integration/modules/test_event.py @@ -16,7 +16,7 @@ import threading from tests.support.case import ModuleCase # Import salt libs -from salt.utils import event +import salt.utils.event as event # Import 3rd-party libs from salt.ext.six.moves.queue import Queue, Empty # pylint: disable=import-error,no-name-in-module diff --git a/tests/integration/modules/test_file.py b/tests/integration/modules/test_file.py index b4ff4f31d80..be96ee4c01b 100644 --- a/tests/integration/modules/test_file.py +++ b/tests/integration/modules/test_file.py @@ -15,7 +15,8 @@ from tests.support.unit import skipIf from tests.support.paths import FILES, TMP # Import salt libs -import salt.utils +import salt.utils.files +import salt.utils.platform class FileModuleTest(ModuleCase): @@ -24,7 +25,7 @@ class FileModuleTest(ModuleCase): ''' def setUp(self): self.myfile = os.path.join(TMP, 'myfile') - with salt.utils.fopen(self.myfile, 'w+') as fp: + with salt.utils.files.fopen(self.myfile, 'w+') as fp: fp.write('Hello\n') self.mydir = os.path.join(TMP, 'mydir/isawesome') if not os.path.isdir(self.mydir): @@ -50,7 +51,7 @@ class FileModuleTest(ModuleCase): shutil.rmtree(self.mydir, ignore_errors=True) super(FileModuleTest, self).tearDown() - @skipIf(salt.utils.is_windows(), 'No chgrp on Windows') + @skipIf(salt.utils.platform.is_windows(), 'No chgrp on Windows') def test_chown(self): user = getpass.getuser() if sys.platform == 'darwin': @@ -63,14 +64,14 @@ class FileModuleTest(ModuleCase): self.assertEqual(fstat.st_uid, os.getuid()) self.assertEqual(fstat.st_gid, grp.getgrnam(group).gr_gid) - @skipIf(salt.utils.is_windows(), 'No chgrp on Windows') + @skipIf(salt.utils.platform.is_windows(), 'No chgrp on Windows') def test_chown_no_user(self): user = 'notanyuseriknow' group = grp.getgrgid(pwd.getpwuid(os.getuid()).pw_gid).gr_name ret = self.run_function('file.chown', arg=[self.myfile, user, group]) self.assertIn('not exist', ret) - @skipIf(salt.utils.is_windows(), 'No chgrp on Windows') + @skipIf(salt.utils.platform.is_windows(), 'No chgrp on Windows') def test_chown_no_user_no_group(self): user = 'notanyuseriknow' group = 'notanygroupyoushoulduse' @@ -78,7 +79,7 @@ class FileModuleTest(ModuleCase): self.assertIn('Group does not exist', ret) self.assertIn('User does not exist', ret) - @skipIf(salt.utils.is_windows(), 'No chgrp on Windows') + @skipIf(salt.utils.platform.is_windows(), 'No chgrp on Windows') def test_chown_no_path(self): user = getpass.getuser() if sys.platform == 'darwin': @@ -89,7 +90,7 @@ class FileModuleTest(ModuleCase): arg=['/tmp/nosuchfile', user, group]) self.assertIn('File not found', ret) - @skipIf(salt.utils.is_windows(), 'No chgrp on Windows') + @skipIf(salt.utils.platform.is_windows(), 'No chgrp on Windows') def test_chown_noop(self): user = '' group = '' @@ -99,7 +100,7 @@ class FileModuleTest(ModuleCase): self.assertEqual(fstat.st_uid, os.getuid()) self.assertEqual(fstat.st_gid, os.getgid()) - @skipIf(salt.utils.is_windows(), 'No chgrp on Windows') + @skipIf(salt.utils.platform.is_windows(), 'No chgrp on Windows') def test_chgrp(self): if sys.platform == 'darwin': group = 'everyone' @@ -110,7 +111,7 @@ class FileModuleTest(ModuleCase): fstat = os.stat(self.myfile) self.assertEqual(fstat.st_gid, grp.getgrnam(group).gr_gid) - @skipIf(salt.utils.is_windows(), 'No chgrp on Windows') + @skipIf(salt.utils.platform.is_windows(), 'No chgrp on Windows') def test_chgrp_failure(self): group = 'thisgroupdoesntexist' ret = self.run_function('file.chgrp', arg=[self.myfile, group]) @@ -123,18 +124,18 @@ class FileModuleTest(ModuleCase): src_patch = os.path.join( FILES, 'file', 'base', 'hello.patch') src_file = os.path.join(TMP, 'src.txt') - with salt.utils.fopen(src_file, 'w+') as fp: + with salt.utils.files.fopen(src_file, 'w+') as fp: fp.write('Hello\n') # dry-run should not modify src_file ret = self.minion_run('file.patch', src_file, src_patch, dry_run=True) assert ret['retcode'] == 0, repr(ret) - with salt.utils.fopen(src_file) as fp: + with salt.utils.files.fopen(src_file) as fp: self.assertEqual(fp.read(), 'Hello\n') ret = self.minion_run('file.patch', src_file, src_patch) assert ret['retcode'] == 0, repr(ret) - with salt.utils.fopen(src_file) as fp: + with salt.utils.files.fopen(src_file) as fp: self.assertEqual(fp.read(), 'Hello world\n') def test_remove_file(self): diff --git a/tests/integration/modules/test_gem.py b/tests/integration/modules/test_gem.py index da902c9f661..b0ea463b320 100644 --- a/tests/integration/modules/test_gem.py +++ b/tests/integration/modules/test_gem.py @@ -12,15 +12,16 @@ from tests.support.unit import skipIf from tests.support.helpers import destructiveTest # Import salt libs -import salt.utils +import salt.utils.path # Import 3rd-party libs from tornado.httpclient import HTTPClient GEM = 'tidy' GEM_VER = '1.1.2' -OLD_GEM = 'thor' -OLD_VERSION = '0.17.0' +OLD_GEM = 'brass' +OLD_VERSION = '1.0.0' +NEW_VERSION = '1.2.1' GEM_LIST = [GEM, OLD_GEM] @@ -35,7 +36,7 @@ def check_status(): @destructiveTest -@skipIf(not salt.utils.which('gem'), 'Gem is not available') +@skipIf(not salt.utils.path.which('gem'), 'Gem is not available') class GemModuleTest(ModuleCase): ''' Validate gem module @@ -129,18 +130,18 @@ class GemModuleTest(ModuleCase): self.run_function('gem.install', [OLD_GEM], version=OLD_VERSION) gem_list = self.run_function('gem.list', [OLD_GEM]) - self.assertEqual({'thor': ['0.17.0']}, gem_list) + self.assertEqual({OLD_GEM: [OLD_VERSION]}, gem_list) self.run_function('gem.update', [OLD_GEM]) gem_list = self.run_function('gem.list', [OLD_GEM]) - self.assertEqual({'thor': ['0.19.4', '0.17.0']}, gem_list) + self.assertEqual({OLD_GEM: [NEW_VERSION, OLD_VERSION]}, gem_list) self.run_function('gem.uninstall', [OLD_GEM]) self.assertFalse(self.run_function('gem.list', [OLD_GEM])) - def test_udpate_system(self): + def test_update_system(self): ''' - gem.udpate_system + gem.update_system ''' ret = self.run_function('gem.update_system') self.assertTrue(ret) diff --git a/tests/integration/modules/test_git.py b/tests/integration/modules/test_git.py index 2720f4dcebf..6fd15242189 100644 --- a/tests/integration/modules/test_git.py +++ b/tests/integration/modules/test_git.py @@ -26,11 +26,12 @@ from tests.support.paths import TMP from tests.support.helpers import skip_if_binaries_missing # Import salt libs -import salt.utils +import salt.utils.files +import salt.utils.platform from salt.utils.versions import LooseVersion # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six log = logging.getLogger(__name__) @@ -40,7 +41,7 @@ def _git_version(): git_version = subprocess.Popen( ['git', '--version'], shell=False, - close_fds=False if salt.utils.is_windows else True, + close_fds=False if salt.utils.platform.is_windows() else True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0] except OSError: @@ -91,7 +92,7 @@ class GitModuleTest(ModuleCase): dir_path = os.path.join(self.repo, dirname) _makedirs(dir_path) for filename in self.files: - with salt.utils.fopen(os.path.join(dir_path, filename), 'w') as fp_: + with salt.utils.files.fopen(os.path.join(dir_path, filename), 'w') as fp_: fp_.write('This is a test file named ' + filename + '.') # Navigate to the root of the repo to init, stage, and commit os.chdir(self.repo) @@ -124,7 +125,7 @@ class GitModuleTest(ModuleCase): ['git', 'checkout', '--quiet', '-b', self.branches[1]] ) # Add a line to the file - with salt.utils.fopen(self.files[0], 'a') as fp_: + with salt.utils.files.fopen(self.files[0], 'a') as fp_: fp_.write('Added a line\n') # Commit the updated file subprocess.check_call( @@ -152,14 +153,14 @@ class GitModuleTest(ModuleCase): files = [os.path.join(newdir_path, x) for x in self.files] files_relpath = [os.path.join(newdir, x) for x in self.files] for path in files: - with salt.utils.fopen(path, 'w') as fp_: + with salt.utils.files.fopen(path, 'w') as fp_: fp_.write( 'This is a test file with relative path {0}.\n' .format(path) ) ret = self.run_function('git.add', [self.repo, newdir]) res = '\n'.join(sorted(['add \'{0}\''.format(x) for x in files_relpath])) - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): res = res.replace('\\', '/') self.assertEqual(ret, res) @@ -169,7 +170,7 @@ class GitModuleTest(ModuleCase): ''' filename = 'quux' file_path = os.path.join(self.repo, filename) - with salt.utils.fopen(file_path, 'w') as fp_: + with salt.utils.files.fopen(file_path, 'w') as fp_: fp_.write('This is a test file named ' + filename + '.\n') ret = self.run_function('git.add', [self.repo, filename]) self.assertEqual(ret, 'add \'{0}\''.format(filename)) @@ -310,7 +311,7 @@ class GitModuleTest(ModuleCase): filename = 'foo' commit_re_prefix = r'^\[master [0-9a-f]+\] ' # Add a line - with salt.utils.fopen(os.path.join(self.repo, filename), 'a') as fp_: + with salt.utils.files.fopen(os.path.join(self.repo, filename), 'a') as fp_: fp_.write('Added a line\n') # Stage the file self.run_function('git.add', [self.repo, filename]) @@ -320,7 +321,7 @@ class GitModuleTest(ModuleCase): # Make sure the expected line is in the output self.assertTrue(bool(re.search(commit_re_prefix + commit_msg, ret))) # Add another line - with salt.utils.fopen(os.path.join(self.repo, filename), 'a') as fp_: + with salt.utils.files.fopen(os.path.join(self.repo, filename), 'a') as fp_: fp_.write('Added another line\n') # Commit the second file without staging commit_msg = 'Add another line to ' + filename @@ -345,7 +346,7 @@ class GitModuleTest(ModuleCase): ['git', 'config', '--global', '--remove-section', 'foo'] ) for cmd in cmds: - with salt.utils.fopen(os.devnull, 'w') as devnull: + with salt.utils.files.fopen(os.devnull, 'w') as devnull: try: subprocess.check_call(cmd, stderr=devnull) except subprocess.CalledProcessError: @@ -597,7 +598,7 @@ class GitModuleTest(ModuleCase): # the full, unshortened name of the folder. Therefore you can't compare # the path returned by `tempfile.mkdtemp` and the results of `git.init` # exactly. - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): new_repo = new_repo.replace('\\', '/') # Get the name of the temp directory @@ -704,7 +705,7 @@ class GitModuleTest(ModuleCase): ''' # Make a change to a different file than the one modifed in setUp file_path = os.path.join(self.repo, self.files[1]) - with salt.utils.fopen(file_path, 'a') as fp_: + with salt.utils.files.fopen(file_path, 'a') as fp_: fp_.write('Added a line\n') # Commit the change self.assertTrue( @@ -834,7 +835,7 @@ class GitModuleTest(ModuleCase): sorted(['rm \'' + os.path.join(entire_dir, x) + '\'' for x in self.files]) ) - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): expected = expected.replace('\\', '/') self.assertEqual( self.run_function('git.rm', [self.repo, entire_dir], opts='-r'), @@ -848,7 +849,7 @@ class GitModuleTest(ModuleCase): # TODO: test more stash actions ''' file_path = os.path.join(self.repo, self.files[0]) - with salt.utils.fopen(file_path, 'a') as fp_: + with salt.utils.files.fopen(file_path, 'a') as fp_: fp_.write('Temp change to be stashed') self.assertTrue( 'ERROR' not in self.run_function('git.stash', [self.repo]) @@ -887,10 +888,10 @@ class GitModuleTest(ModuleCase): 'untracked': ['thisisalsoanewfile'] } for filename in changes['modified']: - with salt.utils.fopen(os.path.join(self.repo, filename), 'a') as fp_: + with salt.utils.files.fopen(os.path.join(self.repo, filename), 'a') as fp_: fp_.write('Added a line\n') for filename in changes['new']: - with salt.utils.fopen(os.path.join(self.repo, filename), 'w') as fp_: + with salt.utils.files.fopen(os.path.join(self.repo, filename), 'w') as fp_: fp_.write('This is a new file named ' + filename + '.') # Stage the new file so it shows up as a 'new' file self.assertTrue( @@ -902,7 +903,7 @@ class GitModuleTest(ModuleCase): for filename in changes['deleted']: self.run_function('git.rm', [self.repo, filename]) for filename in changes['untracked']: - with salt.utils.fopen(os.path.join(self.repo, filename), 'w') as fp_: + with salt.utils.files.fopen(os.path.join(self.repo, filename), 'w') as fp_: fp_.write('This is a new file named ' + filename + '.') self.assertEqual( self.run_function('git.status', [self.repo]), @@ -945,7 +946,7 @@ class GitModuleTest(ModuleCase): worktree_path2 = tempfile.mkdtemp(dir=TMP) # Even though this is Windows, git commands return a unix style path - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): worktree_path = worktree_path.replace('\\', '/') worktree_path2 = worktree_path2.replace('\\', '/') diff --git a/tests/integration/modules/test_grains.py b/tests/integration/modules/test_grains.py index 81858f30533..8908026e077 100644 --- a/tests/integration/modules/test_grains.py +++ b/tests/integration/modules/test_grains.py @@ -5,6 +5,7 @@ Test the grains module # Import python libs from __future__ import absolute_import +import logging import os import time @@ -13,6 +14,8 @@ from tests.support.case import ModuleCase from tests.support.unit import skipIf from tests.support.helpers import destructiveTest +log = logging.getLogger(__name__) + class TestModulesGrains(ModuleCase): ''' @@ -110,11 +113,12 @@ class TestModulesGrains(ModuleCase): ''' test to ensure some core grains are returned ''' - grains = ['os', 'os_family', 'osmajorrelease', 'osrelease', 'osfullname', 'id'] + grains = ('os', 'os_family', 'osmajorrelease', 'osrelease', 'osfullname', 'id') os = self.run_function('grains.get', ['os']) for grain in grains: get_grain = self.run_function('grains.get', [grain]) + log.debug('Value of \'%s\' grain: \'%s\'', grain, get_grain) if os == 'Arch' and grain in ['osmajorrelease']: self.assertEqual(get_grain, '') continue diff --git a/tests/integration/modules/test_groupadd.py b/tests/integration/modules/test_groupadd.py index ed75916ba34..9963793ca1a 100644 --- a/tests/integration/modules/test_groupadd.py +++ b/tests/integration/modules/test_groupadd.py @@ -11,6 +11,9 @@ from tests.support.helpers import destructiveTest, skip_if_not_root # Import 3rd-party libs from salt.ext.six.moves import range +import os +import grp +from salt import utils @skip_if_not_root @@ -57,6 +60,43 @@ class GroupModuleTest(ModuleCase): for x in range(size) ) + def __get_system_group_gid_range(self): + ''' + Returns (SYS_GID_MIN, SYS_GID_MAX) + ''' + defs_file = '/etc/login.defs' + if os.path.exists(defs_file): + with utils.fopen(defs_file) as defs_fd: + login_defs = dict([x.split() + for x in defs_fd.readlines() + if x.strip() + and not x.strip().startswith('#')]) + else: + login_defs = {'SYS_GID_MIN': 101, + 'SYS_GID_MAX': 999} + + gid_min = login_defs.get('SYS_GID_MIN', 101) + gid_max = login_defs.get('SYS_GID_MAX', + int(login_defs.get('GID_MIN', 1000)) - 1) + + return gid_min, gid_max + + def __get_free_system_gid(self): + ''' + Find a free system gid + ''' + + gid_min, gid_max = self.__get_system_group_gid_range() + + busy_gids = [x.gr_gid + for x in grp.getgrall() + if gid_min <= x.gr_gid <= gid_max] + + # find free system gid + for gid in range(gid_min, gid_max + 1): + if gid not in busy_gids: + return gid + @destructiveTest def test_add(self): ''' @@ -70,6 +110,42 @@ class GroupModuleTest(ModuleCase): #try adding the group again self.assertFalse(self.run_function('group.add', [self._group, self._gid])) + @destructiveTest + def test_add_system_group(self): + ''' + Test the add group function with system=True + ''' + + gid_min, gid_max = self.__get_system_group_gid_range() + + # add a new system group + self.assertTrue(self.run_function('group.add', + [self._group, None, True])) + group_info = self.run_function('group.info', [self._group]) + self.assertEqual(group_info['name'], self._group) + self.assertTrue(gid_min <= group_info['gid'] <= gid_max) + #try adding the group again + self.assertFalse(self.run_function('group.add', + [self._group])) + + @destructiveTest + def test_add_system_group_gid(self): + ''' + Test the add group function with system=True and a specific gid + ''' + + gid = self.__get_free_system_gid() + + # add a new system group + self.assertTrue(self.run_function('group.add', + [self._group, gid, True])) + group_info = self.run_function('group.info', [self._group]) + self.assertEqual(group_info['name'], self._group) + self.assertEqual(group_info['gid'], gid) + #try adding the group again + self.assertFalse(self.run_function('group.add', + [self._group, gid])) + @destructiveTest def test_delete(self): ''' diff --git a/tests/integration/modules/test_hosts.py b/tests/integration/modules/test_hosts.py index e1469cac366..55b4bddf0a7 100644 --- a/tests/integration/modules/test_hosts.py +++ b/tests/integration/modules/test_hosts.py @@ -12,7 +12,7 @@ from tests.support.case import ModuleCase from tests.support.paths import FILES, TMP # Import salt libs -import salt.utils +import salt.utils.files HFN = os.path.join(TMP, 'hosts') @@ -167,7 +167,7 @@ class HostsModuleTest(ModuleCase): # use an empty one so we can prove the syntax of the entries # being added by the hosts module self.__clear_hosts() - with salt.utils.fopen(HFN, 'w'): + with salt.utils.files.fopen(HFN, 'w'): pass self.assertTrue( @@ -206,7 +206,7 @@ class HostsModuleTest(ModuleCase): ) # now read the lines and ensure they're formatted correctly - with salt.utils.fopen(HFN, 'r') as fp_: + with salt.utils.files.fopen(HFN, 'r') as fp_: lines = fp_.read().splitlines() self.assertEqual(lines, [ '192.168.1.3\t\thost3.fqdn.com', diff --git a/tests/integration/modules/test_linux_acl.py b/tests/integration/modules/test_linux_acl.py index d8908a93b0a..8f4e448ed20 100644 --- a/tests/integration/modules/test_linux_acl.py +++ b/tests/integration/modules/test_linux_acl.py @@ -13,6 +13,7 @@ from tests.support.helpers import skip_if_binaries_missing # Import salt libs import salt.utils +import salt.utils.files # from salt.modules import linux_acl as acl @@ -28,7 +29,7 @@ class LinuxAclModuleTest(ModuleCase, AdaptedConfigurationTestCaseMixin): def setUp(self): # Blindly copied from tests.integration.modules.file; Refactoring? self.myfile = os.path.join(TMP, 'myfile') - with salt.utils.fopen(self.myfile, 'w+') as fp: + with salt.utils.files.fopen(self.myfile, 'w+') as fp: fp.write('Hello\n') self.mydir = os.path.join(TMP, 'mydir/isawesome') if not os.path.isdir(self.mydir): diff --git a/tests/integration/modules/test_locale.py b/tests/integration/modules/test_locale.py index b202d48aa7c..5059c6252ab 100644 --- a/tests/integration/modules/test_locale.py +++ b/tests/integration/modules/test_locale.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Import python libs +# Import Python libs from __future__ import absolute_import # Import Salt Testing libs @@ -11,8 +11,8 @@ from tests.support.helpers import ( destructiveTest, ) -# Import salt libs -import salt.utils +# Import Salt libs +import salt.utils.platform def _find_new_locale(current_locale): @@ -21,8 +21,8 @@ def _find_new_locale(current_locale): return locale -@skipIf(salt.utils.is_windows(), 'minion is windows') -@skipIf(salt.utils.is_darwin(), 'locale method is not supported on mac') +@skipIf(salt.utils.platform.is_windows(), 'minion is windows') +@skipIf(salt.utils.platform.is_darwin(), 'locale method is not supported on mac') @requires_salt_modules('locale') class LocaleModuleTest(ModuleCase): def test_get_locale(self): diff --git a/tests/integration/modules/test_lxc.py b/tests/integration/modules/test_lxc.py index 75b6b811701..c334f2dadd4 100644 --- a/tests/integration/modules/test_lxc.py +++ b/tests/integration/modules/test_lxc.py @@ -16,7 +16,7 @@ from tests.support.helpers import ( from tests.support.unit import skipIf # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six @skipIf(True, diff --git a/tests/integration/modules/test_mac_brew.py b/tests/integration/modules/test_mac_brew.py index 33001a84a86..942bfa46ada 100644 --- a/tests/integration/modules/test_mac_brew.py +++ b/tests/integration/modules/test_mac_brew.py @@ -12,11 +12,12 @@ from tests.support.unit import skipIf from tests.support.helpers import destructiveTest, skip_if_not_root # Import Salt Libs -import salt.utils +import salt.utils.path +import salt.utils.platform from salt.exceptions import CommandExecutionError -# Import third party libs -import salt.ext.six as six +# Import 3rd-party libs +from salt.ext import six # Brew doesn't support local package installation - So, let's # Grab some small packages available online for brew @@ -26,8 +27,8 @@ DEL_PKG = 'acme' @destructiveTest @skip_if_not_root -@skipIf(not salt.utils.is_darwin(), 'Test only applies to macOS') -@skipIf(not salt.utils.which('brew'), 'This test requires the brew binary') +@skipIf(not salt.utils.platform.is_darwin(), 'Test only applies to macOS') +@skipIf(not salt.utils.path.which('brew'), 'This test requires the brew binary') class BrewModuleTest(ModuleCase): ''' Integration tests for the brew module diff --git a/tests/integration/modules/test_mac_pkgutil.py b/tests/integration/modules/test_mac_pkgutil.py index 6fc1101f8ce..91c4ec371c9 100644 --- a/tests/integration/modules/test_mac_pkgutil.py +++ b/tests/integration/modules/test_mac_pkgutil.py @@ -3,7 +3,7 @@ integration tests for mac_pkgutil ''' -# Import python libs +# Import Python libs from __future__ import absolute_import import os @@ -12,8 +12,9 @@ from tests.support.case import ModuleCase from tests.support.paths import TMP from tests.support.helpers import destructiveTest, skip_if_not_root -# Import salt libs -import salt.utils +# Import Salt libs +import salt.utils.path +import salt.utils.platform TEST_PKG_URL = 'https://distfiles.macports.org/MacPorts/MacPorts-2.3.4-10.11-ElCapitan.pkg' TEST_PKG_NAME = 'org.macports.MacPorts' @@ -30,10 +31,10 @@ class MacPkgutilModuleTest(ModuleCase): ''' Get current settings ''' - if not salt.utils.is_darwin(): + if not salt.utils.platform.is_darwin(): self.skipTest('Test only available on macOS') - if not salt.utils.which('pkgutil'): + if not salt.utils.path.which('pkgutil'): self.skipTest('Test requires pkgutil binary') def tearDown(self): diff --git a/tests/integration/modules/test_mac_ports.py b/tests/integration/modules/test_mac_ports.py index ee9de7c984a..362fd17642f 100644 --- a/tests/integration/modules/test_mac_ports.py +++ b/tests/integration/modules/test_mac_ports.py @@ -3,15 +3,16 @@ integration tests for mac_ports ''' -# Import python libs +# Import Python libs from __future__ import absolute_import, print_function # Import Salt Testing libs from tests.support.case import ModuleCase from tests.support.helpers import destructiveTest, skip_if_not_root -# Import salt libs -import salt.utils +# Import Salt libs +import salt.utils.path +import salt.utils.platform @skip_if_not_root @@ -25,10 +26,10 @@ class MacPortsModuleTest(ModuleCase): ''' Get current settings ''' - if not salt.utils.is_darwin(): + if not salt.utils.platform.is_darwin(): self.skipTest('Test only available on macOS') - if not salt.utils.which('port'): + if not salt.utils.path.which('port'): self.skipTest('Test requires port binary') self.AGREE_INSTALLED = 'agree' in self.run_function('pkg.list_pkgs') diff --git a/tests/integration/modules/test_mac_power.py b/tests/integration/modules/test_mac_power.py index aea67bd41f4..cda74d83adf 100644 --- a/tests/integration/modules/test_mac_power.py +++ b/tests/integration/modules/test_mac_power.py @@ -3,7 +3,7 @@ integration tests for mac_power ''' -# Import python libs +# Import Python libs from __future__ import absolute_import, print_function # Import Salt Testing libs @@ -11,14 +11,15 @@ from tests.support.case import ModuleCase from tests.support.unit import skipIf from tests.support.helpers import destructiveTest, skip_if_not_root, flaky -# Import salt libs -import salt.utils +# Import Salt libs +import salt.utils.path +import salt.utils.platform @skip_if_not_root @flaky -@skipIf(not salt.utils.is_darwin(), 'Test only available on macOS') -@skipIf(not salt.utils.which('systemsetup'), '\'systemsetup\' binary not found in $PATH') +@skipIf(not salt.utils.platform.is_darwin(), 'Test only available on macOS') +@skipIf(not salt.utils.path.which('systemsetup'), '\'systemsetup\' binary not found in $PATH') class MacPowerModuleTest(ModuleCase): ''' Validate the mac_power module @@ -142,8 +143,8 @@ class MacPowerModuleTest(ModuleCase): @skip_if_not_root @flaky -@skipIf(not salt.utils.is_darwin(), 'Test only available on macOS') -@skipIf(not salt.utils.which('systemsetup'), '\'systemsetup\' binary not found in $PATH') +@skipIf(not salt.utils.platform.is_darwin(), 'Test only available on macOS') +@skipIf(not salt.utils.path.which('systemsetup'), '\'systemsetup\' binary not found in $PATH') class MacPowerModuleTestSleepOnPowerButton(ModuleCase): ''' Test power.get_sleep_on_power_button @@ -193,8 +194,8 @@ class MacPowerModuleTestSleepOnPowerButton(ModuleCase): @skip_if_not_root @flaky -@skipIf(not salt.utils.is_darwin(), 'Test only available on macOS') -@skipIf(not salt.utils.which('systemsetup'), '\'systemsetup\' binary not found in $PATH') +@skipIf(not salt.utils.platform.is_darwin(), 'Test only available on macOS') +@skipIf(not salt.utils.path.which('systemsetup'), '\'systemsetup\' binary not found in $PATH') class MacPowerModuleTestRestartPowerFailure(ModuleCase): ''' Test power.get_restart_power_failure @@ -243,8 +244,8 @@ class MacPowerModuleTestRestartPowerFailure(ModuleCase): @skip_if_not_root @flaky -@skipIf(not salt.utils.is_darwin(), 'Test only available on macOS') -@skipIf(not salt.utils.which('systemsetup'), '\'systemsetup\' binary not found in $PATH') +@skipIf(not salt.utils.platform.is_darwin(), 'Test only available on macOS') +@skipIf(not salt.utils.path.which('systemsetup'), '\'systemsetup\' binary not found in $PATH') class MacPowerModuleTestWakeOnNet(ModuleCase): ''' Test power.get_wake_on_network @@ -290,8 +291,8 @@ class MacPowerModuleTestWakeOnNet(ModuleCase): @skip_if_not_root @flaky -@skipIf(not salt.utils.is_darwin(), 'Test only available on macOS') -@skipIf(not salt.utils.which('systemsetup'), '\'systemsetup\' binary not found in $PATH') +@skipIf(not salt.utils.platform.is_darwin(), 'Test only available on macOS') +@skipIf(not salt.utils.path.which('systemsetup'), '\'systemsetup\' binary not found in $PATH') class MacPowerModuleTestWakeOnModem(ModuleCase): ''' Test power.get_wake_on_modem diff --git a/tests/integration/modules/test_mac_service.py b/tests/integration/modules/test_mac_service.py index da6396bf47c..8b21700297d 100644 --- a/tests/integration/modules/test_mac_service.py +++ b/tests/integration/modules/test_mac_service.py @@ -3,7 +3,7 @@ integration tests for mac_service ''' -# Import python libs +# Import Python libs from __future__ import absolute_import, print_function # Import Salt Testing libs @@ -11,13 +11,14 @@ from tests.support.case import ModuleCase from tests.support.unit import skipIf from tests.support.helpers import destructiveTest, skip_if_not_root -# Import salt libs -import salt.utils +# Import Salt libs +import salt.utils.path +import salt.utils.platform -@skipIf(not salt.utils.is_darwin(), 'Test only available on macOS') -@skipIf(not salt.utils.which('launchctl'), 'Test requires launchctl binary') -@skipIf(not salt.utils.which('plutil'), 'Test requires plutil binary') +@skipIf(not salt.utils.platform.is_darwin(), 'Test only available on macOS') +@skipIf(not salt.utils.path.which('launchctl'), 'Test requires launchctl binary') +@skipIf(not salt.utils.path.which('plutil'), 'Test requires plutil binary') @skip_if_not_root class MacServiceModuleTest(ModuleCase): ''' diff --git a/tests/integration/modules/test_mac_shadow.py b/tests/integration/modules/test_mac_shadow.py index b6bb66c9568..13ed8728c82 100644 --- a/tests/integration/modules/test_mac_shadow.py +++ b/tests/integration/modules/test_mac_shadow.py @@ -3,7 +3,7 @@ integration tests for mac_system ''' -# Import python libs +# Import Python libs from __future__ import absolute_import import datetime import random @@ -14,8 +14,9 @@ from tests.support.unit import skipIf from tests.support.case import ModuleCase from tests.support.helpers import destructiveTest, skip_if_not_root -# Import salt libs -import salt.utils +# Import Salt libs +import salt.utils.path +import salt.utils.platform from salt.ext.six.moves import range @@ -34,9 +35,9 @@ NO_USER = __random_string() @skip_if_not_root -@skipIf(not salt.utils.is_darwin(), 'Test only available on macOS') -@skipIf(not salt.utils.which('dscl'), '\'dscl\' binary not found in $PATH') -@skipIf(not salt.utils.which('pwpolicy'), '\'pwpolicy\' binary not found in $PATH') +@skipIf(not salt.utils.platform.is_darwin(), 'Test only available on macOS') +@skipIf(not salt.utils.path.which('dscl'), '\'dscl\' binary not found in $PATH') +@skipIf(not salt.utils.path.which('pwpolicy'), '\'pwpolicy\' binary not found in $PATH') class MacShadowModuleTest(ModuleCase): ''' Validate the mac_system module diff --git a/tests/integration/modules/test_mac_softwareupdate.py b/tests/integration/modules/test_mac_softwareupdate.py index f74ece4b22a..4a00067e120 100644 --- a/tests/integration/modules/test_mac_softwareupdate.py +++ b/tests/integration/modules/test_mac_softwareupdate.py @@ -3,7 +3,7 @@ integration tests for mac_softwareupdate ''' -# Import python libs +# Import Python libs from __future__ import absolute_import # Import Salt Testing libs @@ -11,13 +11,14 @@ from tests.support.unit import skipIf from tests.support.case import ModuleCase from tests.support.helpers import destructiveTest, skip_if_not_root -# Import salt libs -import salt.utils +# Import Salt libs +import salt.utils.path +import salt.utils.platform @skip_if_not_root -@skipIf(not salt.utils.is_darwin(), 'Test only available on macOS') -@skipIf(not salt.utils.which('softwareupdate'), '\'softwareupdate\' binary not found in $PATH') +@skipIf(not salt.utils.platform.is_darwin(), 'Test only available on macOS') +@skipIf(not salt.utils.path.which('softwareupdate'), '\'softwareupdate\' binary not found in $PATH') class MacSoftwareUpdateModuleTest(ModuleCase): ''' Validate the mac_softwareupdate module diff --git a/tests/integration/modules/test_mac_system.py b/tests/integration/modules/test_mac_system.py index 308187c08a5..ab0b0f13980 100644 --- a/tests/integration/modules/test_mac_system.py +++ b/tests/integration/modules/test_mac_system.py @@ -14,7 +14,8 @@ from tests.support.unit import skipIf from tests.support.helpers import destructiveTest, skip_if_not_root # Import salt libs -import salt.utils +import salt.utils.path +import salt.utils.platform from salt.ext.six.moves import range @@ -33,8 +34,8 @@ SET_SUBNET_NAME = __random_string() @skip_if_not_root -@skipIf(not salt.utils.is_darwin(), 'Test only available on macOS') -@skipIf(not salt.utils.which('systemsetup'), '\'systemsetup\' binary not found in $PATH') +@skipIf(not salt.utils.platform.is_darwin(), 'Test only available on macOS') +@skipIf(not salt.utils.path.which('systemsetup'), '\'systemsetup\' binary not found in $PATH') class MacSystemModuleTest(ModuleCase): ''' Validate the mac_system module diff --git a/tests/integration/modules/test_mac_timezone.py b/tests/integration/modules/test_mac_timezone.py index 18e5e5f9dec..bdfe0d0549a 100644 --- a/tests/integration/modules/test_mac_timezone.py +++ b/tests/integration/modules/test_mac_timezone.py @@ -10,7 +10,7 @@ Time sync do the following: - Set time to 'Do not sync' ''' -# Import python libs +# Import Python libs from __future__ import absolute_import import datetime @@ -19,13 +19,14 @@ from tests.support.case import ModuleCase from tests.support.unit import skipIf from tests.support.helpers import destructiveTest, skip_if_not_root -# Import salt libs -import salt.utils +# Import Salt libs +import salt.utils.path +import salt.utils.platform @skip_if_not_root -@skipIf(not salt.utils.is_darwin(), 'Test only available on macOS') -@skipIf(not salt.utils.which('systemsetup'), '\'systemsetup\' binary not found in $PATH') +@skipIf(not salt.utils.platform.is_darwin(), 'Test only available on macOS') +@skipIf(not salt.utils.path.which('systemsetup'), '\'systemsetup\' binary not found in $PATH') class MacTimezoneModuleTest(ModuleCase): ''' Validate the mac_timezone module diff --git a/tests/integration/modules/test_mac_xattr.py b/tests/integration/modules/test_mac_xattr.py index ff5165b661f..52c9016970a 100644 --- a/tests/integration/modules/test_mac_xattr.py +++ b/tests/integration/modules/test_mac_xattr.py @@ -3,7 +3,7 @@ integration tests for mac_xattr ''' -# Import python libs +# Import Python libs from __future__ import absolute_import import os @@ -11,8 +11,9 @@ import os from tests.support.case import ModuleCase from tests.support.paths import TMP -# Import salt libs -import salt.utils +# Import Salt libs +import salt.utils.path +import salt.utils.platform TEST_FILE = os.path.join(TMP, 'xattr_test_file.txt') NO_FILE = os.path.join(TMP, 'xattr_no_file.txt') @@ -27,10 +28,10 @@ class MacXattrModuleTest(ModuleCase): ''' Create test file for testing extended attributes ''' - if not salt.utils.is_darwin(): + if not salt.utils.platform.is_darwin(): self.skipTest('Test only available on macOS') - if not salt.utils.which('xattr'): + if not salt.utils.path.which('xattr'): self.skipTest('Test requires xattr binary') self.run_function('file.touch', [TEST_FILE]) diff --git a/tests/integration/modules/test_mysql.py b/tests/integration/modules/test_mysql.py index 20b79da9083..c6c82b7165c 100644 --- a/tests/integration/modules/test_mysql.py +++ b/tests/integration/modules/test_mysql.py @@ -11,11 +11,11 @@ from tests.support.helpers import destructiveTest from tests.support.mixins import SaltReturnAssertsMixin # Import salt libs -import salt.utils +import salt.utils.path from salt.modules import mysql as mysqlmod # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import range # pylint: disable=import-error,redefined-builtin log = logging.getLogger(__name__) @@ -26,7 +26,7 @@ try: except Exception: NO_MYSQL = True -if not salt.utils.which('mysqladmin'): +if not salt.utils.path.which('mysqladmin'): NO_MYSQL = True diff --git a/tests/integration/modules/test_nilrt_ip.py b/tests/integration/modules/test_nilrt_ip.py index c199a0a8f8c..1412cffb2d2 100644 --- a/tests/integration/modules/test_nilrt_ip.py +++ b/tests/integration/modules/test_nilrt_ip.py @@ -3,7 +3,7 @@ integration tests for nilirt_ip ''' -# Import python libs +# Import Python libs from __future__ import absolute_import import time @@ -12,12 +12,12 @@ from tests.support.case import ModuleCase from tests.support.unit import skipIf from tests.support.helpers import destructiveTest, skip_if_not_root -# Import salt libs -import salt.utils +# Import Salt libs +import salt.utils.platform @skip_if_not_root -@skipIf(not salt.utils.is_linux(), 'These tests can only be run on linux') +@skipIf(not salt.utils.platform.is_linux(), 'These tests can only be run on linux') class Nilrt_ipModuleTest(ModuleCase): ''' Validate the nilrt_ip module diff --git a/tests/integration/modules/test_pip.py b/tests/integration/modules/test_pip.py index 0937434cba6..dfebb969250 100644 --- a/tests/integration/modules/test_pip.py +++ b/tests/integration/modules/test_pip.py @@ -21,11 +21,12 @@ from tests.support.paths import TMP from tests.support.helpers import skip_if_not_root # Import salt libs -import salt.utils +import salt.utils.files +import salt.utils.path from salt.modules.virtualenv_mod import KNOWN_BINARY_NAMES -@skipIf(salt.utils.which_bin(KNOWN_BINARY_NAMES) is None, 'virtualenv not installed') +@skipIf(salt.utils.path.which_bin(KNOWN_BINARY_NAMES) is None, 'virtualenv not installed') class PipModuleTest(ModuleCase): def setUp(self): @@ -99,13 +100,13 @@ class PipModuleTest(ModuleCase): req2_filename = os.path.join(self.venv_dir, 'requirements2.txt') req2b_filename = os.path.join(self.venv_dir, 'requirements2b.txt') - with salt.utils.fopen(req1_filename, 'w') as f: + with salt.utils.files.fopen(req1_filename, 'w') as f: f.write('-r requirements1b.txt\n') - with salt.utils.fopen(req1b_filename, 'w') as f: + with salt.utils.files.fopen(req1b_filename, 'w') as f: f.write('flake8\n') - with salt.utils.fopen(req2_filename, 'w') as f: + with salt.utils.files.fopen(req2_filename, 'w') as f: f.write('-r requirements2b.txt\n') - with salt.utils.fopen(req2b_filename, 'w') as f: + with salt.utils.files.fopen(req2b_filename, 'w') as f: f.write('pep8\n') this_user = pwd.getpwuid(os.getuid())[0] @@ -137,13 +138,13 @@ class PipModuleTest(ModuleCase): req2_filename = os.path.join(self.venv_dir, 'requirements2.txt') req2b_filename = os.path.join(self.venv_dir, 'requirements2b.txt') - with salt.utils.fopen(req1_filename, 'w') as f: + with salt.utils.files.fopen(req1_filename, 'w') as f: f.write('-r requirements1b.txt\n') - with salt.utils.fopen(req1b_filename, 'w') as f: + with salt.utils.files.fopen(req1b_filename, 'w') as f: f.write('flake8\n') - with salt.utils.fopen(req2_filename, 'w') as f: + with salt.utils.files.fopen(req2_filename, 'w') as f: f.write('-r requirements2b.txt\n') - with salt.utils.fopen(req2b_filename, 'w') as f: + with salt.utils.files.fopen(req2b_filename, 'w') as f: f.write('pep8\n') this_user = pwd.getpwuid(os.getuid())[0] @@ -172,9 +173,9 @@ class PipModuleTest(ModuleCase): req1_filename = os.path.join(self.venv_dir, 'requirements.txt') req2_filename = os.path.join(self.venv_dir, 'requirements2.txt') - with salt.utils.fopen(req1_filename, 'w') as f: + with salt.utils.files.fopen(req1_filename, 'w') as f: f.write('flake8\n') - with salt.utils.fopen(req2_filename, 'w') as f: + with salt.utils.files.fopen(req2_filename, 'w') as f: f.write('pep8\n') this_user = pwd.getpwuid(os.getuid())[0] @@ -209,9 +210,9 @@ class PipModuleTest(ModuleCase): req1_filepath = os.path.join(req_cwd, req1_filename) req2_filepath = os.path.join(req_cwd, req2_filename) - with salt.utils.fopen(req1_filepath, 'w') as f: + with salt.utils.files.fopen(req1_filepath, 'w') as f: f.write('flake8\n') - with salt.utils.fopen(req2_filepath, 'w') as f: + with salt.utils.files.fopen(req2_filepath, 'w') as f: f.write('pep8\n') this_user = pwd.getpwuid(os.getuid())[0] @@ -241,9 +242,9 @@ class PipModuleTest(ModuleCase): req1_filename = os.path.join(self.venv_dir, 'requirements.txt') req2_filename = os.path.join(self.venv_dir, 'requirements2.txt') - with salt.utils.fopen(req1_filename, 'w') as f: + with salt.utils.files.fopen(req1_filename, 'w') as f: f.write('-r requirements2.txt') - with salt.utils.fopen(req2_filename, 'w') as f: + with salt.utils.files.fopen(req2_filename, 'w') as f: f.write('pep8') this_user = pwd.getpwuid(os.getuid())[0] @@ -272,9 +273,9 @@ class PipModuleTest(ModuleCase): req1_file = os.path.join(self.venv_dir, req1_filename) req2_file = os.path.join(self.venv_dir, req2_filename) - with salt.utils.fopen(req1_file, 'w') as f: + with salt.utils.files.fopen(req1_file, 'w') as f: f.write('-r requirements2.txt') - with salt.utils.fopen(req2_file, 'w') as f: + with salt.utils.files.fopen(req2_file, 'w') as f: f.write('pep8') this_user = pwd.getpwuid(os.getuid())[0] @@ -297,9 +298,9 @@ class PipModuleTest(ModuleCase): # Create a requirements file that depends on another one. req1_filename = os.path.join(self.venv_dir, 'requirements.txt') req2_filename = os.path.join(self.venv_dir, 'requirements2.txt') - with salt.utils.fopen(req1_filename, 'w') as f: + with salt.utils.files.fopen(req1_filename, 'w') as f: f.write('-r requirements2.txt') - with salt.utils.fopen(req2_filename, 'w') as f: + with salt.utils.files.fopen(req2_filename, 'w') as f: f.write('pep8') this_user = pwd.getpwuid(os.getuid())[0] diff --git a/tests/integration/modules/test_pkg.py b/tests/integration/modules/test_pkg.py index b438dda96ca..6c90dc9eb6a 100644 --- a/tests/integration/modules/test_pkg.py +++ b/tests/integration/modules/test_pkg.py @@ -14,9 +14,9 @@ from tests.support.helpers import ( ) from tests.support.unit import skipIf -# Import salt libs -import salt.utils +# Import Salt libs import salt.utils.pkg +import salt.utils.platform class PkgModuleTest(ModuleCase, SaltReturnAssertsMixin): @@ -24,7 +24,7 @@ class PkgModuleTest(ModuleCase, SaltReturnAssertsMixin): Validate the pkg module ''' def setUp(self): - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): self.run_function('pkg.refresh_db') def test_list(self): @@ -258,7 +258,7 @@ class PkgModuleTest(ModuleCase, SaltReturnAssertsMixin): keys = ret.keys() self.assertIn('rpm', keys) self.assertIn('yum', keys) - elif os_family == 'SUSE': + elif os_family == 'Suse': ret = self.run_function(func, ['less', 'zypper']) keys = ret.keys() self.assertIn('less', keys) @@ -266,7 +266,7 @@ class PkgModuleTest(ModuleCase, SaltReturnAssertsMixin): @requires_network() @destructiveTest - @skipIf(salt.utils.is_windows(), 'pkg.upgrade not available on Windows') + @skipIf(salt.utils.platform.is_windows(), 'pkg.upgrade not available on Windows') def test_pkg_upgrade_has_pending_upgrades(self): ''' Test running a system upgrade when there are packages that need upgrading diff --git a/tests/integration/modules/test_shadow.py b/tests/integration/modules/test_shadow.py index 28571f19f3b..5d8393f3a2c 100644 --- a/tests/integration/modules/test_shadow.py +++ b/tests/integration/modules/test_shadow.py @@ -3,7 +3,7 @@ integration tests for shadow linux ''' -# Import python libs +# Import Python libs from __future__ import absolute_import import random import string @@ -14,13 +14,14 @@ from tests.support.case import ModuleCase from tests.support.unit import skipIf from tests.support.helpers import destructiveTest, skip_if_not_root -# Import salt libs -import salt.utils +# Import Salt libs +import salt.utils.files +import salt.utils.platform from salt.ext.six.moves import range @skip_if_not_root -@skipIf(not salt.utils.is_linux(), 'These tests can only be run on linux') +@skipIf(not salt.utils.platform.is_linux(), 'These tests can only be run on linux') class ShadowModuleTest(ModuleCase): ''' Validate the linux shadow system module @@ -217,7 +218,7 @@ class ShadowModuleTest(ModuleCase): #saving shadow file if not os.access("/etc/shadow", os.R_OK | os.W_OK): self.skipTest('Could not save initial state of /etc/shadow') - with salt.utils.fopen('/etc/shadow', 'r') as sFile: + with salt.utils.files.fopen('/etc/shadow', 'r') as sFile: shadow = sFile.read() #set root password self.assertTrue(self.run_function('shadow.set_password', ['root', self._password])) @@ -228,5 +229,5 @@ class ShadowModuleTest(ModuleCase): self.assertEqual( self.run_function('shadow.info', ['root'])['passwd'], '') #restore shadow file - with salt.utils.fopen('/etc/shadow', 'w') as sFile: + with salt.utils.files.fopen('/etc/shadow', 'w') as sFile: sFile.write(shadow) diff --git a/tests/integration/modules/test_ssh.py b/tests/integration/modules/test_ssh.py index ac2d2b4d7a2..b565129ef08 100644 --- a/tests/integration/modules/test_ssh.py +++ b/tests/integration/modules/test_ssh.py @@ -14,7 +14,7 @@ from tests.support.paths import FILES, TMP from tests.support.helpers import skip_if_binaries_missing # Import salt libs -import salt.utils +import salt.utils.files # Import 3rd-party libs from tornado.httpclient import HTTPClient @@ -51,7 +51,7 @@ class SSHModuleTest(ModuleCase): os.makedirs(SUBSALT_DIR) ssh_raw_path = os.path.join(FILES, 'ssh', 'raw') - with salt.utils.fopen(ssh_raw_path) as fd: + with salt.utils.files.fopen(ssh_raw_path) as fd: self.key = fd.read().strip() def tearDown(self): diff --git a/tests/integration/modules/test_state.py b/tests/integration/modules/test_state.py index e75373f4154..9b1d33ce3bf 100644 --- a/tests/integration/modules/test_state.py +++ b/tests/integration/modules/test_state.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Import python libs +# Import Python libs from __future__ import absolute_import import os import shutil @@ -14,12 +14,14 @@ from tests.support.unit import skipIf from tests.support.paths import TMP from tests.support.mixins import SaltReturnAssertsMixin -# Import salt libs -import salt.utils +# Import Salt libs +import salt.utils.files +import salt.utils.path +import salt.utils.platform from salt.modules.virtualenv_mod import KNOWN_BINARY_NAMES # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six class StateModuleTest(ModuleCase, SaltReturnAssertsMixin): @@ -107,7 +109,7 @@ class StateModuleTest(ModuleCase, SaltReturnAssertsMixin): self.assertTrue(isinstance(ret['__sls__'], type(None))) for state, ret in sls2.items(): - self.assertTrue(isinstance(ret['__sls__'], str)) + self.assertTrue(isinstance(ret['__sls__'], six.string_types)) def _remove_request_cache_file(self): ''' @@ -154,14 +156,14 @@ class StateModuleTest(ModuleCase, SaltReturnAssertsMixin): ''' self._remove_request_cache_file() - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): self.run_function('state.request', mods='modules.state.requested_win') else: self.run_function('state.request', mods='modules.state.requested') ret = self.run_function('state.run_request') - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): key = 'cmd_|-count_root_dir_contents_|-Get-ChildItem C:\\\\ | Measure-Object | %{$_.Count}_|-run' else: key = 'cmd_|-count_root_dir_contents_|-ls -a / | wc -l_|-run' @@ -197,7 +199,7 @@ class StateModuleTest(ModuleCase, SaltReturnAssertsMixin): ret = self.run_function('state.sls', mods='testappend.step-2') self.assertSaltTrueReturn(ret) - with salt.utils.fopen(testfile, 'r') as fp_: + with salt.utils.files.fopen(testfile, 'r') as fp_: testfile_contents = fp_.read() contents = textwrap.dedent('''\ @@ -207,7 +209,7 @@ class StateModuleTest(ModuleCase, SaltReturnAssertsMixin): fi ''') - if not salt.utils.is_windows(): + if not salt.utils.platform.is_windows(): contents += os.linesep contents += textwrap.dedent('''\ @@ -217,7 +219,7 @@ class StateModuleTest(ModuleCase, SaltReturnAssertsMixin): fi ''') - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): new_contents = contents.splitlines() contents = os.linesep.join(new_contents) contents += os.linesep @@ -232,7 +234,7 @@ class StateModuleTest(ModuleCase, SaltReturnAssertsMixin): ret = self.run_function('state.sls', mods='testappend.step-1') self.assertSaltTrueReturn(ret) - with salt.utils.fopen(testfile, 'r') as fp_: + with salt.utils.files.fopen(testfile, 'r') as fp_: testfile_contents = fp_.read() self.assertMultiLineEqual(contents, testfile_contents) @@ -271,7 +273,7 @@ class StateModuleTest(ModuleCase, SaltReturnAssertsMixin): fi ''') - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): new_contents = expected.splitlines() expected = os.linesep.join(new_contents) expected += os.linesep @@ -299,7 +301,7 @@ class StateModuleTest(ModuleCase, SaltReturnAssertsMixin): # Does it match? try: - with salt.utils.fopen(testfile, 'r') as fp_: + with salt.utils.files.fopen(testfile, 'r') as fp_: contents = fp_.read() self.assertMultiLineEqual(expected, contents) # Make sure we don't re-append existing text @@ -313,7 +315,7 @@ class StateModuleTest(ModuleCase, SaltReturnAssertsMixin): ) self.assertSaltTrueReturn(ret) - with salt.utils.fopen(testfile, 'r') as fp_: + with salt.utils.files.fopen(testfile, 'r') as fp_: contents = fp_.read() self.assertMultiLineEqual(expected, contents) except Exception: @@ -364,7 +366,7 @@ class StateModuleTest(ModuleCase, SaltReturnAssertsMixin): if os.path.isfile(fname): os.remove(fname) - @skipIf(salt.utils.which_bin(KNOWN_BINARY_NAMES) is None, 'virtualenv not installed') + @skipIf(salt.utils.path.which_bin(KNOWN_BINARY_NAMES) is None, 'virtualenv not installed') def test_issue_2068_template_str(self): venv_dir = os.path.join( TMP, 'issue-2068-template-str' @@ -387,7 +389,7 @@ class StateModuleTest(ModuleCase, SaltReturnAssertsMixin): 'files', 'file', 'base', 'issue-2068-template-str-no-dot.sls' ) - with salt.utils.fopen(template_path, 'r') as fp_: + with salt.utils.files.fopen(template_path, 'r') as fp_: template = fp_.read() ret = self.run_function( 'state.template_str', [template], timeout=120 @@ -413,7 +415,7 @@ class StateModuleTest(ModuleCase, SaltReturnAssertsMixin): 'files', 'file', 'base', 'issue-2068-template-str.sls' ) - with salt.utils.fopen(template_path, 'r') as fp_: + with salt.utils.files.fopen(template_path, 'r') as fp_: template = fp_.read() ret = self.run_function( 'state.template_str', [template], timeout=120 @@ -939,7 +941,7 @@ class StateModuleTest(ModuleCase, SaltReturnAssertsMixin): ) self.assertSaltTrueReturn(ret) self.assertTrue(os.path.isfile(tgt)) - with salt.utils.fopen(tgt, 'r') as cheese: + with salt.utils.files.fopen(tgt, 'r') as cheese: data = cheese.read() self.assertIn('Gromit', data) self.assertIn('Comte', data) diff --git a/tests/integration/modules/test_supervisord.py b/tests/integration/modules/test_supervisord.py index 0329baef5b6..4d390558fa1 100644 --- a/tests/integration/modules/test_supervisord.py +++ b/tests/integration/modules/test_supervisord.py @@ -12,16 +12,16 @@ from tests.support.unit import skipIf from tests.support.paths import TMP # Import salt libs -import salt.utils +import salt.utils.path from salt.modules.virtualenv_mod import KNOWN_BINARY_NAMES # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six @skipIf(six.PY3, 'supervisor does not work under python 3') -@skipIf(salt.utils.which_bin(KNOWN_BINARY_NAMES) is None, 'virtualenv not installed') -@skipIf(salt.utils.which('supervisorctl') is None, 'supervisord not installed') +@skipIf(salt.utils.path.which_bin(KNOWN_BINARY_NAMES) is None, 'virtualenv not installed') +@skipIf(salt.utils.path.which('supervisorctl') is None, 'supervisord not installed') class SupervisordModuleTest(ModuleCase): ''' Validates the supervisorctl functions. diff --git a/tests/integration/modules/test_system.py b/tests/integration/modules/test_system.py index 734939c09f3..9a3a42d9837 100644 --- a/tests/integration/modules/test_system.py +++ b/tests/integration/modules/test_system.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Import python libs +# Import Python libs from __future__ import absolute_import import datetime import logging @@ -13,15 +13,17 @@ from tests.support.case import ModuleCase from tests.support.unit import skipIf from tests.support.helpers import destructiveTest, skip_if_not_root -# Import salt libs -import salt.utils +# Import Salt libs +import salt.utils.files +import salt.utils.path +import salt.utils.platform import salt.states.file from salt.ext.six.moves import range log = logging.getLogger(__name__) -@skipIf(not salt.utils.is_linux(), 'These tests can only be run on linux') +@skipIf(not salt.utils.platform.is_linux(), 'These tests can only be run on linux') class SystemModuleTest(ModuleCase): ''' Validate the date/time functions in the system module @@ -108,7 +110,7 @@ class SystemModuleTest(ModuleCase): log.debug('Comparing hwclock to sys clock') with os.fdopen(rpipeFd, "r") as rpipe: with os.fdopen(wpipeFd, "w") as wpipe: - with salt.utils.fopen(os.devnull, "r") as nulFd: + with salt.utils.files.fopen(os.devnull, "r") as nulFd: p = subprocess.Popen(args=['hwclock', '--compare'], stdin=nulFd, stdout=wpipeFd, stderr=subprocess.PIPE) p.communicate() @@ -140,14 +142,14 @@ class SystemModuleTest(ModuleCase): def _save_machine_info(self): if os.path.isfile('/etc/machine-info'): - with salt.utils.fopen('/etc/machine-info', 'r') as mach_info: + with salt.utils.files.fopen('/etc/machine-info', 'r') as mach_info: self._machine_info = mach_info.read() else: self._machine_info = False def _restore_machine_info(self): if self._machine_info is not False: - with salt.utils.fopen('/etc/machine-info', 'w') as mach_info: + with salt.utils.files.fopen('/etc/machine-info', 'w') as mach_info: mach_info.write(self._machine_info) else: self.run_function('file.remove', ['/etc/machine-info']) @@ -299,7 +301,7 @@ class SystemModuleTest(ModuleCase): ''' res = self.run_function('system.get_computer_desc') - hostname_cmd = salt.utils.which('hostnamectl') + hostname_cmd = salt.utils.path.which('hostnamectl') if hostname_cmd: desc = self.run_function('cmd.run', ["hostnamectl status --pretty"]) self.assertEqual(res, desc) @@ -307,7 +309,7 @@ class SystemModuleTest(ModuleCase): if not os.path.isfile('/etc/machine-info'): self.assertFalse(res) else: - with salt.utils.fopen('/etc/machine-info', 'r') as mach_info: + with salt.utils.files.fopen('/etc/machine-info', 'r') as mach_info: data = mach_info.read() self.assertIn(res, data.decode('string_escape')) diff --git a/tests/integration/modules/test_useradd.py b/tests/integration/modules/test_useradd.py index 8e8b2138e2d..0791f2057eb 100644 --- a/tests/integration/modules/test_useradd.py +++ b/tests/integration/modules/test_useradd.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Import python libs +# Import Python libs from __future__ import absolute_import import string import random @@ -14,15 +14,15 @@ from tests.support.helpers import ( requires_system_grains ) -# Import salt libs -import salt.utils +# Import Salt libs +import salt.utils.platform # Import 3rd-party libs from salt.ext.six.moves import range # pylint: disable=import-error,redefined-builtin @destructiveTest -@skipIf(not salt.utils.is_linux(), 'These tests can only be run on linux') +@skipIf(not salt.utils.platform.is_linux(), 'These tests can only be run on linux') @skip_if_not_root class UseraddModuleTestLinux(ModuleCase): @@ -108,7 +108,7 @@ class UseraddModuleTestLinux(ModuleCase): @destructiveTest -@skipIf(not salt.utils.is_windows(), 'These tests can only be run on Windows') +@skipIf(not salt.utils.platform.is_windows(), 'These tests can only be run on Windows') @skip_if_not_root class UseraddModuleTestWindows(ModuleCase): diff --git a/tests/integration/modules/test_virtualenv.py b/tests/integration/modules/test_virtualenv.py index 79541fdc093..dc9da0ce321 100644 --- a/tests/integration/modules/test_virtualenv.py +++ b/tests/integration/modules/test_virtualenv.py @@ -11,11 +11,11 @@ from tests.support.unit import skipIf from tests.support.paths import TMP # Import salt libs -import salt.utils +import salt.utils.path from salt.modules.virtualenv_mod import KNOWN_BINARY_NAMES -@skipIf(salt.utils.which_bin(KNOWN_BINARY_NAMES) is None, 'virtualenv not installed') +@skipIf(salt.utils.path.which_bin(KNOWN_BINARY_NAMES) is None, 'virtualenv not installed') class VirtualenvModuleTest(ModuleCase): ''' Validate the virtualenv module diff --git a/tests/integration/netapi/rest_cherrypy/test_app.py b/tests/integration/netapi/rest_cherrypy/test_app.py index af01725135f..78a9071301e 100644 --- a/tests/integration/netapi/rest_cherrypy/test_app.py +++ b/tests/integration/netapi/rest_cherrypy/test_app.py @@ -6,6 +6,7 @@ import json # Import salt libs import salt.utils +import salt.utils.stringutils # Import test support libs import tests.support.cherrypy_testclasses as cptc @@ -195,7 +196,7 @@ class TestArgKwarg(cptc.BaseRestCherryPyTest): 'Accept': 'application/json', } ) - resp = json.loads(salt.utils.to_str(response.body[0])) + resp = json.loads(salt.utils.stringutils.to_str(response.body[0])) self.assertEqual(resp['return'][0]['args'], [1234]) self.assertEqual(resp['return'][0]['kwargs'], {'ext_source': 'redis'}) @@ -253,6 +254,6 @@ class TestJobs(cptc.BaseRestCherryPyTest): 'X-Auth-Token': self._token(), }) - resp = json.loads(salt.utils.to_str(response.body[0])) + resp = json.loads(salt.utils.stringutils.to_str(response.body[0])) self.assertIn('test.ping', str(resp['return'])) self.assertEqual(response.status, '200 OK') diff --git a/tests/integration/netapi/rest_cherrypy/test_app_pam.py b/tests/integration/netapi/rest_cherrypy/test_app_pam.py index 5c16868d22f..9fda228c084 100644 --- a/tests/integration/netapi/rest_cherrypy/test_app_pam.py +++ b/tests/integration/netapi/rest_cherrypy/test_app_pam.py @@ -3,7 +3,7 @@ Integration Tests for restcherry salt-api with pam eauth ''' -# Import python libs +# Import Python libs from __future__ import absolute_import # Import test support libs @@ -13,7 +13,7 @@ from tests.support.helpers import destructiveTest, skip_if_not_root import tests.support.cherrypy_testclasses as cptc # Import Salt Libs -import salt.utils +import salt.utils.platform # Import 3rd-party libs from salt.ext.six.moves.urllib.parse import urlencode # pylint: disable=no-name-in-module,import-error @@ -42,8 +42,13 @@ class TestAuthPAM(cptc.BaseRestCherryPyTest, ModuleCase): super(TestAuthPAM, self).setUp() try: add_user = self.run_function('user.add', [USERA], createhome=False) - add_pwd = self.run_function('shadow.set_password', - [USERA, USERA_PWD if salt.utils.is_darwin() else HASHED_USERA_PWD]) + add_pwd = self.run_function( + 'shadow.set_password', + [ + USERA, + USERA_PWD if salt.utils.platform.is_darwin() else HASHED_USERA_PWD + ] + ) self.assertTrue(add_user) self.assertTrue(add_pwd) user_list = self.run_function('user.list_users') diff --git a/tests/integration/netapi/rest_tornado/test_app.py b/tests/integration/netapi/rest_tornado/test_app.py index 1712f80d454..6597329a7c6 100644 --- a/tests/integration/netapi/rest_tornado/test_app.py +++ b/tests/integration/netapi/rest_tornado/test_app.py @@ -9,6 +9,7 @@ import threading # Import Salt Libs import salt.utils +import salt.utils.stringutils from salt.netapi.rest_tornado import saltnado from salt.utils.versions import StrictVersion @@ -18,7 +19,7 @@ from tests.support.helpers import flaky from tests.support.unit import skipIf # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six try: import zmq from zmq.eventloop.ioloop import ZMQIOLoop @@ -574,7 +575,7 @@ class TestWebhookSaltAPIHandler(_SaltnadoIntegrationTestCase): self.assertIn('headers', event['data']) self.assertEqual( event['data']['post'], - {'foo': salt.utils.to_bytes('bar')} + {'foo': salt.utils.stringutils.to_bytes('bar')} ) finally: self._future_resolved.clear() diff --git a/tests/integration/pillar/test_git_pillar.py b/tests/integration/pillar/test_git_pillar.py index 305e7c591f9..9cb9cb53313 100644 --- a/tests/integration/pillar/test_git_pillar.py +++ b/tests/integration/pillar/test_git_pillar.py @@ -85,7 +85,8 @@ from tests.support.mock import NO_MOCK, NO_MOCK_REASON from tests.support.unit import skipIf # Import Salt libs -import salt.utils +import salt.utils.path +import salt.utils.platform from salt.utils.gitfs import GITPYTHON_MINVER, PYGIT2_MINVER from salt.utils.versions import LooseVersion from salt.modules.virtualenv_mod import KNOWN_BINARY_NAMES as VIRTUALENV_NAMES @@ -106,9 +107,9 @@ try: except ImportError: HAS_PYGIT2 = False -HAS_SSHD = bool(salt.utils.which('sshd')) -HAS_NGINX = bool(salt.utils.which('nginx')) -HAS_VIRTUALENV = bool(salt.utils.which_bin(VIRTUALENV_NAMES)) +HAS_SSHD = bool(salt.utils.path.which('sshd')) +HAS_NGINX = bool(salt.utils.path.which('nginx')) +HAS_VIRTUALENV = bool(salt.utils.path.which_bin(VIRTUALENV_NAMES)) def _rand_key_name(length): @@ -118,7 +119,7 @@ def _rand_key_name(length): def _windows_or_mac(): - return salt.utils.is_windows() or salt.utils.is_darwin() + return salt.utils.platform.is_windows() or salt.utils.platform.is_darwin() class GitPythonMixin(object): diff --git a/tests/integration/renderers/test_pydsl.py b/tests/integration/renderers/test_pydsl.py index e41e4297bf3..0d904b4ce09 100644 --- a/tests/integration/renderers/test_pydsl.py +++ b/tests/integration/renderers/test_pydsl.py @@ -9,7 +9,8 @@ import textwrap from tests.support.case import ModuleCase # Import Salt libs -import salt.utils +import salt.utils.files +import salt.utils.platform class PyDSLRendererIncludeTestCase(ModuleCase): @@ -36,7 +37,7 @@ class PyDSLRendererIncludeTestCase(ModuleCase): # Windows adds `linefeed` in addition to `newline`. There's also an # unexplainable space before the `linefeed`... - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): expected = 'X1 \r\n' \ 'X2 \r\n' \ 'X3 \r\n' \ @@ -47,7 +48,7 @@ class PyDSLRendererIncludeTestCase(ModuleCase): 'hello green 2 \r\n' \ 'hello blue 3 \r\n' - with salt.utils.fopen('/tmp/output', 'r') as f: + with salt.utils.files.fopen('/tmp/output', 'r') as f: ret = f.read() os.remove('/tmp/output') diff --git a/tests/integration/runners/test_fileserver.py b/tests/integration/runners/test_fileserver.py index 813201baf81..5f18dcbf5ed 100644 --- a/tests/integration/runners/test_fileserver.py +++ b/tests/integration/runners/test_fileserver.py @@ -10,8 +10,8 @@ import contextlib from tests.support.case import ShellCase from tests.support.unit import skipIf -# Import salt libs -import salt.utils +# Import Salt libs +import salt.utils.platform class FileserverTest(ShellCase): @@ -163,7 +163,7 @@ class FileserverTest(ShellCase): # Git doesn't handle symlinks in Windows. See the thread below: # http://stackoverflow.com/questions/5917249/git-symlinks-in-windows - @skipIf(salt.utils.is_windows(), + @skipIf(salt.utils.platform.is_windows(), 'Git for Windows does not preserve symbolic links when cloning') def test_symlink_list(self): ''' diff --git a/tests/integration/runners/test_runner_returns.py b/tests/integration/runners/test_runner_returns.py index 2e326f7f5e3..86820a96680 100644 --- a/tests/integration/runners/test_runner_returns.py +++ b/tests/integration/runners/test_runner_returns.py @@ -15,7 +15,8 @@ from tests.support.runtests import RUNTIME_VARS # Import salt libs import salt.payload -import salt.utils +import salt.utils.args +import salt.utils.files import salt.utils.jid @@ -60,9 +61,9 @@ class RunnerReturnsTest(ShellCase): ''' # Remove pub_kwargs data['fun_args'][1] = \ - salt.utils.clean_kwargs(**data['fun_args'][1]) + salt.utils.args.clean_kwargs(**data['fun_args'][1]) data['return']['kwargs'] = \ - salt.utils.clean_kwargs(**data['return']['kwargs']) + salt.utils.args.clean_kwargs(**data['return']['kwargs']) # Pop off the timestamp (do not provide a 2nd argument, if the stamp is # missing we want to know!) @@ -118,7 +119,7 @@ class RunnerReturnsTest(ShellCase): 'return.p', ) serial = salt.payload.Serial(self.master_opts) - with salt.utils.fopen(serialized_return, 'rb') as fp_: + with salt.utils.files.fopen(serialized_return, 'rb') as fp_: deserialized = serial.loads(fp_.read()) self.clean_return(deserialized['return']) diff --git a/tests/integration/runners/test_state.py b/tests/integration/runners/test_state.py index 08d88275700..e14dd062511 100644 --- a/tests/integration/runners/test_state.py +++ b/tests/integration/runners/test_state.py @@ -22,7 +22,9 @@ from tests.support.paths import TMP # Import Salt Libs import salt.utils +import salt.utils.platform import salt.utils.event +import salt.utils.files class StateRunnerTest(ShellCase): @@ -101,7 +103,7 @@ class StateRunnerTest(ShellCase): server_thread.join() -@skipIf(salt.utils.is_windows(), '*NIX-only test') +@skipIf(salt.utils.platform.is_windows(), '*NIX-only test') class OrchEventTest(ShellCase): ''' Tests for orchestration events @@ -153,14 +155,14 @@ class OrchEventTest(ShellCase): }) state_sls = os.path.join(self.base_env, 'test_state.sls') - with salt.utils.fopen(state_sls, 'w') as fp_: + with salt.utils.files.fopen(state_sls, 'w') as fp_: fp_.write(textwrap.dedent(''' date: cmd.run ''')) orch_sls = os.path.join(self.base_env, 'test_orch.sls') - with salt.utils.fopen(orch_sls, 'w') as fp_: + with salt.utils.files.fopen(orch_sls, 'w') as fp_: fp_.write(textwrap.dedent(''' date_cmd: salt.state: diff --git a/tests/integration/sdb/test_env.py b/tests/integration/sdb/test_env.py index e759fe295cd..0fcccee6159 100644 --- a/tests/integration/sdb/test_env.py +++ b/tests/integration/sdb/test_env.py @@ -12,7 +12,7 @@ from tests.support.paths import FILES from tests.support.mixins import SaltReturnAssertsMixin # Import salt libs -import salt.utils +import salt.utils.files STATE_DIR = os.path.join(FILES, 'file', 'base') @@ -23,7 +23,7 @@ class EnvTestCase(ModuleCase, SaltReturnAssertsMixin): self.state_name = 'test_sdb_env' self.state_file_name = self.state_name + '.sls' self.state_file_set_var = os.path.join(STATE_DIR, self.state_file_name) - with salt.utils.fopen(self.state_file_set_var, 'w') as wfh: + with salt.utils.files.fopen(self.state_file_set_var, 'w') as wfh: wfh.write(textwrap.dedent('''\ set some env var: cmd.run: diff --git a/tests/integration/shell/test_auth.py b/tests/integration/shell/test_auth.py index 3c4c7000a2e..31b7aa11f45 100644 --- a/tests/integration/shell/test_auth.py +++ b/tests/integration/shell/test_auth.py @@ -4,8 +4,9 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ''' -# Import python libs +# Import Python libs from __future__ import absolute_import +import logging import pwd import grp import random @@ -14,13 +15,15 @@ import random from tests.support.case import ShellCase from tests.support.helpers import destructiveTest, skip_if_not_root -# Import salt libs -import salt.utils +# Import Salt libs +import salt.utils.platform from salt.utils.pycrypto import gen_hash # Import 3rd-party libs from salt.ext.six.moves import range # pylint: disable=import-error,redefined-builtin +log = logging.getLogger(__name__) + def gen_password(): ''' @@ -54,7 +57,6 @@ class AuthTest(ShellCase): group = 'saltops' def setUp(self): - # This is a little wasteful but shouldn't be a problem for user in (self.userA, self.userB): try: pwd.getpwnam(user) @@ -68,6 +70,21 @@ class AuthTest(ShellCase): self.run_call('group.add {0}'.format(self.group)) self.run_call('user.chgroups {0} {1} True'.format(self.userB, self.group)) + def tearDown(self): + for user in (self.userA, self.userB): + try: + pwd.getpwnam(user) + except KeyError: + pass + else: + self.run_call('user.delete {0}'.format(user)) + try: + grp.getgrnam(self.group) + except KeyError: + pass + else: + self.run_call('group.delete {0}'.format(self.group)) + def test_pam_auth_valid_user(self): ''' test that pam auth mechanism works with a valid user @@ -77,7 +94,7 @@ class AuthTest(ShellCase): # set user password set_pw_cmd = "shadow.set_password {0} '{1}'".format( self.userA, - password if salt.utils.is_darwin() else hashed_pwd + password if salt.utils.platform.is_darwin() else hashed_pwd ) self.run_call(set_pw_cmd) @@ -85,6 +102,7 @@ class AuthTest(ShellCase): cmd = ('-a pam "*" test.ping ' '--username {0} --password {1}'.format(self.userA, password)) resp = self.run_salt(cmd) + log.debug('resp = %s', resp) self.assertTrue( 'minion:' in resp ) @@ -109,7 +127,7 @@ class AuthTest(ShellCase): # set user password set_pw_cmd = "shadow.set_password {0} '{1}'".format( self.userB, - password if salt.utils.is_darwin() else hashed_pwd + password if salt.utils.platform.is_darwin() else hashed_pwd ) self.run_call(set_pw_cmd) @@ -121,10 +139,3 @@ class AuthTest(ShellCase): self.assertTrue( 'minion:' in resp ) - - def test_zzzz_tearDown(self): - for user in (self.userA, self.userB): - if pwd.getpwnam(user): - self.run_call('user.delete {0}'.format(user)) - if grp.getgrnam(self.group): - self.run_call('group.delete {0}'.format(self.group)) diff --git a/tests/integration/shell/test_call.py b/tests/integration/shell/test_call.py index 90fd271e8c2..a86cf1a822e 100644 --- a/tests/integration/shell/test_call.py +++ b/tests/integration/shell/test_call.py @@ -29,8 +29,8 @@ from tests.support.helpers import destructiveTest from tests.integration.utils import testprogram # Import salt libs -import salt.utils -import salt.ext.six as six +import salt.utils.files +from salt.ext import six log = logging.getLogger(__name__) @@ -174,7 +174,7 @@ class CallTest(ShellCase, testprogram.TestProgramCase, ShellCaseCommonTestsMixin if not os.path.isdir(config_dir): os.makedirs(config_dir) - with salt.utils.fopen(self.get_config_file_path('master')) as fhr: + with salt.utils.files.fopen(self.get_config_file_path('master')) as fhr: master_config = yaml.load(fhr.read()) master_root_dir = master_config['root_dir'] @@ -204,7 +204,7 @@ class CallTest(ShellCase, testprogram.TestProgramCase, ShellCaseCommonTestsMixin start = datetime.now() # Let's first test with a master running - with salt.utils.fopen(minion_config_file, 'w') as fh_: + with salt.utils.files.fopen(minion_config_file, 'w') as fh_: fh_.write( yaml.dump(minion_config, default_flow_style=False) ) @@ -234,7 +234,7 @@ class CallTest(ShellCase, testprogram.TestProgramCase, ShellCaseCommonTestsMixin # Now let's remove the master configuration minion_config.pop('master') minion_config.pop('master_port') - with salt.utils.fopen(minion_config_file, 'w') as fh_: + with salt.utils.files.fopen(minion_config_file, 'w') as fh_: fh_.write( yaml.dump(minion_config, default_flow_style=False) ) @@ -282,7 +282,7 @@ class CallTest(ShellCase, testprogram.TestProgramCase, ShellCaseCommonTestsMixin # Should work with local file client minion_config['file_client'] = 'local' - with salt.utils.fopen(minion_config_file, 'w') as fh_: + with salt.utils.files.fopen(minion_config_file, 'w') as fh_: fh_.write( yaml.dump(minion_config, default_flow_style=False) ) @@ -309,10 +309,10 @@ class CallTest(ShellCase, testprogram.TestProgramCase, ShellCaseCommonTestsMixin os.chdir(config_dir) - with salt.utils.fopen(self.get_config_file_path('minion'), 'r') as fh_: + with salt.utils.files.fopen(self.get_config_file_path('minion'), 'r') as fh_: minion_config = yaml.load(fh_.read()) minion_config['log_file'] = 'file:///dev/log/LOG_LOCAL3' - with salt.utils.fopen(os.path.join(config_dir, 'minion'), 'w') as fh_: + with salt.utils.files.fopen(os.path.join(config_dir, 'minion'), 'w') as fh_: fh_.write( yaml.dump(minion_config, default_flow_style=False) ) @@ -353,7 +353,7 @@ class CallTest(ShellCase, testprogram.TestProgramCase, ShellCaseCommonTestsMixin with_retcode=True ) - with salt.utils.fopen(output_file_append) as ofa: + with salt.utils.files.fopen(output_file_append) as ofa: output = ofa.read() self.run_script( @@ -365,7 +365,7 @@ class CallTest(ShellCase, testprogram.TestProgramCase, ShellCaseCommonTestsMixin catch_stderr=True, with_retcode=True ) - with salt.utils.fopen(output_file_append) as ofa: + with salt.utils.files.fopen(output_file_append) as ofa: self.assertEqual(ofa.read(), output + output) finally: if os.path.exists(output_file_append): @@ -426,6 +426,21 @@ class CallTest(ShellCase, testprogram.TestProgramCase, ShellCaseCommonTestsMixin # Restore umask os.umask(current_umask) + @skipIf(sys.platform.startswith('win'), 'This test does not apply on Win') + def test_42116_cli_pillar_override(self): + ret = self.run_call( + 'state.apply issue-42116-cli-pillar-override ' + 'pillar=\'{"myhost": "localhost"}\'' + ) + for line in ret: + line = line.lstrip() + if line == 'Comment: Command "ping -c 2 localhost" run': + # Successful test + break + else: + log.debug('salt-call output:\n\n%s', '\n'.join(ret)) + self.fail('CLI pillar override not found in pillar data') + def tearDown(self): ''' Teardown method to remove installed packages diff --git a/tests/integration/shell/test_cp.py b/tests/integration/shell/test_cp.py index d94b34f9262..1e7a5978e1b 100644 --- a/tests/integration/shell/test_cp.py +++ b/tests/integration/shell/test_cp.py @@ -24,10 +24,10 @@ from tests.support.paths import TMP from tests.support.mixins import ShellCaseCommonTestsMixin # Import salt libs -import salt.utils +import salt.utils.files # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six class CopyTest(ShellCase, ShellCaseCommonTestsMixin): @@ -53,7 +53,7 @@ class CopyTest(ShellCase, ShellCaseCommonTestsMixin): 'files', 'file', 'base', 'testfile' ) ) - with salt.utils.fopen(testfile, 'r') as fh_: + with salt.utils.files.fopen(testfile, 'r') as fh_: testfile_contents = fh_.read() for idx, minion in enumerate(minions): @@ -125,10 +125,10 @@ class CopyTest(ShellCase, ShellCaseCommonTestsMixin): raise config_file_name = 'master' - with salt.utils.fopen(self.get_config_file_path(config_file_name), 'r') as fhr: + with salt.utils.files.fopen(self.get_config_file_path(config_file_name), 'r') as fhr: config = yaml.load(fhr.read()) config['log_file'] = 'file:///dev/log/LOG_LOCAL3' - with salt.utils.fopen(os.path.join(config_dir, config_file_name), 'w') as fhw: + with salt.utils.files.fopen(os.path.join(config_dir, config_file_name), 'w') as fhw: fhw.write( yaml.dump(config, default_flow_style=False) ) @@ -137,7 +137,7 @@ class CopyTest(ShellCase, ShellCaseCommonTestsMixin): fd_, fn_ = tempfile.mkstemp() os.close(fd_) - with salt.utils.fopen(fn_, 'w') as fp_: + with salt.utils.files.fopen(fn_, 'w') as fp_: fp_.write('Hello world!\n') ret = self.run_script( diff --git a/tests/integration/shell/test_enabled.py b/tests/integration/shell/test_enabled.py index 1b3685dfaf7..c59a81d8bf9 100644 --- a/tests/integration/shell/test_enabled.py +++ b/tests/integration/shell/test_enabled.py @@ -10,7 +10,7 @@ from tests.support.case import ModuleCase from tests.support.paths import FILES # Import Salt Libs -import salt.utils +import salt.utils.files STATE_DIR = os.path.join(FILES, 'file', 'base') @@ -58,7 +58,7 @@ class EnabledTest(ModuleCase): ret_key = 'test_|-shell_enabled_|-{0}_|-configurable_test_state'.format(enabled_ret) try: - with salt.utils.fopen(state_file, 'w') as fp_: + with salt.utils.files.fopen(state_file, 'w') as fp_: fp_.write(textwrap.dedent('''\ {{% set shell_enabled = salt['cmd.shell']("{0}").strip() %}} @@ -87,7 +87,7 @@ class EnabledTest(ModuleCase): ret_key = 'test_|-shell_enabled_|-{0}_|-configurable_test_state'.format(disabled_ret) try: - with salt.utils.fopen(state_file, 'w') as fp_: + with salt.utils.files.fopen(state_file, 'w') as fp_: fp_.write(textwrap.dedent('''\ {{% set shell_disabled = salt['cmd.run']("{0}") %}} diff --git a/tests/integration/shell/test_key.py b/tests/integration/shell/test_key.py index fa6d5c36091..005f82d4525 100644 --- a/tests/integration/shell/test_key.py +++ b/tests/integration/shell/test_key.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -# Import python libs +# Import Python libs from __future__ import absolute_import import os import shutil @@ -14,8 +14,9 @@ from tests.support.mixins import ShellCaseCommonTestsMixin # Import 3rd-party libs import yaml -# Import salt libs -import salt.utils +# Import Salt libs +import salt.utils.files +import salt.utils.platform USERA = 'saltdev' USERA_PWD = 'saltdev' @@ -36,7 +37,7 @@ class KeyTest(ShellCase, ShellCaseCommonTestsMixin): try: add_user = self.run_call('user.add {0} createhome=False'.format(USERA)) add_pwd = self.run_call('shadow.set_password {0} \'{1}\''.format(USERA, - USERA_PWD if salt.utils.is_darwin() else HASHED_USERA_PWD)) + USERA_PWD if salt.utils.platform.is_darwin() else HASHED_USERA_PWD)) self.assertTrue(add_user) self.assertTrue(add_pwd) user_list = self.run_call('user.list_users') @@ -255,10 +256,10 @@ class KeyTest(ShellCase, ShellCaseCommonTestsMixin): os.chdir(config_dir) config_file_name = 'master' - with salt.utils.fopen(self.get_config_file_path(config_file_name), 'r') as fhr: + with salt.utils.files.fopen(self.get_config_file_path(config_file_name), 'r') as fhr: config = yaml.load(fhr.read()) config['log_file'] = 'file:///dev/log/LOG_LOCAL3' - with salt.utils.fopen(os.path.join(config_dir, config_file_name), 'w') as fhw: + with salt.utils.files.fopen(os.path.join(config_dir, config_file_name), 'w') as fhw: fhw.write( yaml.dump(config, default_flow_style=False) ) diff --git a/tests/integration/shell/test_master.py b/tests/integration/shell/test_master.py index 1a3bfdd2036..929c9a9653a 100644 --- a/tests/integration/shell/test_master.py +++ b/tests/integration/shell/test_master.py @@ -17,7 +17,7 @@ import shutil import yaml # Import salt libs -import salt.utils +import salt.utils.files # Import salt test libs import tests.integration.utils @@ -41,14 +41,14 @@ class MasterTest(ShellCase, testprogram.TestProgramCase, ShellCaseCommonTestsMix config_file_name = 'master' pid_path = os.path.join(config_dir, '{0}.pid'.format(config_file_name)) - with salt.utils.fopen(self.get_config_file_path(config_file_name), 'r') as fhr: + with salt.utils.files.fopen(self.get_config_file_path(config_file_name), 'r') as fhr: config = yaml.load(fhr.read()) config['root_dir'] = config_dir config['log_file'] = 'file:///tmp/log/LOG_LOCAL3' config['ret_port'] = config['ret_port'] + 10 config['publish_port'] = config['publish_port'] + 10 - with salt.utils.fopen(os.path.join(config_dir, config_file_name), 'w') as fhw: + with salt.utils.files.fopen(os.path.join(config_dir, config_file_name), 'w') as fhw: fhw.write( yaml.dump(config, default_flow_style=False) ) @@ -66,7 +66,7 @@ class MasterTest(ShellCase, testprogram.TestProgramCase, ShellCaseCommonTestsMix # Now kill it if still running if os.path.exists(pid_path): - with salt.utils.fopen(pid_path) as fhr: + with salt.utils.files.fopen(pid_path) as fhr: try: os.kill(int(fhr.read()), signal.SIGKILL) except OSError: diff --git a/tests/integration/shell/test_matcher.py b/tests/integration/shell/test_matcher.py index d30d15f143d..f6b371e2bb7 100644 --- a/tests/integration/shell/test_matcher.py +++ b/tests/integration/shell/test_matcher.py @@ -15,7 +15,7 @@ from tests.support.paths import TMP from tests.support.mixins import ShellCaseCommonTestsMixin # Import salt libs -import salt.utils +import salt.utils.files def minion_in_returns(minion, lines): @@ -348,12 +348,12 @@ class MatchTest(ShellCase, ShellCaseCommonTestsMixin): os.chdir(config_dir) config_file_name = 'master' - with salt.utils.fopen(self.get_config_file_path(config_file_name), 'r') as fhr: + with salt.utils.files.fopen(self.get_config_file_path(config_file_name), 'r') as fhr: config = yaml.load( fhr.read() ) config['log_file'] = 'file:///dev/log/LOG_LOCAL3' - with salt.utils.fopen(os.path.join(config_dir, config_file_name), 'w') as fhw: + with salt.utils.files.fopen(os.path.join(config_dir, config_file_name), 'w') as fhw: fhw.write( yaml.dump(config, default_flow_style=False) ) diff --git a/tests/integration/shell/test_minion.py b/tests/integration/shell/test_minion.py index b5c528aabb3..6f440ae6831 100644 --- a/tests/integration/shell/test_minion.py +++ b/tests/integration/shell/test_minion.py @@ -29,10 +29,10 @@ from tests.support.mixins import ShellCaseCommonTestsMixin from tests.integration.utils import testprogram # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six # Import salt libs -import salt.utils +import salt.utils.files log = logging.getLogger(__name__) @@ -60,11 +60,11 @@ class MinionTest(ShellCase, testprogram.TestProgramCase, ShellCaseCommonTestsMix config_file_name = 'minion' pid_path = os.path.join(config_dir, '{0}.pid'.format(config_file_name)) - with salt.utils.fopen(self.get_config_file_path(config_file_name), 'r') as fhr: + with salt.utils.files.fopen(self.get_config_file_path(config_file_name), 'r') as fhr: config = yaml.load(fhr.read()) config['log_file'] = 'file:///tmp/log/LOG_LOCAL3' - with salt.utils.fopen(os.path.join(config_dir, config_file_name), 'w') as fhw: + with salt.utils.files.fopen(os.path.join(config_dir, config_file_name), 'w') as fhw: fhw.write( yaml.dump(config, default_flow_style=False) ) @@ -82,7 +82,7 @@ class MinionTest(ShellCase, testprogram.TestProgramCase, ShellCaseCommonTestsMix # Now kill it if still running if os.path.exists(pid_path): - with salt.utils.fopen(pid_path) as fhr: + with salt.utils.files.fopen(pid_path) as fhr: try: os.kill(int(fhr.read()), signal.SIGKILL) except OSError: @@ -204,7 +204,7 @@ class MinionTest(ShellCase, testprogram.TestProgramCase, ShellCaseCommonTestsMix default_dir = os.path.join(sysconf_dir, 'default') if not os.path.exists(default_dir): os.makedirs(default_dir) - with salt.utils.fopen(os.path.join(default_dir, 'salt'), 'w') as defaults: + with salt.utils.files.fopen(os.path.join(default_dir, 'salt'), 'w') as defaults: # Test suites is quite slow - extend the timeout defaults.write( 'TIMEOUT=60\n' diff --git a/tests/integration/shell/test_runner.py b/tests/integration/shell/test_runner.py index 0d9541f2e99..3cdaa7cffb5 100644 --- a/tests/integration/shell/test_runner.py +++ b/tests/integration/shell/test_runner.py @@ -4,7 +4,7 @@ Tests for the salt-run command ''' -# Import python libs +# Import Python libs from __future__ import absolute_import import os import shutil @@ -19,8 +19,9 @@ from tests.support.helpers import skip_if_not_root # Import 3rd-party libs import yaml -# Import salt libs -import salt.utils +# Import Salt libs +import salt.utils.files +import salt.utils.platform USERA = 'saltdev' USERA_PWD = 'saltdev' @@ -41,7 +42,7 @@ class RunTest(ShellCase, testprogram.TestProgramCase, ShellCaseCommonTestsMixin) try: add_user = self.run_call('user.add {0} createhome=False'.format(USERA)) add_pwd = self.run_call('shadow.set_password {0} \'{1}\''.format(USERA, - USERA_PWD if salt.utils.is_darwin() else HASHED_USERA_PWD)) + USERA_PWD if salt.utils.platform.is_darwin() else HASHED_USERA_PWD)) self.assertTrue(add_user) self.assertTrue(add_pwd) user_list = self.run_call('user.list_users') @@ -100,10 +101,10 @@ class RunTest(ShellCase, testprogram.TestProgramCase, ShellCaseCommonTestsMixin) os.chdir(config_dir) config_file_name = 'master' - with salt.utils.fopen(self.get_config_file_path(config_file_name), 'r') as fhr: + with salt.utils.files.fopen(self.get_config_file_path(config_file_name), 'r') as fhr: config = yaml.load(fhr.read()) config['log_file'] = 'file:///dev/log/LOG_LOCAL3' - with salt.utils.fopen(os.path.join(config_dir, config_file_name), 'w') as fhw: + with salt.utils.files.fopen(os.path.join(config_dir, config_file_name), 'w') as fhw: fhw.write( yaml.dump(config, default_flow_style=False) ) @@ -117,7 +118,7 @@ class RunTest(ShellCase, testprogram.TestProgramCase, ShellCaseCommonTestsMixin) with_retcode=True ) try: - self.assertIn("'doc.runner:'", ret[0]) + self.assertIn('doc.runner:', ret[0]) self.assertFalse(os.path.isdir(os.path.join(config_dir, 'file:'))) except AssertionError: if os.path.exists('/dev/log') and ret[2] != 2: diff --git a/tests/integration/shell/test_syndic.py b/tests/integration/shell/test_syndic.py index fcc82e85f03..239b848b73a 100644 --- a/tests/integration/shell/test_syndic.py +++ b/tests/integration/shell/test_syndic.py @@ -24,7 +24,7 @@ from tests.support.mixins import ShellCaseCommonTestsMixin from tests.integration.utils import testprogram # Import salt libs -import salt.utils +import salt.utils.files log = logging.getLogger(__name__) @@ -47,7 +47,7 @@ class SyndicTest(ShellCase, testprogram.TestProgramCase, ShellCaseCommonTestsMix for fname in ('master', 'minion'): pid_path = os.path.join(config_dir, '{0}.pid'.format(fname)) - with salt.utils.fopen(self.get_config_file_path(fname), 'r') as fhr: + with salt.utils.files.fopen(self.get_config_file_path(fname), 'r') as fhr: config = yaml.load(fhr.read()) config['log_file'] = config['syndic_log_file'] = 'file:///tmp/log/LOG_LOCAL3' config['root_dir'] = config_dir @@ -55,7 +55,7 @@ class SyndicTest(ShellCase, testprogram.TestProgramCase, ShellCaseCommonTestsMix config['ret_port'] = int(config['ret_port']) + 10 config['publish_port'] = int(config['publish_port']) + 10 - with salt.utils.fopen(os.path.join(config_dir, fname), 'w') as fhw: + with salt.utils.files.fopen(os.path.join(config_dir, fname), 'w') as fhw: fhw.write( yaml.dump(config, default_flow_style=False) ) @@ -73,7 +73,7 @@ class SyndicTest(ShellCase, testprogram.TestProgramCase, ShellCaseCommonTestsMix # Now kill it if still running if os.path.exists(pid_path): - with salt.utils.fopen(pid_path) as fhr: + with salt.utils.files.fopen(pid_path) as fhr: try: os.kill(int(fhr.read()), signal.SIGKILL) except OSError: diff --git a/tests/integration/states/test_archive.py b/tests/integration/states/test_archive.py index b1777785ca3..d8063ee261b 100644 --- a/tests/integration/states/test_archive.py +++ b/tests/integration/states/test_archive.py @@ -2,7 +2,7 @@ ''' Tests for the archive state ''' -# Import python libs +# Import Python libs from __future__ import absolute_import import errno import logging @@ -12,20 +12,25 @@ import os from tests.support.case import ModuleCase from tests.support.helpers import skip_if_not_root, Webserver from tests.support.mixins import SaltReturnAssertsMixin +from tests.support.paths import FILES -# Import salt libs -import salt.utils +# Import Salt libs +import salt.utils.files +import salt.utils.platform # Setup logging log = logging.getLogger(__name__) -if salt.utils.is_windows(): - ARCHIVE_DIR = os.path.join('c:/', 'tmp') -else: - ARCHIVE_DIR = '/tmp/archive' +ARCHIVE_DIR = os.path.join('c:/', 'tmp') \ + if salt.utils.platform.is_windows() \ + else '/tmp/archive' +ARCHIVE_NAME = 'custom.tar.gz' +ARCHIVE_TAR_SOURCE = 'http://localhost:{0}/{1}'.format(9999, ARCHIVE_NAME) +ARCHIVE_LOCAL_TAR_SOURCE = 'file://{0}'.format(os.path.join(FILES, 'file', 'base', ARCHIVE_NAME)) UNTAR_FILE = os.path.join(ARCHIVE_DIR, 'custom/README') ARCHIVE_TAR_HASH = 'md5=7643861ac07c30fe7d2310e9f25ca514' +ARCHIVE_TAR_BAD_HASH = 'md5=d41d8cd98f00b204e9800998ecf8427e' class ArchiveTest(ModuleCase, SaltReturnAssertsMixin): @@ -51,7 +56,7 @@ class ArchiveTest(ModuleCase, SaltReturnAssertsMixin): @staticmethod def _clear_archive_dir(): try: - salt.utils.rm_rf(ARCHIVE_DIR) + salt.utils.files.rm_rf(ARCHIVE_DIR) except OSError as exc: if exc.errno != errno.ENOENT: raise @@ -100,7 +105,7 @@ class ArchiveTest(ModuleCase, SaltReturnAssertsMixin): test archive.extracted with user and group set to "root" ''' r_group = 'root' - if salt.utils.is_darwin(): + if salt.utils.platform.is_darwin(): r_group = 'wheel' ret = self.run_state('archive.extracted', name=ARCHIVE_DIR, source=self.archive_tar_source, archive_format='tar', @@ -178,3 +183,52 @@ class ArchiveTest(ModuleCase, SaltReturnAssertsMixin): self.assertSaltTrueReturn(ret) self._check_extracted(UNTAR_FILE) + + def test_local_archive_extracted(self): + ''' + test archive.extracted with local file + ''' + ret = self.run_state('archive.extracted', name=ARCHIVE_DIR, + source=ARCHIVE_LOCAL_TAR_SOURCE, archive_format='tar') + log.debug('ret = %s', ret) + + self.assertSaltTrueReturn(ret) + + self._check_extracted(UNTAR_FILE) + + def test_local_archive_extracted_skip_verify(self): + ''' + test archive.extracted with local file, bad hash and skip_verify + ''' + ret = self.run_state('archive.extracted', name=ARCHIVE_DIR, + source=ARCHIVE_LOCAL_TAR_SOURCE, archive_format='tar', + source_hash=ARCHIVE_TAR_BAD_HASH, skip_verify=True) + log.debug('ret = %s', ret) + + self.assertSaltTrueReturn(ret) + + self._check_extracted(UNTAR_FILE) + + def test_local_archive_extracted_with_source_hash(self): + ''' + test archive.extracted with local file and valid hash + ''' + ret = self.run_state('archive.extracted', name=ARCHIVE_DIR, + source=ARCHIVE_LOCAL_TAR_SOURCE, archive_format='tar', + source_hash=ARCHIVE_TAR_HASH) + log.debug('ret = %s', ret) + + self.assertSaltTrueReturn(ret) + + self._check_extracted(UNTAR_FILE) + + def test_local_archive_extracted_with_bad_source_hash(self): + ''' + test archive.extracted with local file and bad hash + ''' + ret = self.run_state('archive.extracted', name=ARCHIVE_DIR, + source=ARCHIVE_LOCAL_TAR_SOURCE, archive_format='tar', + source_hash=ARCHIVE_TAR_BAD_HASH) + log.debug('ret = %s', ret) + + self.assertSaltFalseReturn(ret) diff --git a/tests/integration/states/test_bower.py b/tests/integration/states/test_bower.py index 47ad72bb324..afbc2c4b982 100644 --- a/tests/integration/states/test_bower.py +++ b/tests/integration/states/test_bower.py @@ -13,10 +13,10 @@ from tests.support.helpers import destructiveTest from tests.support.mixins import SaltReturnAssertsMixin # Import salt libs -import salt.utils +import salt.utils.path -@skipIf(salt.utils.which('bower') is None, 'bower not installed') +@skipIf(salt.utils.path.which('bower') is None, 'bower not installed') class BowerStateTest(ModuleCase, SaltReturnAssertsMixin): @destructiveTest diff --git a/tests/integration/states/test_cmd.py b/tests/integration/states/test_cmd.py index e401cec76e2..ca40ae06e51 100644 --- a/tests/integration/states/test_cmd.py +++ b/tests/integration/states/test_cmd.py @@ -2,7 +2,7 @@ ''' Tests for the file state ''' -# Import python libs +# Import Python libs from __future__ import absolute_import import errno import os @@ -14,10 +14,11 @@ from tests.support.case import ModuleCase from tests.support.paths import TMP_STATE_TREE from tests.support.mixins import SaltReturnAssertsMixin -# Import salt libs -import salt.utils +# Import Salt libs +import salt.utils.files +import salt.utils.platform -IS_WINDOWS = salt.utils.is_windows() +IS_WINDOWS = salt.utils.platform.is_windows() class CMDTest(ModuleCase, SaltReturnAssertsMixin): @@ -84,7 +85,7 @@ class CMDRunRedirectTest(ModuleCase, SaltReturnAssertsMixin): test cmd.run unless ''' state_key = 'cmd_|-{0}_|-{0}_|-run'.format(self.test_tmp_path) - with salt.utils.fopen(self.state_file, 'w') as fb_: + with salt.utils.files.fopen(self.state_file, 'w') as fb_: fb_.write(textwrap.dedent(''' {0}: cmd.run: @@ -115,7 +116,7 @@ class CMDRunRedirectTest(ModuleCase, SaltReturnAssertsMixin): test cmd.run creates already there ''' state_key = 'cmd_|-echo >> {0}_|-echo >> {0}_|-run'.format(self.test_file) - with salt.utils.fopen(self.state_file, 'w') as fb_: + with salt.utils.files.fopen(self.state_file, 'w') as fb_: fb_.write(textwrap.dedent(''' echo >> {0}: cmd.run: @@ -132,7 +133,7 @@ class CMDRunRedirectTest(ModuleCase, SaltReturnAssertsMixin): ''' os.remove(self.test_file) state_key = 'cmd_|-echo >> {0}_|-echo >> {0}_|-run'.format(self.test_file) - with salt.utils.fopen(self.state_file, 'w') as fb_: + with salt.utils.files.fopen(self.state_file, 'w') as fb_: fb_.write(textwrap.dedent(''' echo >> {0}: cmd.run: @@ -148,7 +149,7 @@ class CMDRunRedirectTest(ModuleCase, SaltReturnAssertsMixin): test cmd.run with shell redirect ''' state_key = 'cmd_|-echo test > {0}_|-echo test > {0}_|-run'.format(self.test_file) - with salt.utils.fopen(self.state_file, 'w') as fb_: + with salt.utils.files.fopen(self.state_file, 'w') as fb_: fb_.write(textwrap.dedent(''' echo test > {0}: cmd.run @@ -179,7 +180,7 @@ class CMDRunWatchTest(ModuleCase, SaltReturnAssertsMixin): saltines_key = 'cmd_|-saltines_|-echo changed=true_|-run' biscuits_key = 'cmd_|-biscuits_|-echo biscuits_|-wait' - with salt.utils.fopen(self.state_file, 'w') as fb_: + with salt.utils.files.fopen(self.state_file, 'w') as fb_: fb_.write(textwrap.dedent(''' saltines: cmd.run: diff --git a/tests/integration/states/test_docker_container.py b/tests/integration/states/test_docker_container.py index 061bcbb060a..9f31e9de6a7 100644 --- a/tests/integration/states/test_docker_container.py +++ b/tests/integration/states/test_docker_container.py @@ -22,7 +22,8 @@ from tests.support.helpers import destructiveTest from tests.support.mixins import SaltReturnAssertsMixin # Import Salt Libs -import salt.utils +import salt.utils.files +import salt.utils.path # Import 3rd-party libs from salt.ext.six.moves import range # pylint: disable=import-error,redefined-builtin @@ -49,8 +50,8 @@ def with_random_name(func): @destructiveTest -@skipIf(not salt.utils.which('busybox'), 'Busybox not installed') -@skipIf(not salt.utils.which('dockerd'), 'Docker not installed') +@skipIf(not salt.utils.path.which('busybox'), 'Busybox not installed') +@skipIf(not salt.utils.path.which('dockerd'), 'Docker not installed') class DockerContainerTestCase(ModuleCase, SaltReturnAssertsMixin): ''' Test docker_container states @@ -80,7 +81,7 @@ class DockerContainerTestCase(ModuleCase, SaltReturnAssertsMixin): raise Exception('Failed to build image') try: - salt.utils.rm_rf(cls.image_build_rootdir) + salt.utils.files.rm_rf(cls.image_build_rootdir) except OSError as exc: if exc.errno != errno.ENOENT: raise @@ -124,7 +125,7 @@ class DockerContainerTestCase(ModuleCase, SaltReturnAssertsMixin): if name in self.run_function('docker.list_containers', all=True): self.run_function('docker.rm', [name], force=True) try: - salt.utils.rm_rf(bind_dir_host) + salt.utils.files.rm_rf(bind_dir_host) except OSError as exc: if exc.errno != errno.ENOENT: raise diff --git a/tests/integration/states/test_file.py b/tests/integration/states/test_file.py index 486ace44157..d145efdbb18 100644 --- a/tests/integration/states/test_file.py +++ b/tests/integration/states/test_file.py @@ -4,7 +4,7 @@ Tests for the file state ''' -# Import python libs +# Import Python libs from __future__ import absolute_import import errno import glob @@ -31,8 +31,11 @@ from tests.support.helpers import ( ) from tests.support.mixins import SaltReturnAssertsMixin -# Import salt libs -import salt.utils +# Import Salt libs +import salt.utils # Can be removed once normalize_mode is moved +import salt.utils.files +import salt.utils.path +import salt.utils.platform HAS_PWD = True try: @@ -47,10 +50,10 @@ except ImportError: HAS_GRP = False # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import range # pylint: disable=import-error,redefined-builtin -IS_WINDOWS = salt.utils.is_windows() +IS_WINDOWS = salt.utils.platform.is_windows() STATE_DIR = os.path.join(FILES, 'file', 'base') if IS_WINDOWS: @@ -166,7 +169,7 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin): file.absent ''' name = os.path.join(TMP, 'file_to_kill') - with salt.utils.fopen(name, 'w+') as fp_: + with salt.utils.files.fopen(name, 'w+') as fp_: fp_.write('killme') ret = self.run_state('file.absent', name=name) self.assertSaltTrueReturn(ret) @@ -212,7 +215,7 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin): file.absent test interface ''' name = os.path.join(TMP, 'file_to_kill') - with salt.utils.fopen(name, 'w+') as fp_: + with salt.utils.files.fopen(name, 'w+') as fp_: fp_.write('killme') ret = self.run_state('file.absent', test=True, name=name) try: @@ -232,9 +235,9 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin): src = os.path.join( FILES, 'file', 'base', 'grail', 'scene33' ) - with salt.utils.fopen(src, 'r') as fp_: + with salt.utils.files.fopen(src, 'r') as fp_: master_data = fp_.read() - with salt.utils.fopen(name, 'r') as fp_: + with salt.utils.files.fopen(name, 'r') as fp_: minion_data = fp_.read() self.assertEqual(master_data, minion_data) self.assertSaltTrueReturn(ret) @@ -344,7 +347,7 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin): self.run_function('state.sls', [state_file]) self.assertTrue(os.path.exists(grain_path)) - with salt.utils.fopen(grain_path, 'r') as fp_: + with salt.utils.files.fopen(grain_path, 'r') as fp_: file_contents = fp_.readlines() self.assertTrue(re.match('^minion$', file_contents[0])) @@ -426,7 +429,7 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin): file.managed test interface ''' name = os.path.join(TMP, 'grail_not_scene33') - with salt.utils.fopen(name, 'wb') as fp_: + with salt.utils.files.fopen(name, 'wb') as fp_: fp_.write(six.b('test_managed_show_changes_false\n')) ret = self.run_state( @@ -453,9 +456,9 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin): state_key = 'file_|-{0}_|-{0}_|-managed'.format(funny_file) try: - with salt.utils.fopen(funny_url_path, 'w'): + with salt.utils.files.fopen(funny_url_path, 'w'): pass - with salt.utils.fopen(state_file, 'w') as fp_: + with salt.utils.files.fopen(state_file, 'w') as fp_: fp_.write(textwrap.dedent('''\ {0}: file.managed: @@ -494,7 +497,7 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin): state_keys[typ] = 'file_|-{0} file_|-{1}_|-managed'.format(typ, managed_files[typ]) try: - with salt.utils.fopen(state_file, 'w') as fd_: + with salt.utils.files.fopen(state_file, 'w') as fd_: fd_.write(textwrap.dedent('''\ bool file: file.managed: @@ -546,9 +549,9 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin): Test file.managed passing a basic check_cmd kwarg. See Issue #38111. ''' r_group = 'root' - if salt.utils.is_darwin(): + if salt.utils.platform.is_darwin(): r_group = 'wheel' - if not salt.utils.which('visudo'): + if not salt.utils.path.which('visudo'): self.fail('sudo is missing') try: ret = self.run_state( @@ -660,14 +663,14 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin): os.makedirs(name) strayfile = os.path.join(name, 'strayfile') - with salt.utils.fopen(strayfile, 'w'): + with salt.utils.files.fopen(strayfile, 'w'): pass straydir = os.path.join(name, 'straydir') if not os.path.isdir(straydir): os.makedirs(straydir) - with salt.utils.fopen(os.path.join(straydir, 'strayfile2'), 'w'): + with salt.utils.files.fopen(os.path.join(straydir, 'strayfile2'), 'w'): pass ret = self.run_state('file.directory', name=name, clean=True) @@ -688,7 +691,7 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin): os.makedirs(name) strayfile = os.path.join(name, 'strayfile') - with salt.utils.fopen(strayfile, 'w'): + with salt.utils.files.fopen(strayfile, 'w'): pass straydir = os.path.join(name, 'straydir') @@ -696,11 +699,11 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin): os.makedirs(straydir) strayfile2 = os.path.join(straydir, 'strayfile2') - with salt.utils.fopen(strayfile2, 'w'): + with salt.utils.files.fopen(strayfile2, 'w'): pass keepfile = os.path.join(straydir, 'keepfile') - with salt.utils.fopen(keepfile, 'w'): + with salt.utils.files.fopen(keepfile, 'w'): pass exclude_pat = 'E@^straydir(|/keepfile)$' @@ -729,7 +732,7 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin): os.makedirs(name) strayfile = os.path.join(name, 'strayfile') - with salt.utils.fopen(strayfile, 'w'): + with salt.utils.files.fopen(strayfile, 'w'): pass straydir = os.path.join(name, 'straydir') @@ -737,11 +740,11 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin): os.makedirs(straydir) strayfile2 = os.path.join(straydir, 'strayfile2') - with salt.utils.fopen(strayfile2, 'w'): + with salt.utils.files.fopen(strayfile2, 'w'): pass keepfile = os.path.join(straydir, 'keepfile') - with salt.utils.fopen(keepfile, 'w'): + with salt.utils.files.fopen(keepfile, 'w'): pass exclude_pat = 'E@^straydir(|/keepfile)$' @@ -780,11 +783,11 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin): self.addCleanup(lambda: shutil.rmtree(directory)) wrong_file = os.path.join(directory, "wrong") - with salt.utils.fopen(wrong_file, "w") as fp: + with salt.utils.files.fopen(wrong_file, "w") as fp: fp.write("foo") good_file = os.path.join(directory, "bar") - with salt.utils.fopen(state_file, 'w') as fp: + with salt.utils.files.fopen(state_file, 'w') as fp: self.addCleanup(lambda: os.remove(state_file)) fp.write(textwrap.dedent('''\ some_dir: @@ -815,11 +818,11 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin): self.addCleanup(lambda: shutil.rmtree(directory)) wrong_file = os.path.join(directory, "wrong") - with salt.utils.fopen(wrong_file, "w") as fp: + with salt.utils.files.fopen(wrong_file, "w") as fp: fp.write("foo") good_file = os.path.join(directory, "bar") - with salt.utils.fopen(state_file, 'w') as fp: + with salt.utils.files.fopen(state_file, 'w') as fp: self.addCleanup(lambda: os.remove(state_file)) fp.write(textwrap.dedent('''\ some_dir: @@ -851,11 +854,11 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin): self.addCleanup(lambda: shutil.rmtree(directory)) wrong_file = os.path.join(directory, "wrong") - with salt.utils.fopen(wrong_file, "w") as fp: + with salt.utils.files.fopen(wrong_file, "w") as fp: fp.write("foo") good_file = os.path.join(directory, "bar") - with salt.utils.fopen(state_file, 'w') as fp: + with salt.utils.files.fopen(state_file, 'w') as fp: self.addCleanup(lambda: os.remove(state_file)) fp.write(textwrap.dedent('''\ some_dir: @@ -992,7 +995,7 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin): template='jinja', defaults={'spam': _ts}) try: self.assertSaltTrueReturn(ret) - with salt.utils.fopen(os.path.join(name, 'scene33'), 'r') as fp_: + with salt.utils.files.fopen(os.path.join(name, 'scene33'), 'r') as fp_: contents = fp_.read() self.assertIn(_ts, contents) finally: @@ -1006,11 +1009,11 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin): if not os.path.isdir(name): os.makedirs(name) strayfile = os.path.join(name, 'strayfile') - with salt.utils.fopen(strayfile, 'w'): + with salt.utils.files.fopen(strayfile, 'w'): pass # Corner cases: replacing file with a directory and vice versa - with salt.utils.fopen(os.path.join(name, '36'), 'w'): + with salt.utils.files.fopen(os.path.join(name, '36'), 'w'): pass os.makedirs(os.path.join(name, 'scene33')) ret = self.run_state( @@ -1031,11 +1034,11 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin): if not os.path.isdir(name): os.makedirs(name) strayfile = os.path.join(name, 'strayfile') - with salt.utils.fopen(strayfile, 'w'): + with salt.utils.files.fopen(strayfile, 'w'): pass # Corner cases: replacing file with a directory and vice versa - with salt.utils.fopen(os.path.join(name, '32'), 'w'): + with salt.utils.files.fopen(os.path.join(name, '32'), 'w'): pass os.makedirs(os.path.join(name, 'scene34')) ret = self.run_state('file.recurse', @@ -1086,14 +1089,14 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin): file.replace ''' name = os.path.join(TMP, 'replace_test') - with salt.utils.fopen(name, 'w+') as fp_: + with salt.utils.files.fopen(name, 'w+') as fp_: fp_.write('change_me') ret = self.run_state('file.replace', name=name, pattern='change', repl='salt', backup=False) try: - with salt.utils.fopen(name, 'r') as fp_: + with salt.utils.files.fopen(name, 'r') as fp_: self.assertIn('salt', fp_.read()) self.assertSaltTrueReturn(ret) @@ -1116,7 +1119,7 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin): test_name = 'test_replace_issue_18612' path_test = os.path.join(TMP, test_name) - with salt.utils.fopen(path_test, 'w+') as fp_test_: + with salt.utils.files.fopen(path_test, 'w+') as fp_test_: fp_test_.write('# en_US.UTF-8') ret = [] @@ -1126,11 +1129,11 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin): try: # ensure, the number of lines didn't change, even after invoking 'file.replace' 3 times - with salt.utils.fopen(path_test, 'r') as fp_test_: + with salt.utils.files.fopen(path_test, 'r') as fp_test_: self.assertTrue((sum(1 for _ in fp_test_) == 1)) # ensure, the replacement succeeded - with salt.utils.fopen(path_test, 'r') as fp_test_: + with salt.utils.files.fopen(path_test, 'r') as fp_test_: self.assertTrue(fp_test_.read().startswith('en_US.UTF-8')) # ensure, all runs of 'file.replace' reported success @@ -1392,7 +1395,7 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin): 'finally': 'the last item'}, formatter='json') - with salt.utils.fopen(path_test, 'r') as fp_: + with salt.utils.files.fopen(path_test, 'r') as fp_: serialized_file = fp_.read() expected_file = '''{ @@ -1473,7 +1476,7 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin): name = os.path.join(TMP, 'comment_test') try: # write a line to file - with salt.utils.fopen(name, 'w+') as fp_: + with salt.utils.files.fopen(name, 'w+') as fp_: fp_.write('comment_me') # Look for changes with test=True: return should be "None" at the first run @@ -1485,7 +1488,7 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin): # result is positive self.assertSaltTrueReturn(ret) # line is commented - with salt.utils.fopen(name, 'r') as fp_: + with salt.utils.files.fopen(name, 'r') as fp_: self.assertTrue(fp_.read().startswith('#comment')) # comment twice @@ -1494,7 +1497,7 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin): # result is still positive self.assertSaltTrueReturn(ret) # line is still commented - with salt.utils.fopen(name, 'r') as fp_: + with salt.utils.files.fopen(name, 'r') as fp_: self.assertTrue(fp_.read().startswith('#comment')) # Test previously commented file returns "True" now and not "None" with test=True @@ -1510,12 +1513,12 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin): ''' name = os.path.join(TMP, 'comment_test_test') try: - with salt.utils.fopen(name, 'w+') as fp_: + with salt.utils.files.fopen(name, 'w+') as fp_: fp_.write('comment_me') ret = self.run_state( 'file.comment', test=True, name=name, regex='.*comment.*', ) - with salt.utils.fopen(name, 'r') as fp_: + with salt.utils.files.fopen(name, 'r') as fp_: self.assertNotIn('#comment', fp_.read()) self.assertSaltNoneReturn(ret) finally: @@ -1527,10 +1530,10 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin): ''' name = os.path.join(TMP, 'uncomment_test') try: - with salt.utils.fopen(name, 'w+') as fp_: + with salt.utils.files.fopen(name, 'w+') as fp_: fp_.write('#comment_me') ret = self.run_state('file.uncomment', name=name, regex='^comment') - with salt.utils.fopen(name, 'r') as fp_: + with salt.utils.files.fopen(name, 'r') as fp_: self.assertNotIn('#comment', fp_.read()) self.assertSaltTrueReturn(ret) finally: @@ -1542,12 +1545,12 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin): ''' name = os.path.join(TMP, 'uncomment_test_test') try: - with salt.utils.fopen(name, 'w+') as fp_: + with salt.utils.files.fopen(name, 'w+') as fp_: fp_.write('#comment_me') ret = self.run_state( 'file.uncomment', test=True, name=name, regex='^comment.*' ) - with salt.utils.fopen(name, 'r') as fp_: + with salt.utils.files.fopen(name, 'r') as fp_: self.assertIn('#comment', fp_.read()) self.assertSaltNoneReturn(ret) finally: @@ -1559,10 +1562,10 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin): ''' name = os.path.join(TMP, 'append_test') try: - with salt.utils.fopen(name, 'w+') as fp_: + with salt.utils.files.fopen(name, 'w+') as fp_: fp_.write('#salty!') ret = self.run_state('file.append', name=name, text='cheese') - with salt.utils.fopen(name, 'r') as fp_: + with salt.utils.files.fopen(name, 'r') as fp_: self.assertIn('cheese', fp_.read()) self.assertSaltTrueReturn(ret) finally: @@ -1574,12 +1577,12 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin): ''' name = os.path.join(TMP, 'append_test_test') try: - with salt.utils.fopen(name, 'w+') as fp_: + with salt.utils.files.fopen(name, 'w+') as fp_: fp_.write('#salty!') ret = self.run_state( 'file.append', test=True, name=name, text='cheese' ) - with salt.utils.fopen(name, 'r') as fp_: + with salt.utils.files.fopen(name, 'r') as fp_: self.assertNotIn('cheese', fp_.read()) self.assertSaltNoneReturn(ret) finally: @@ -1743,7 +1746,7 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin): 'state.sls', mods='testappend.issue-2227' ) self.assertSaltTrueReturn(ret) - with salt.utils.fopen(tmp_file_append, 'r') as fp_: + with salt.utils.files.fopen(tmp_file_append, 'r') as fp_: contents_pre = fp_.read() # It should not append text again @@ -1752,7 +1755,7 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin): ) self.assertSaltTrueReturn(ret) - with salt.utils.fopen(tmp_file_append, 'r') as fp_: + with salt.utils.files.fopen(tmp_file_append, 'r') as fp_: contents_post = fp_.read() self.assertEqual(contents_pre, contents_post) @@ -1768,7 +1771,7 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin): if not self.run_function('cmd.has_exec', ['patch']): self.skipTest('patch is not installed') src_file = os.path.join(TMP, 'src.txt') - with salt.utils.fopen(src_file, 'w+') as fp: + with salt.utils.files.fopen(src_file, 'w+') as fp: fp.write(src) ret = self.run_state( 'file.patch', @@ -1781,7 +1784,7 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin): def test_patch(self): src_file, ret = self.do_patch() self.assertSaltTrueReturn(ret) - with salt.utils.fopen(src_file) as fp: + with salt.utils.files.fopen(src_file) as fp: self.assertEqual(fp.read(), 'Hello world\n') def test_patch_hash_mismatch(self): @@ -1801,7 +1804,7 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin): # Get a path to the temporary file tmp_file = os.path.join(TMP, 'issue-2041-comment.txt') # Write some data to it - with salt.utils.fopen(tmp_file, 'w') as fp_: + with salt.utils.files.fopen(tmp_file, 'w') as fp_: fp_.write('hello\nworld\n') # create the sls template template_lines = [ @@ -1836,7 +1839,7 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin): # Get a path to the temporary file tmp_file = os.path.join(TMP, 'issue-2379-file-append.txt') # Write some data to it - with salt.utils.fopen(tmp_file, 'w') as fp_: + with salt.utils.files.fopen(tmp_file, 'w') as fp_: fp_.write( 'hello\nworld\n' # Some junk '#PermitRootLogin yes\n' # Commented text @@ -1958,7 +1961,7 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin): " - backup: '.bak2'", ' - show_changes: True', ''] - with salt.utils.fopen(template_path, 'w') as fp_: + with salt.utils.files.fopen(template_path, 'w') as fp_: fp_.write( os.linesep.join(sls_template).format(testcase_filedest)) @@ -1966,7 +1969,7 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin): ret = self.run_function('state.sls', mods='issue-8343') for name, step in six.iteritems(ret): self.assertSaltTrueReturn({name: step}) - with salt.utils.fopen(testcase_filedest) as fp_: + with salt.utils.files.fopen(testcase_filedest) as fp_: contents = fp_.read().split(os.linesep) self.assertEqual( ['#-- start salt managed zonestart -- PLEASE, DO NOT EDIT', @@ -2025,14 +2028,14 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin): ' - show_changes: True' ] - with salt.utils.fopen(template_path, 'w') as fp_: + with salt.utils.files.fopen(template_path, 'w') as fp_: fp_.write(os.linesep.join(sls_template).format(testcase_filedest)) try: ret = self.run_function('state.sls', mods='issue-11003') for name, step in six.iteritems(ret): self.assertSaltTrueReturn({name: step}) - with salt.utils.fopen(testcase_filedest) as fp_: + with salt.utils.files.fopen(testcase_filedest) as fp_: contents = fp_.read().split(os.linesep) begin = contents.index( @@ -2057,66 +2060,62 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin): This is more generic than just a file test. Feel free to move ''' - # Get a path to the temporary file - # 한국어 시험 (korean) - # '\xed\x95\x9c\xea\xb5\xad\xec\x96\xb4 \xec\x8b\x9c\xed\x97\x98' (utf-8) - # u'\ud55c\uad6d\uc5b4 \uc2dc\ud5d8' (unicode) - korean_1 = '한국어 시험' - korean_utf8_1 = ('\xed\x95\x9c\xea\xb5\xad\xec\x96\xb4' - ' \xec\x8b\x9c\xed\x97\x98') - korean_unicode_1 = u'\ud55c\uad6d\uc5b4 \uc2dc\ud5d8' - korean_2 = '첫 번째 행' - korean_utf8_2 = '\xec\xb2\xab \xeb\xb2\x88\xec\xa7\xb8 \xed\x96\x89' - korean_unicode_2 = u'\uccab \ubc88\uc9f8 \ud589' - korean_3 = '마지막 행' - korean_utf8_3 = '\xeb\xa7\x88\xec\xa7\x80\xeb\xa7\x89 \xed\x96\x89' - korean_unicode_3 = u'\ub9c8\uc9c0\ub9c9 \ud589' - test_file = os.path.join(TMP, - 'salt_utf8_tests/'+korean_utf8_1+'.txt' + korean_1 = u'한국어 시험' + korean_2 = u'첫 번째 행' + korean_3 = u'마지막 행' + test_file = os.path.join( + TMP, + u'salt_utf8_tests', + u'{0}.txt'.format(korean_1) ) + test_file_encoded = salt.utils.stringutils.to_str(test_file) template_path = os.path.join(TMP_STATE_TREE, 'issue-8947.sls') # create the sls template template_lines = [ - '# -*- coding: utf-8 -*-', - 'some-utf8-file-create:', - ' file.managed:', - " - name: '{0}'".format(test_file), - " - contents: {0}".format(korean_utf8_1), - ' - makedirs: True', - ' - replace: True', - ' - show_diff: True', - 'some-utf8-file-create2:', - ' file.managed:', - " - name: '{0}'".format(test_file), - ' - contents: |', - ' {0}'.format(korean_utf8_2), - ' {0}'.format(korean_utf8_1), - ' {0}'.format(korean_utf8_3), - ' - replace: True', - ' - show_diff: True', - 'some-utf8-file-exists:', - ' file.exists:', - " - name: '{0}'".format(test_file), - ' - require:', - ' - file: some-utf8-file-create2', - 'some-utf8-file-content-test:', - ' cmd.run:', - ' - name: \'cat "{0}"\''.format(test_file), - ' - require:', - ' - file: some-utf8-file-exists', - 'some-utf8-file-content-remove:', - ' cmd.run:', - ' - name: \'rm -f "{0}"\''.format(test_file), - ' - require:', - ' - cmd: some-utf8-file-content-test', - 'some-utf8-file-removed:', - ' file.missing:', - " - name: '{0}'".format(test_file), - ' - require:', - ' - cmd: some-utf8-file-content-remove', + u'# -*- coding: utf-8 -*-', + u'some-utf8-file-create:', + u' file.managed:', + u" - name: '{0}'".format(test_file), + u" - contents: {0}".format(korean_1), + u' - makedirs: True', + u' - replace: True', + u' - show_diff: True', + u'some-utf8-file-create2:', + u' file.managed:', + u" - name: '{0}'".format(test_file), + u' - contents: |', + u' {0}'.format(korean_2), + u' {0}'.format(korean_1), + u' {0}'.format(korean_3), + u' - replace: True', + u' - show_diff: True', + u'some-utf8-file-exists:', + u' file.exists:', + u" - name: '{0}'".format(test_file), + u' - require:', + u' - file: some-utf8-file-create2', + u'some-utf8-file-content-test:', + u' cmd.run:', + u' - name: \'cat "{0}"\''.format(test_file), + u' - require:', + u' - file: some-utf8-file-exists', + u'some-utf8-file-content-remove:', + u' cmd.run:', + u' - name: \'rm -f "{0}"\''.format(test_file), + u' - require:', + u' - cmd: some-utf8-file-content-test', + u'some-utf8-file-removed:', + u' file.missing:', + u" - name: '{0}'".format(test_file), + u' - require:', + u' - cmd: some-utf8-file-content-remove', ] - with salt.utils.fopen(template_path, 'w') as fp_: - fp_.write(os.linesep.join(template_lines)) + with salt.utils.files.fopen(template_path, 'wb') as fp_: + fp_.write( + salt.utils.stringutils.to_bytes( + os.linesep.join(template_lines) + ) + ) try: ret = self.run_function('state.sls', mods='issue-8947') if not isinstance(ret, dict): @@ -2129,54 +2128,52 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin): utf_diff = '--- \n+++ \n@@ -1,1 +1,3 @@\n' else: utf_diff = '--- \n+++ \n@@ -1 +1,3 @@\n' - utf_diff += '+\xec\xb2\xab \xeb\xb2\x88\xec\xa7\xb8 \xed\x96\x89\n \xed\x95\x9c\xea\xb5\xad\xec\x96\xb4 \xec\x8b\x9c\xed\x97\x98\n+\xeb\xa7\x88\xec\xa7\x80\xeb\xa7\x89 \xed\x96\x89\n' + #utf_diff += '+\xec\xb2\xab \xeb\xb2\x88\xec\xa7\xb8 \xed\x96\x89\n \xed\x95\x9c\xea\xb5\xad\xec\x96\xb4 \xec\x8b\x9c\xed\x97\x98\n+\xeb\xa7\x88\xec\xa7\x80\xeb\xa7\x89 \xed\x96\x89\n' + utf_diff += salt.utils.stringutils.to_str( + u'+첫 번째 행\n' + u' 한국어 시험\n' + u'+마지막 행\n' + ) # using unicode.encode('utf-8') we should get the same as # an utf-8 string expected = { - ('file_|-some-utf8-file-create_|-{0}' - '_|-managed').format(test_file): { - 'name': '{0}'.format(test_file), + 'file_|-some-utf8-file-create_|-{0}_|-managed'.format(test_file_encoded): { + 'name': '{0}'.format(test_file_encoded), '__run_num__': 0, - 'comment': 'File {0} updated'.format(test_file), + 'comment': 'File {0} updated'.format(test_file_encoded), 'diff': 'New file' }, - ('file_|-some-utf8-file-create2_|-{0}' - '_|-managed').format(test_file): { - 'name': '{0}'.format(test_file), + 'file_|-some-utf8-file-create2_|-{0}_|-managed'.format(test_file_encoded): { + 'name': '{0}'.format(test_file_encoded), '__run_num__': 1, - 'comment': 'File {0} updated'.format(test_file), + 'comment': 'File {0} updated'.format(test_file_encoded), 'diff': utf_diff }, - ('file_|-some-utf8-file-exists_|-{0}' - '_|-exists').format(test_file): { - 'name': '{0}'.format(test_file), + 'file_|-some-utf8-file-exists_|-{0}_|-exists'.format(test_file_encoded): { + 'name': '{0}'.format(test_file_encoded), '__run_num__': 2, - 'comment': 'Path {0} exists'.format(test_file) + 'comment': 'Path {0} exists'.format(test_file_encoded) }, - ('cmd_|-some-utf8-file-content-test_|-cat "{0}"' - '_|-run').format(test_file): { - 'name': 'cat "{0}"'.format(test_file), + 'cmd_|-some-utf8-file-content-test_|-cat "{0}"_|-run'.format(test_file_encoded): { + 'name': 'cat "{0}"'.format(test_file_encoded), '__run_num__': 3, - 'comment': 'Command "cat "{0}"" run'.format(test_file), + 'comment': 'Command "cat "{0}"" run'.format(test_file_encoded), 'stdout': '{0}\n{1}\n{2}'.format( - korean_unicode_2.encode('utf-8'), - korean_unicode_1.encode('utf-8'), - korean_unicode_3.encode('utf-8') + salt.utils.stringutils.to_str(korean_2), + salt.utils.stringutils.to_str(korean_1), + salt.utils.stringutils.to_str(korean_3), ) }, - ('cmd_|-some-utf8-file-content-remove_|-rm -f "{0}"' - '_|-run').format(test_file): { - 'name': 'rm -f "{0}"'.format(test_file), + 'cmd_|-some-utf8-file-content-remove_|-rm -f "{0}"_|-run'.format(test_file_encoded): { + 'name': 'rm -f "{0}"'.format(test_file_encoded), '__run_num__': 4, - 'comment': 'Command "rm -f "{0}"" run'.format(test_file), + 'comment': 'Command "rm -f "{0}"" run'.format(test_file_encoded), 'stdout': '' }, - ('file_|-some-utf8-file-removed_|-{0}' - '_|-missing').format(test_file): { - 'name': '{0}'.format(test_file), + 'file_|-some-utf8-file-removed_|-{0}_|-missing'.format(test_file_encoded): { + 'name': '{0}'.format(test_file_encoded), '__run_num__': 5, - 'comment': - 'Path {0} is missing'.format(test_file), + 'comment': 'Path {0} is missing'.format(test_file_encoded), } } result = {} @@ -2196,11 +2193,12 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin): self.maxDiff = None self.assertEqual(expected, result) - cat_id = ('cmd_|-some-utf8-file-content-test_|-cat "{0}"' - '_|-run').format(test_file) + cat_id = 'cmd_|-some-utf8-file-content-test_|-cat "{0}"_|-run'.format(test_file_encoded) self.assertEqual( result[cat_id]['stdout'], - korean_2 + '\n' + korean_1 + '\n' + korean_3 + salt.utils.stringutils.to_str( + korean_2 + '\n' + korean_1 + '\n' + korean_3 + ) ) finally: if os.path.isdir(test_file): @@ -2247,7 +2245,7 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin): self.assertEqual(pwd.getpwuid(onestats.st_uid).pw_name, user) self.assertEqual(pwd.getpwuid(twostats.st_uid).pw_name, 'root') self.assertEqual(grp.getgrgid(onestats.st_gid).gr_name, group) - if salt.utils.which('id'): + if salt.utils.path.which('id'): root_group = self.run_function('user.primary_group', ['root']) self.assertEqual(grp.getgrgid(twostats.st_gid).gr_name, root_group) finally: @@ -2318,7 +2316,7 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin): if exc.errno != errno.EBADF: raise exc - with salt.utils.fopen(source, 'w') as fp_: + with salt.utils.files.fopen(source, 'w') as fp_: fp_.write('{{ foo }}\n') try: @@ -2347,7 +2345,7 @@ class FileTest(ModuleCase, SaltReturnAssertsMixin): if exc.errno != errno.EBADF: raise exc - with salt.utils.fopen(source, 'w') as fp_: + with salt.utils.files.fopen(source, 'w') as fp_: fp_.write('{{ foo }}\n') try: diff --git a/tests/integration/states/test_git.py b/tests/integration/states/test_git.py index 3b63a24b2d2..e4601574ae2 100644 --- a/tests/integration/states/test_git.py +++ b/tests/integration/states/test_git.py @@ -20,7 +20,8 @@ from tests.support.paths import TMP from tests.support.mixins import SaltReturnAssertsMixin # Import salt libs -import salt.utils +import salt.utils.files +import salt.utils.path from salt.utils.versions import LooseVersion as _LooseVersion @@ -32,7 +33,7 @@ def __check_git_version(caller, min_version, skip_msg): actual_setup = getattr(caller, 'setUp', None) def setUp(self, *args, **kwargs): - if not salt.utils.which('git'): + if not salt.utils.path.which('git'): self.skipTest('git is not installed') git_version = self.run_function('git.version') if _LooseVersion(git_version) < _LooseVersion(min_version): @@ -44,7 +45,7 @@ def __check_git_version(caller, min_version, skip_msg): @functools.wraps(caller) def wrapper(self, *args, **kwargs): - if not salt.utils.which('git'): + if not salt.utils.path.which('git'): self.skipTest('git is not installed') git_version = self.run_function('git.version') if _LooseVersion(git_version) < _LooseVersion(min_version): @@ -227,7 +228,7 @@ class GitTest(ModuleCase, SaltReturnAssertsMixin): self.assertTrue(os.path.isdir(os.path.join(name, '.git'))) # Make change to LICENSE file. - with salt.utils.fopen(os.path.join(name, 'LICENSE'), 'a') as fp_: + with salt.utils.files.fopen(os.path.join(name, 'LICENSE'), 'a') as fp_: fp_.write('Lorem ipsum dolor blah blah blah....\n') # Make sure that we now have uncommitted changes @@ -295,7 +296,7 @@ class GitTest(ModuleCase, SaltReturnAssertsMixin): # Make a change to the repo by editing the file in the admin copy # of the repo and committing. head_pre = _head(admin_dir) - with salt.utils.fopen(os.path.join(admin_dir, 'LICENSE'), 'a') as fp_: + with salt.utils.files.fopen(os.path.join(admin_dir, 'LICENSE'), 'a') as fp_: fp_.write('Hello world!') self.run_function( 'git.commit', [admin_dir, 'added a line'], @@ -342,7 +343,7 @@ class GitTest(ModuleCase, SaltReturnAssertsMixin): # Check out a new branch in the clone and make a commit, to ensure # that when we re-run the state, it is not a fast-forward change self.run_function('git.checkout', [name, 'new_branch'], opts='-b') - with salt.utils.fopen(os.path.join(name, 'foo'), 'w'): + with salt.utils.files.fopen(os.path.join(name, 'foo'), 'w'): pass self.run_function('git.add', [name, '.']) self.run_function( @@ -405,7 +406,7 @@ class GitTest(ModuleCase, SaltReturnAssertsMixin): try: # Add and commit a file - with salt.utils.fopen(os.path.join(name, 'foo.txt'), 'w') as fp_: + with salt.utils.files.fopen(os.path.join(name, 'foo.txt'), 'w') as fp_: fp_.write('Hello world\n') self.run_function('git.add', [name, '.']) self.run_function( @@ -422,7 +423,7 @@ class GitTest(ModuleCase, SaltReturnAssertsMixin): self.assertSaltTrueReturn(ret) # Add another commit - with salt.utils.fopen(os.path.join(name, 'foo.txt'), 'w') as fp_: + with salt.utils.files.fopen(os.path.join(name, 'foo.txt'), 'w') as fp_: fp_.write('Added a line\n') self.run_function( 'git.commit', [name, 'added a line'], @@ -472,7 +473,7 @@ class GitTest(ModuleCase, SaltReturnAssertsMixin): try: fname = os.path.join(name, 'stoptheprocess') - with salt.utils.fopen(fname, 'a') as fh_: + with salt.utils.files.fopen(fname, 'a') as fh_: fh_.write('') ret = self.run_state( @@ -542,7 +543,7 @@ class LocalRepoGitTest(ModuleCase, SaltReturnAssertsMixin): # Clone bare repo self.run_function('git.clone', [admin], url=repo) # Create, add, commit, and push file - with salt.utils.fopen(os.path.join(admin, 'foo'), 'w'): + with salt.utils.files.fopen(os.path.join(admin, 'foo'), 'w'): pass self.run_function('git.add', [admin, '.']) self.run_function( diff --git a/tests/integration/states/test_host.py b/tests/integration/states/test_host.py index ab662e61bec..41d69a006c2 100644 --- a/tests/integration/states/test_host.py +++ b/tests/integration/states/test_host.py @@ -14,7 +14,7 @@ from tests.support.paths import FILES, TMP from tests.support.mixins import SaltReturnAssertsMixin # Import salt libs -import salt.utils +import salt.utils.files HFILE = os.path.join(TMP, 'hosts') @@ -41,6 +41,6 @@ class HostTest(ModuleCase, SaltReturnAssertsMixin): ip = '10.10.10.10' ret = self.run_state('host.present', name=name, ip=ip) self.assertSaltTrueReturn(ret) - with salt.utils.fopen(HFILE) as fp_: + with salt.utils.files.fopen(HFILE) as fp_: output = fp_.read() self.assertIn('{0}\t\t{1}'.format(ip, name), output) diff --git a/tests/integration/states/test_match.py b/tests/integration/states/test_match.py index d2c7ac2f114..da93fb5a80b 100644 --- a/tests/integration/states/test_match.py +++ b/tests/integration/states/test_match.py @@ -17,7 +17,7 @@ from tests.support.paths import FILES from tests.support.helpers import skip_if_not_root # Import salt libs -import salt.utils +import salt.utils.files STATE_DIR = os.path.join(FILES, 'file', 'base') @@ -34,7 +34,7 @@ class StateMatchTest(ModuleCase): top_filename = 'issue-2167-ipcidr-match.sls' top_file = os.path.join(STATE_DIR, top_filename) try: - with salt.utils.fopen(top_file, 'w') as fp_: + with salt.utils.files.fopen(top_file, 'w') as fp_: fp_.write( 'base:\n' ' {0}:\n' diff --git a/tests/integration/states/test_mysql.py b/tests/integration/states/test_mysql.py index fc62d09095b..0b3cdef4123 100644 --- a/tests/integration/states/test_mysql.py +++ b/tests/integration/states/test_mysql.py @@ -14,8 +14,8 @@ from tests.support.helpers import destructiveTest from tests.support.mixins import SaltReturnAssertsMixin # Import salt libs -import salt.utils -import salt.ext.six as six +import salt.utils.path +from salt.ext import six from salt.modules import mysql as mysqlmod log = logging.getLogger(__name__) @@ -26,7 +26,7 @@ try: except ImportError: NO_MYSQL = True -if not salt.utils.which('mysqladmin'): +if not salt.utils.path.which('mysqladmin'): NO_MYSQL = True diff --git a/tests/integration/states/test_npm.py b/tests/integration/states/test_npm.py index 6aad2a6094b..d93825b593d 100644 --- a/tests/integration/states/test_npm.py +++ b/tests/integration/states/test_npm.py @@ -14,10 +14,14 @@ from tests.support.helpers import destructiveTest, requires_network from tests.support.mixins import SaltReturnAssertsMixin # Import salt libs -import salt.utils +import salt.modules.cmdmod as cmd +import salt.utils.path +from salt.utils.versions import LooseVersion + +MAX_NPM_VERSION = '5.0.0' -@skipIf(salt.utils.which('npm') is None, 'npm not installed') +@skipIf(salt.utils.path.which('npm') is None, 'npm not installed') class NpmStateTest(ModuleCase, SaltReturnAssertsMixin): @requires_network() @@ -38,7 +42,7 @@ class NpmStateTest(ModuleCase, SaltReturnAssertsMixin): ''' Determine if URL-referenced NPM module can be successfully installed. ''' - ret = self.run_state('npm.installed', name='git://github.com/request/request') + ret = self.run_state('npm.installed', name='request/request#v2.81.1') self.assertSaltTrueReturn(ret) ret = self.run_state('npm.removed', name='git://github.com/request/request') self.assertSaltTrueReturn(ret) @@ -50,13 +54,15 @@ class NpmStateTest(ModuleCase, SaltReturnAssertsMixin): Basic test to determine if NPM module successfully installs multiple packages. ''' - ret = self.run_state('npm.installed', name=None, pkgs=['pm2', 'grunt']) + ret = self.run_state('npm.installed', name='unused', pkgs=['pm2', 'grunt']) self.assertSaltTrueReturn(ret) + @skipIf(salt.utils.path.which('npm') and LooseVersion(cmd.run('npm -v')) >= LooseVersion(MAX_NPM_VERSION), + 'Skip with npm >= 5.0.0 until #41770 is fixed') @destructiveTest def test_npm_cache_clean(self): ''' Basic test to determine if NPM successfully cleans its cached packages. ''' - ret = self.run_state('npm.cache_cleaned', name=None, force=True) + ret = self.run_state('npm.cache_cleaned', name='unused', force=True) self.assertSaltTrueReturn(ret) diff --git a/tests/integration/states/test_pip.py b/tests/integration/states/test_pip.py index 43cb50a24f7..5cd95d4788d 100644 --- a/tests/integration/states/test_pip.py +++ b/tests/integration/states/test_pip.py @@ -26,13 +26,15 @@ from tests.support.helpers import ( skip_if_not_root ) # Import salt libs -from tests.support.case import ModuleCase -import salt.utils +import salt.utils.files +import salt.utils.path +import salt.utils.versions from salt.modules.virtualenv_mod import KNOWN_BINARY_NAMES from salt.exceptions import CommandExecutionError +from tests.support.case import ModuleCase # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six class VirtualEnv(object): @@ -49,7 +51,7 @@ class VirtualEnv(object): shutil.rmtree(self.venv_dir) -@skipIf(salt.utils.which_bin(KNOWN_BINARY_NAMES) is None, 'virtualenv not installed') +@skipIf(salt.utils.path.which_bin(KNOWN_BINARY_NAMES) is None, 'virtualenv not installed') class PipStateTest(ModuleCase, SaltReturnAssertsMixin): @skip_if_not_root @@ -237,7 +239,7 @@ class PipStateTest(ModuleCase, SaltReturnAssertsMixin): ) except (AssertionError, CommandExecutionError): pip_version = self.run_function('pip.version', [venv_dir]) - if salt.utils.compare_versions(ver1=pip_version, oper='>=', ver2='7.0.0'): + if salt.utils.versions.compare(ver1=pip_version, oper='>=', ver2='7.0.0'): self.skipTest('the --mirrors arg has been deprecated and removed in pip==7.0.0') finally: if os.path.isdir(venv_dir): @@ -293,7 +295,7 @@ class PipStateTest(ModuleCase, SaltReturnAssertsMixin): req_filename = os.path.join( RUNTIME_VARS.TMP_STATE_TREE, 'issue-6912-requirements.txt' ) - with salt.utils.fopen(req_filename, 'wb') as reqf: + with salt.utils.files.fopen(req_filename, 'wb') as reqf: reqf.write(six.b('pep8')) try: @@ -360,7 +362,7 @@ class PipStateTest(ModuleCase, SaltReturnAssertsMixin): req_filename = os.path.join( RUNTIME_VARS.TMP_STATE_TREE, 'issue-6912-requirements.txt' ) - with salt.utils.fopen(req_filename, 'wb') as reqf: + with salt.utils.files.fopen(req_filename, 'wb') as reqf: reqf.write(six.b('pep8')) try: @@ -455,7 +457,7 @@ class PipStateTest(ModuleCase, SaltReturnAssertsMixin): requirements_file = os.path.join( RUNTIME_VARS.TMP_PRODENV_STATE_TREE, 'prod-env-requirements.txt' ) - with salt.utils.fopen(requirements_file, 'wb') as reqf: + with salt.utils.files.fopen(requirements_file, 'wb') as reqf: reqf.write(six.b('pep8\n')) try: diff --git a/tests/integration/states/test_pkg.py b/tests/integration/states/test_pkg.py index f5e1be2bbc3..27103072c7f 100644 --- a/tests/integration/states/test_pkg.py +++ b/tests/integration/states/test_pkg.py @@ -3,7 +3,7 @@ ''' tests for pkg state ''' -# Import python libs +# Import Python libs from __future__ import absolute_import import logging import os @@ -20,8 +20,9 @@ from tests.support.helpers import ( flaky ) -# Import salt libs -import salt.utils +# Import Salt libs +import salt.utils.path +import salt.utils.platform # Import 3rd-party libs from salt.ext.six.moves import range # pylint: disable=import-error,redefined-builtin @@ -100,7 +101,7 @@ def pkgmgr_avail(run_function, grains): Return True if any locks are found for path ''' # Try lsof if it's available - if salt.utils.which('lsof'): + if salt.utils.path.which('lsof'): lock = run_function('cmd.run', ['lsof {0}'.format(path)]) return True if len(lock) else False @@ -160,7 +161,7 @@ class PkgTest(ModuleCase, SaltReturnAssertsMixin): self.run_function('pkg.refresh_db') __testcontext__['refresh'] = True - @skipIf(salt.utils.is_windows(), 'minion is windows') + @skipIf(salt.utils.platform.is_windows(), 'minion is windows') @requires_system_grains def test_pkg_001_installed(self, grains=None): ''' @@ -191,7 +192,7 @@ class PkgTest(ModuleCase, SaltReturnAssertsMixin): ret = self.run_state('pkg.removed', name=target) self.assertSaltTrueReturn(ret) - @skipIf(salt.utils.is_windows(), 'minion is windows') + @skipIf(salt.utils.platform.is_windows(), 'minion is windows') @requires_system_grains def test_pkg_002_installed_with_version(self, grains=None): ''' @@ -239,7 +240,7 @@ class PkgTest(ModuleCase, SaltReturnAssertsMixin): ret = self.run_state('pkg.removed', name=target) self.assertSaltTrueReturn(ret) - @skipIf(salt.utils.is_windows(), 'minion is windows') + @skipIf(salt.utils.platform.is_windows(), 'minion is windows') @requires_system_grains def test_pkg_003_installed_multipkg(self, grains=None): ''' @@ -271,7 +272,7 @@ class PkgTest(ModuleCase, SaltReturnAssertsMixin): ret = self.run_state('pkg.removed', name=None, pkgs=pkg_targets) self.assertSaltTrueReturn(ret) - @skipIf(salt.utils.is_windows(), 'minion is windows') + @skipIf(salt.utils.platform.is_windows(), 'minion is windows') @requires_system_grains def test_pkg_004_installed_multipkg_with_version(self, grains=None): ''' @@ -320,7 +321,7 @@ class PkgTest(ModuleCase, SaltReturnAssertsMixin): ret = self.run_state('pkg.removed', name=None, pkgs=pkg_targets) self.assertSaltTrueReturn(ret) - @skipIf(salt.utils.is_windows(), 'minion is windows') + @skipIf(salt.utils.platform.is_windows(), 'minion is windows') @requires_system_grains def test_pkg_005_installed_32bit(self, grains=None): ''' @@ -357,7 +358,7 @@ class PkgTest(ModuleCase, SaltReturnAssertsMixin): ret = self.run_state('pkg.removed', name=target) self.assertSaltTrueReturn(ret) - @skipIf(salt.utils.is_windows(), 'minion is windows') + @skipIf(salt.utils.platform.is_windows(), 'minion is windows') @requires_system_grains def test_pkg_006_installed_32bit_with_version(self, grains=None): ''' @@ -405,7 +406,7 @@ class PkgTest(ModuleCase, SaltReturnAssertsMixin): ret = self.run_state('pkg.removed', name=target) self.assertSaltTrueReturn(ret) - @skipIf(salt.utils.is_windows(), 'minion is windows') + @skipIf(salt.utils.platform.is_windows(), 'minion is windows') @requires_system_grains def test_pkg_007_with_dot_in_pkgname(self, grains=None): ''' @@ -436,7 +437,7 @@ class PkgTest(ModuleCase, SaltReturnAssertsMixin): ret = self.run_state('pkg.removed', name=target) self.assertSaltTrueReturn(ret) - @skipIf(salt.utils.is_windows(), 'minion is windows') + @skipIf(salt.utils.platform.is_windows(), 'minion is windows') @requires_system_grains def test_pkg_008_epoch_in_version(self, grains=None): ''' @@ -470,7 +471,7 @@ class PkgTest(ModuleCase, SaltReturnAssertsMixin): ret = self.run_state('pkg.removed', name=target) self.assertSaltTrueReturn(ret) - @skipIf(salt.utils.is_windows(), 'minion is windows') + @skipIf(salt.utils.platform.is_windows(), 'minion is windows') def test_pkg_009_latest_with_epoch(self): ''' This tests for the following issue: @@ -487,7 +488,7 @@ class PkgTest(ModuleCase, SaltReturnAssertsMixin): refresh=False) self.assertSaltTrueReturn(ret) - @skipIf(salt.utils.is_windows(), 'minion is windows') + @skipIf(salt.utils.platform.is_windows(), 'minion is windows') @requires_salt_modules('pkg.info_installed') def test_pkg_010_latest_with_epoch_and_info_installed(self): ''' @@ -507,7 +508,7 @@ class PkgTest(ModuleCase, SaltReturnAssertsMixin): ret = self.run_function('pkg.info_installed', [package]) self.assertTrue(pkgquery in str(ret)) - @skipIf(salt.utils.is_windows(), 'minion is windows') + @skipIf(salt.utils.platform.is_windows(), 'minion is windows') @requires_system_grains def test_pkg_011_latest(self, grains=None): ''' @@ -539,7 +540,7 @@ class PkgTest(ModuleCase, SaltReturnAssertsMixin): ret = self.run_state('pkg.removed', name=target) self.assertSaltTrueReturn(ret) - @skipIf(salt.utils.is_windows(), 'minion is windows') + @skipIf(salt.utils.platform.is_windows(), 'minion is windows') @requires_system_grains def test_pkg_012_latest_only_upgrade(self, grains=None): ''' @@ -725,6 +726,43 @@ class PkgTest(ModuleCase, SaltReturnAssertsMixin): ret = self.run_state('pkg.removed', name=target) self.assertSaltTrueReturn(ret) + def test_pkg_014_installed_missing_release(self, grains=None): # pylint: disable=unused-argument + ''' + Tests that a version number missing the release portion still resolves + as correctly installed. For example, version 2.0.2 instead of 2.0.2-1.el7 + ''' + os_family = grains.get('os_family', '') + + if os_family.lower() != 'redhat': + self.skipTest('Test only runs on RedHat OS family') + + pkg_targets = _PKG_TARGETS.get(os_family, []) + + # Make sure that we have targets that match the os_family. If this + # fails then the _PKG_TARGETS dict above needs to have an entry added, + # with two packages that are not installed before these tests are run + self.assertTrue(pkg_targets) + + target = pkg_targets[0] + version = self.run_function('pkg.version', [target]) + + # If this assert fails, we need to find new targets, this test needs to + # be able to test successful installation of packages, so this package + # needs to not be installed before we run the states below + self.assertFalse(version) + + ret = self.run_state( + 'pkg.installed', + name=target, + version=salt.utils.str_version_to_evr(version)[1], + refresh=False, + ) + self.assertSaltTrueReturn(ret) + + # Clean up + ret = self.run_state('pkg.removed', name=target) + self.assertSaltTrueReturn(ret) + @requires_salt_modules('pkg.group_install') @requires_system_grains def test_group_installed_handle_missing_package_group(self, grains=None): # pylint: disable=unused-argument diff --git a/tests/integration/states/test_pkgrepo.py b/tests/integration/states/test_pkgrepo.py index eeef55140b8..9c4ece89d6f 100644 --- a/tests/integration/states/test_pkgrepo.py +++ b/tests/integration/states/test_pkgrepo.py @@ -15,11 +15,11 @@ from tests.support.helpers import ( requires_system_grains ) -# Import salt libs -import salt.utils +# Import Salt libs +import salt.utils.platform # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six class PkgrepoTest(ModuleCase, SaltReturnAssertsMixin): @@ -27,7 +27,7 @@ class PkgrepoTest(ModuleCase, SaltReturnAssertsMixin): pkgrepo state tests ''' @destructiveTest - @skipIf(salt.utils.is_windows(), 'minion is windows') + @skipIf(salt.utils.platform.is_windows(), 'minion is windows') @requires_system_grains def test_pkgrepo_01_managed(self, grains): ''' @@ -57,7 +57,7 @@ class PkgrepoTest(ModuleCase, SaltReturnAssertsMixin): self.assertSaltTrueReturn(dict([(state_id, state_result)])) @destructiveTest - @skipIf(salt.utils.is_windows(), 'minion is windows') + @skipIf(salt.utils.platform.is_windows(), 'minion is windows') def test_pkgrepo_02_absent(self): ''' This is a destructive test as it removes the repository added in the diff --git a/tests/integration/states/test_renderers.py b/tests/integration/states/test_renderers.py index c7e916e8e3e..256aabd230e 100644 --- a/tests/integration/states/test_renderers.py +++ b/tests/integration/states/test_renderers.py @@ -21,3 +21,12 @@ class TestJinjaRenderer(ModuleCase): ret = self.run_function('state.sls', ['jinja_dot_notation']) for state_ret in ret.values(): self.assertTrue(state_ret['result']) + + def test_salt_contains_function(self): + ''' + Test if we are able to check if a function exists inside the "salt" + wrapper (AliasLoader) which is available on Jinja templates. + ''' + ret = self.run_function('state.sls', ['jinja_salt_contains_function']) + for state_ret in ret.values(): + self.assertTrue(state_ret['result']) diff --git a/tests/integration/states/test_service.py b/tests/integration/states/test_service.py index ec50bad4d8e..955c009f030 100644 --- a/tests/integration/states/test_service.py +++ b/tests/integration/states/test_service.py @@ -12,14 +12,14 @@ from tests.support.helpers import destructiveTest from tests.support.mixins import SaltReturnAssertsMixin # Import salt libs -import salt.utils +import salt.utils.path INIT_DELAY = 5 SERVICE_NAME = 'crond' @destructiveTest -@skipIf(salt.utils.which('crond') is None, 'crond not installed') +@skipIf(salt.utils.path.which('crond') is None, 'crond not installed') class ServiceTest(ModuleCase, SaltReturnAssertsMixin): ''' Validate the service state diff --git a/tests/integration/states/test_ssh.py b/tests/integration/states/test_ssh.py index 2d4070079ec..e09c6ad6648 100644 --- a/tests/integration/states/test_ssh.py +++ b/tests/integration/states/test_ssh.py @@ -20,7 +20,7 @@ from tests.support.helpers import ( ) # Import salt libs -import salt.utils +import salt.utils.files KNOWN_HOSTS = os.path.join(RUNTIME_VARS.TMP, 'known_hosts') GITHUB_FINGERPRINT = '9d:38:5b:83:a9:17:52:92:56:1a:5e:c4:d4:81:8e:0a:ca:51:a2:64:f1:74:20:11:2e:f8:8a:c3:a1:39:49:8f' @@ -188,7 +188,7 @@ class SSHAuthStateTests(ModuleCase, SaltReturnAssertsMixin): self.assertSaltStateChangesEqual( ret, {'AAAAB3NzaC1kcQ9J5bYTEyZ==': 'New'} ) - with salt.utils.fopen(authorized_keys_file, 'r') as fhr: + with salt.utils.files.fopen(authorized_keys_file, 'r') as fhr: self.assertEqual( fhr.read(), 'ssh-rsa AAAAB3NzaC1kc3MAAACBAL0sQ9fJ5bYTEyY== root\n' @@ -206,13 +206,13 @@ class SSHAuthStateTests(ModuleCase, SaltReturnAssertsMixin): key_fname = 'issue_10198.id_rsa.pub' # Create the keyfile that we expect to get back on the state call - with salt.utils.fopen(os.path.join(RUNTIME_VARS.TMP_PRODENV_STATE_TREE, key_fname), 'w') as kfh: + with salt.utils.files.fopen(os.path.join(RUNTIME_VARS.TMP_PRODENV_STATE_TREE, key_fname), 'w') as kfh: kfh.write( 'ssh-rsa AAAAB3NzaC1kcQ9J5bYTEyZ== {0}\n'.format(username) ) # Create a bogus key file on base environment - with salt.utils.fopen(os.path.join(RUNTIME_VARS.TMP_STATE_TREE, key_fname), 'w') as kfh: + with salt.utils.files.fopen(os.path.join(RUNTIME_VARS.TMP_STATE_TREE, key_fname), 'w') as kfh: kfh.write( 'ssh-rsa BAAAB3NzaC1kcQ9J5bYTEyZ== {0}\n'.format(username) ) @@ -226,7 +226,7 @@ class SSHAuthStateTests(ModuleCase, SaltReturnAssertsMixin): comment=username ) self.assertSaltTrueReturn(ret) - with salt.utils.fopen(authorized_keys_file, 'r') as fhr: + with salt.utils.files.fopen(authorized_keys_file, 'r') as fhr: self.assertEqual( fhr.read(), 'ssh-rsa AAAAB3NzaC1kcQ9J5bYTEyZ== {0}\n'.format(username) @@ -244,7 +244,7 @@ class SSHAuthStateTests(ModuleCase, SaltReturnAssertsMixin): saltenv='prod' ) self.assertSaltTrueReturn(ret) - with salt.utils.fopen(authorized_keys_file, 'r') as fhr: + with salt.utils.files.fopen(authorized_keys_file, 'r') as fhr: self.assertEqual( fhr.read(), 'ssh-rsa AAAAB3NzaC1kcQ9J5bYTEyZ== {0}\n'.format(username) diff --git a/tests/integration/states/test_supervisord.py b/tests/integration/states/test_supervisord.py index 5593617321c..c4192a627ae 100644 --- a/tests/integration/states/test_supervisord.py +++ b/tests/integration/states/test_supervisord.py @@ -17,16 +17,16 @@ from tests.support.paths import TMP from tests.support.mixins import SaltReturnAssertsMixin # Import salt libs -import salt.utils +import salt.utils.path from salt.modules.virtualenv_mod import KNOWN_BINARY_NAMES # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six @skipIf(six.PY3, 'supervisor does not work under python 3') -@skipIf(salt.utils.which_bin(KNOWN_BINARY_NAMES) is None, 'virtualenv not installed') -@skipIf(salt.utils.which('supervisorctl') is None, 'supervisord not installed') +@skipIf(salt.utils.path.which_bin(KNOWN_BINARY_NAMES) is None, 'virtualenv not installed') +@skipIf(salt.utils.path.which('supervisorctl') is None, 'supervisord not installed') class SupervisordTest(ModuleCase, SaltReturnAssertsMixin): ''' Validate the supervisord states. diff --git a/tests/integration/states/test_user.py b/tests/integration/states/test_user.py index 1317f12f970..9ec182f6c50 100644 --- a/tests/integration/states/test_user.py +++ b/tests/integration/states/test_user.py @@ -7,7 +7,7 @@ user present user present with custom homedir ''' -# Import python libs +# Import Python libs from __future__ import absolute_import import os import sys @@ -20,10 +20,10 @@ from tests.support.unit import skipIf from tests.support.helpers import destructiveTest, requires_system_grains, skip_if_not_root from tests.support.mixins import SaltReturnAssertsMixin -# Import salt libs -import salt.utils +# Import Salt libs +import salt.utils.platform -if salt.utils.is_darwin(): +if salt.utils.platform.is_darwin(): USER = 'macuser' GROUP = 'macuser' GID = randint(400, 500) @@ -45,7 +45,7 @@ class UserTest(ModuleCase, SaltReturnAssertsMixin): user_home = '/var/lib/salt_test' def setUp(self): - if salt.utils.is_darwin(): + if salt.utils.platform.is_darwin(): #on mac we need to add user, because there is #no creationtime for nobody user. add_user = self.run_function('user.add', [USER], gid=GID) @@ -84,7 +84,7 @@ class UserTest(ModuleCase, SaltReturnAssertsMixin): And then destroys that user. Assume that it will break any system you run it on. ''' - if salt.utils.is_darwin(): + if salt.utils.platform.is_darwin(): HOMEDIR = '/Users/home_of_' + self.user_name else: HOMEDIR = '/home/home_of_' + self.user_name @@ -104,7 +104,7 @@ class UserTest(ModuleCase, SaltReturnAssertsMixin): ret = self.run_state('user.present', name=self.user_name, home=self.user_home) self.assertSaltTrueReturn(ret) - if not salt.utils.is_darwin(): + if not salt.utils.platform.is_darwin(): self.assertTrue(os.path.isdir(self.user_home)) @requires_system_grains @@ -128,7 +128,7 @@ class UserTest(ModuleCase, SaltReturnAssertsMixin): self.assertReturnNonEmptySaltType(ret) group_name = grp.getgrgid(ret['gid']).gr_name - if not salt.utils.is_darwin(): + if not salt.utils.platform.is_darwin(): self.assertTrue(os.path.isdir(self.user_home)) if grains['os_family'] in ('Suse',): self.assertEqual(group_name, 'users') @@ -153,7 +153,7 @@ class UserTest(ModuleCase, SaltReturnAssertsMixin): self.assertReturnNonEmptySaltType(ret) group_name = grp.getgrgid(ret['gid']).gr_name - if not salt.utils.is_darwin(): + if not salt.utils.platform.is_darwin(): self.assertTrue(os.path.isdir(self.user_home)) self.assertEqual(group_name, self.user_name) ret = self.run_state('user.absent', name=self.user_name) @@ -229,13 +229,13 @@ class UserTest(ModuleCase, SaltReturnAssertsMixin): self.assertReturnNonEmptySaltType(ret) self.assertEqual('', ret['fullname']) # MacOS does not supply the following GECOS fields - if not salt.utils.is_darwin(): + if not salt.utils.platform.is_darwin(): self.assertEqual('', ret['roomnumber']) self.assertEqual('', ret['workphone']) self.assertEqual('', ret['homephone']) def tearDown(self): - if salt.utils.is_darwin(): + if salt.utils.platform.is_darwin(): check_user = self.run_function('user.list_users') if USER in check_user: del_user = self.run_function('user.delete', [USER], remove=True) diff --git a/tests/integration/states/test_virtualenv.py b/tests/integration/states/test_virtualenv.py index a8cc2c246b1..3d2d7aa4da3 100644 --- a/tests/integration/states/test_virtualenv.py +++ b/tests/integration/states/test_virtualenv.py @@ -7,7 +7,7 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ''' -# Import python libs +# Import Python libs from __future__ import absolute_import import os import shutil @@ -19,12 +19,14 @@ from tests.support.helpers import destructiveTest, skip_if_not_root from tests.support.mixins import SaltReturnAssertsMixin from tests.support.runtests import RUNTIME_VARS -# Import salt libs -import salt.utils +# Import Salt libs +import salt.utils.files +import salt.utils.path +import salt.utils.platform from salt.modules.virtualenv_mod import KNOWN_BINARY_NAMES -@skipIf(salt.utils.which_bin(KNOWN_BINARY_NAMES) is None, 'virtualenv not installed') +@skipIf(salt.utils.path.which_bin(KNOWN_BINARY_NAMES) is None, 'virtualenv not installed') class VirtualenvTest(ModuleCase, SaltReturnAssertsMixin): @destructiveTest @skip_if_not_root @@ -34,7 +36,7 @@ class VirtualenvTest(ModuleCase, SaltReturnAssertsMixin): uinfo = self.run_function('user.info', [user]) - if salt.utils.is_darwin(): + if salt.utils.platform.is_darwin(): # MacOS does not support createhome with user.present self.assertSaltTrueReturn(self.run_state('file.directory', name=uinfo['home'], user=user, group=uinfo['groups'][0], dir_mode=755)) @@ -78,7 +80,7 @@ class VirtualenvTest(ModuleCase, SaltReturnAssertsMixin): ] # Let's populate the requirements file, just pep-8 for now - with salt.utils.fopen(requirements_file_path, 'a') as fhw: + with salt.utils.files.fopen(requirements_file_path, 'a') as fhw: fhw.write('pep8==1.3.3\n') # Let's run our state!!! @@ -106,7 +108,7 @@ class VirtualenvTest(ModuleCase, SaltReturnAssertsMixin): self.assertNotIn('zope.interface==4.0.1', ret) # Now let's update the requirements file, which is now cached. - with salt.utils.fopen(requirements_file_path, 'w') as fhw: + with salt.utils.files.fopen(requirements_file_path, 'w') as fhw: fhw.write('zope.interface==4.0.1\n') # Let's run our state!!! diff --git a/tests/integration/states/test_zookeeper.py b/tests/integration/states/test_zookeeper.py index 262cb6fcb97..aa611c806b8 100644 --- a/tests/integration/states/test_zookeeper.py +++ b/tests/integration/states/test_zookeeper.py @@ -13,7 +13,7 @@ from tests.support.helpers import destructiveTest from tests.support.mixins import SaltReturnAssertsMixin # Import Salt Libs -import salt.utils +import salt.utils.path try: import kazoo # pylint: disable=import-error,unused-import @@ -23,7 +23,7 @@ except ImportError: @destructiveTest -@skipIf(not salt.utils.which('dockerd'), 'Docker not installed') +@skipIf(not salt.utils.path.which('dockerd'), 'Docker not installed') @skipIf(not HAS_KAZOO, 'kazoo python library not installed') class ZookeeperTestCase(ModuleCase, SaltReturnAssertsMixin): ''' diff --git a/tests/integration/utils/testprogram.py b/tests/integration/utils/testprogram.py index c90c4062423..80d2b9128d6 100644 --- a/tests/integration/utils/testprogram.py +++ b/tests/integration/utils/testprogram.py @@ -21,11 +21,11 @@ import time import yaml -import salt.utils +import salt.utils.files import salt.utils.process import salt.utils.psutil_compat as psutils import salt.defaults.exitcodes as exitcodes -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import range # pylint: disable=import-error,redefined-builtin from tests.support.unit import TestCase @@ -210,7 +210,7 @@ class TestProgram(six.with_metaclass(TestProgramMeta, object)): if not config: return cpath = self.abs_path(self.config_file_get(config)) - with salt.utils.fopen(cpath, 'w') as cfo: + with salt.utils.files.fopen(cpath, 'w') as cfo: cfg = self.config_stringify(config) log.debug('Writing configuration for {0} to {1}:\n{2}'.format(self.name, cpath, cfg)) cfo.write(cfg) @@ -660,7 +660,7 @@ class TestSaltProgram(six.with_metaclass(TestSaltProgramMeta, TestProgram)): '''Generate the script file that calls python objects and libraries.''' lines = [] script_source = os.path.join(CODE_DIR, 'scripts', self.script) - with salt.utils.fopen(script_source, 'r') as sso: + with salt.utils.files.fopen(script_source, 'r') as sso: lines.extend(sso.readlines()) if lines[0].startswith('#!'): lines.pop(0) @@ -668,7 +668,7 @@ class TestSaltProgram(six.with_metaclass(TestSaltProgramMeta, TestProgram)): script_path = self.abs_path(os.path.join(self.script_dir, self.script)) log.debug('Installing "{0}" to "{1}"'.format(script_source, script_path)) - with salt.utils.fopen(script_path, 'w') as sdo: + with salt.utils.files.fopen(script_path, 'w') as sdo: sdo.write(''.join(lines)) sdo.flush() @@ -773,13 +773,23 @@ class TestDaemon(TestProgram): if six.PY3: cmdline = ' '.join(cmdline) for proc in psutils.process_iter(): - for item in proc.cmdline(): - if cmdline in item: - ret.append(proc) + try: + for item in proc.cmdline(): + if cmdline in item: + ret.append(proc) + except psutils.NoSuchProcess: + # Process exited between when process_iter was invoked and + # when we tried to invoke this instance's cmdline() func. + continue else: cmd_len = len(cmdline) for proc in psutils.process_iter(): - proc_cmdline = proc.cmdline() + try: + proc_cmdline = proc.cmdline() + except psutils.NoSuchProcess: + # Process exited between when process_iter was invoked and + # when we tried to invoke this instance's cmdline() func. + continue if any((cmdline == proc_cmdline[n:n + cmd_len]) for n in range(len(proc_cmdline) - cmd_len + 1)): ret.append(proc) diff --git a/tests/integration/wheel/test_client.py b/tests/integration/wheel/test_client.py index 2fab5c72f4a..7b6fe3f58eb 100644 --- a/tests/integration/wheel/test_client.py +++ b/tests/integration/wheel/test_client.py @@ -10,7 +10,7 @@ from tests.support.mixins import AdaptedConfigurationTestCaseMixin # Import Salt libs import salt.auth import salt.wheel -import salt.utils +import salt.utils.platform class WheelModuleTest(TestCase, AdaptedConfigurationTestCaseMixin): @@ -81,7 +81,7 @@ class WheelModuleTest(TestCase, AdaptedConfigurationTestCaseMixin): # Remove this skipIf when Issue #39616 is resolved # https://github.com/saltstack/salt/issues/39616 - @skipIf(salt.utils.is_windows(), + @skipIf(salt.utils.platform.is_windows(), 'Causes pickling error on Windows: Issue #39616') def test_cmd_async(self): low = { diff --git a/tests/jenkins.py b/tests/jenkins.py index c7cf12c393b..2eda9878f98 100644 --- a/tests/jenkins.py +++ b/tests/jenkins.py @@ -21,7 +21,8 @@ import subprocess import random # Import Salt libs -import salt.utils +import salt.utils.files +import salt.utils.stringutils try: from salt.utils.nb_popen import NonBlockingPopen except ImportError: @@ -154,7 +155,7 @@ def echo_parseable_environment(options, parser): '.github_token' ) if os.path.isfile(github_access_token_path): - with salt.utils.fopen(github_access_token_path) as rfh: + with salt.utils.files.fopen(github_access_token_path) as rfh: headers = { 'Authorization': 'token {0}'.format(rfh.read().strip()) } @@ -462,7 +463,7 @@ def run(opts): delete_vm(opts) sys.exit(retcode) - outstr = salt.utils.to_str(stdout).strip() + outstr = salt.utils.stringutils.to_str(stdout).strip() if not outstr: print('Failed to get the bootstrapped minion version(no output). Exit code: {0}'.format(retcode)) sys.stdout.flush() @@ -532,9 +533,9 @@ def run(opts): stdout, stderr = proc.communicate() if stdout: - print(salt.utils.to_str(stdout)) + print(salt.utils.stringutils.to_str(stdout)) if stderr: - print(salt.utils.to_str(stderr)) + print(salt.utils.stringutils.to_str(stderr)) sys.stdout.flush() retcode = proc.returncode @@ -572,7 +573,7 @@ def run(opts): # DO NOT print the state return here! print('Cloud configuration files provisioned via pillar.') if stderr: - print(salt.utils.to_str(stderr)) + print(salt.utils.stringutils.to_str(stderr)) sys.stdout.flush() retcode = proc.returncode @@ -608,9 +609,9 @@ def run(opts): stdout, stderr = proc.communicate() if stdout: - print(salt.utils.to_str(stdout)) + print(salt.utils.stringutils.to_str(stdout)) if stderr: - print(salt.utils.to_str(stderr)) + print(salt.utils.stringutils.to_str(stderr)) sys.stdout.flush() retcode = proc.returncode @@ -666,7 +667,7 @@ def run(opts): sys.exit(retcode) print('matches!') except ValueError: - print('Failed to load any JSON from \'{0}\''.format(salt.utils.to_str(stdout).strip())) + print('Failed to load any JSON from \'{0}\''.format(salt.utils.stringutils.to_str(stdout).strip())) if opts.test_git_commit is not None: test_git_commit_downtime = random.randint(1, opts.splay) @@ -713,7 +714,7 @@ def run(opts): sys.exit(retcode) print('matches!') except ValueError: - print('Failed to load any JSON from \'{0}\''.format(salt.utils.to_str(stdout).strip())) + print('Failed to load any JSON from \'{0}\''.format(salt.utils.stringutils.to_str(stdout).strip())) # Run tests here test_begin_downtime = random.randint(3, opts.splay) @@ -736,11 +737,11 @@ def run(opts): ) stdout, stderr = proc.communicate() - outstr = salt.utils.to_str(stdout) + outstr = salt.utils.stringutils.to_str(stdout) if outstr: print(outstr) if stderr: - print(salt.utils.to_str(stderr)) + print(salt.utils.stringutils.to_str(stderr)) sys.stdout.flush() try: @@ -780,9 +781,9 @@ def run(opts): stdout, stderr = proc.communicate() if stdout: - print(salt.utils.to_str(stdout)) + print(salt.utils.stringutils.to_str(stdout)) if stderr: - print(salt.utils.to_str(stderr)) + print(salt.utils.stringutils.to_str(stderr)) sys.stdout.flush() # Grab packages and log file (or just log file if build failed) diff --git a/tests/minionswarm.py b/tests/minionswarm.py index e5ea7f1d715..ab4ece1fb51 100644 --- a/tests/minionswarm.py +++ b/tests/minionswarm.py @@ -26,7 +26,7 @@ import salt # Import third party libs import yaml -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import range # pylint: disable=import-error,redefined-builtin diff --git a/tests/perf/fire_fake.py b/tests/perf/fire_fake.py index 6b3cadd9ecb..224bacca8dc 100644 --- a/tests/perf/fire_fake.py +++ b/tests/perf/fire_fake.py @@ -3,7 +3,6 @@ from __future__ import absolute_import, print_function # Import system libs import sys -import time import datetime # Import salt libs diff --git a/tests/runtests.py b/tests/runtests.py index a71db050d6f..b698adc077b 100755 --- a/tests/runtests.py +++ b/tests/runtests.py @@ -57,9 +57,9 @@ except ImportError as exc: raise exc from tests.integration import TestDaemon # pylint: disable=W0403 -import salt.utils +import salt.utils.platform -if not salt.utils.is_windows(): +if not salt.utils.platform.is_windows(): import resource # Import Salt Testing libs @@ -435,8 +435,10 @@ class SaltTestsuiteParser(SaltCoverageTestingParser): # Turn on expensive tests execution os.environ['EXPENSIVE_TESTS'] = 'True' - import salt.utils - if salt.utils.is_windows(): + # This fails even with salt.utils.platform imported in the global + # scope, unless we import it again here. + import salt.utils.platform + if salt.utils.platform.is_windows(): import salt.utils.win_functions current_user = salt.utils.win_functions.get_current_user() if current_user == 'SYSTEM': @@ -448,7 +450,7 @@ class SaltTestsuiteParser(SaltCoverageTestingParser): if self.options.coverage and any(( self.options.name, - is_admin, + not is_admin, not self.options.run_destructive)) \ and self._check_enabled_suites(include_unit=True): self.error( @@ -490,7 +492,7 @@ class SaltTestsuiteParser(SaltCoverageTestingParser): return self.run_suite(full_path, display_name, suffix='test_*.py') def start_daemons_only(self): - if not salt.utils.is_windows(): + if not salt.utils.platform.is_windows(): self.set_filehandle_limits('integration') try: print_header( @@ -566,7 +568,7 @@ class SaltTestsuiteParser(SaltCoverageTestingParser): for integration tests or unit tests ''' # Get current limits - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): import win32file prev_hard = win32file._getmaxstdio() prev_soft = 512 @@ -602,7 +604,7 @@ class SaltTestsuiteParser(SaltCoverageTestingParser): '{0}, hard: {1}'.format(soft, hard) ) try: - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): hard = 2048 if hard > 2048 else hard win32file._setmaxstdio(hard) else: @@ -639,7 +641,7 @@ class SaltTestsuiteParser(SaltCoverageTestingParser): # `unit.` to --name. We don't need the tests daemon # running return [True] - if not salt.utils.is_windows(): + if not salt.utils.platform.is_windows(): self.set_filehandle_limits('integration') try: diff --git a/tests/support/case.py b/tests/support/case.py index 9bce03f5a65..1a757f5350d 100644 --- a/tests/support/case.py +++ b/tests/support/case.py @@ -35,7 +35,7 @@ from tests.support.mixins import AdaptedConfigurationTestCaseMixin, SaltClientTe from tests.support.paths import ScriptPathMixin, INTEGRATION_TEST_DIR, CODE_DIR, PYEXEC, SCRIPT_DIR # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import cStringIO # pylint: disable=import-error STATE_FUNCTION_RUNNING_RE = re.compile( @@ -83,9 +83,9 @@ class ShellTestCase(TestCase, AdaptedConfigurationTestCaseMixin): log.debug('Generating {0}'.format(script_path)) # Late import - import salt.utils + import salt.utils.files - with salt.utils.fopen(script_path, 'w') as sfh: + with salt.utils.files.fopen(script_path, 'w') as sfh: script_template = SCRIPT_TEMPLATES.get(script_name, None) if script_template is None: script_template = SCRIPT_TEMPLATES.get('common', None) @@ -371,7 +371,7 @@ class ShellTestCase(TestCase, AdaptedConfigurationTestCaseMixin): finally: try: if os.path.exists(tmp_file.name): - if isinstance(tmp_file.name, str): + if isinstance(tmp_file.name, six.string_types): # tmp_file.name is an int when using SpooledTemporaryFiles # int types cannot be used with os.remove() in Python 3 os.remove(tmp_file.name) @@ -403,7 +403,7 @@ class ShellTestCase(TestCase, AdaptedConfigurationTestCaseMixin): finally: try: if os.path.exists(tmp_file.name): - if isinstance(tmp_file.name, str): + if isinstance(tmp_file.name, six.string_types): # tmp_file.name is an int when using SpooledTemporaryFiles # int types cannot be used with os.remove() in Python 3 os.remove(tmp_file.name) diff --git a/tests/support/cptestcase.py b/tests/support/cptestcase.py index ea2845f46d0..cb61020840e 100644 --- a/tests/support/cptestcase.py +++ b/tests/support/cptestcase.py @@ -37,7 +37,7 @@ from tests.support.case import TestCase # Import 3rd-party libs # pylint: disable=import-error import cherrypy # pylint: disable=3rd-party-module-not-gated -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import StringIO # pylint: enable=import-error diff --git a/tests/support/gitfs.py b/tests/support/gitfs.py index 4c9a5f6d1a8..411bfd27ce1 100644 --- a/tests/support/gitfs.py +++ b/tests/support/gitfs.py @@ -18,7 +18,8 @@ import time import yaml # Import Salt libs -import salt.utils +import salt.utils.files +import salt.utils.path from salt.fileserver import gitfs from salt.pillar import git_pillar from salt.ext.six.moves import range # pylint: disable=redefined-builtin @@ -341,7 +342,6 @@ class GitPillarTestBase(GitTestBase, LoaderModuleMockMixin): return git_pillar.ext_pillar( 'minion', ext_pillar_opts['ext_pillar'][0]['git'], - {} ) def make_repo(self, root_dir, user='root'): @@ -384,14 +384,14 @@ class GitPillarTestBase(GitTestBase, LoaderModuleMockMixin): user=user, ) - with salt.utils.fopen( + with salt.utils.files.fopen( os.path.join(self.admin_repo, 'top.sls'), 'w') as fp_: fp_.write(textwrap.dedent('''\ base: '*': - foo ''')) - with salt.utils.fopen( + with salt.utils.files.fopen( os.path.join(self.admin_repo, 'foo.sls'), 'w') as fp_: fp_.write(textwrap.dedent('''\ branch: master @@ -405,7 +405,7 @@ class GitPillarTestBase(GitTestBase, LoaderModuleMockMixin): master: True ''')) # Add another file to be referenced using git_pillar_includes - with salt.utils.fopen( + with salt.utils.files.fopen( os.path.join(self.admin_repo, 'bar.sls'), 'w') as fp_: fp_.write('included_pillar: True\n') _push('master', 'initial commit') @@ -421,14 +421,14 @@ class GitPillarTestBase(GitTestBase, LoaderModuleMockMixin): 'git.rm', [self.admin_repo, 'bar.sls'], user=user) - with salt.utils.fopen( + with salt.utils.files.fopen( os.path.join(self.admin_repo, 'top.sls'), 'w') as fp_: fp_.write(textwrap.dedent('''\ dev: '*': - foo ''')) - with salt.utils.fopen( + with salt.utils.files.fopen( os.path.join(self.admin_repo, 'foo.sls'), 'w') as fp_: fp_.write(textwrap.dedent('''\ branch: dev @@ -455,7 +455,7 @@ class GitPillarTestBase(GitTestBase, LoaderModuleMockMixin): 'git.rm', [self.admin_repo, 'foo.sls'], user=user) - with salt.utils.fopen( + with salt.utils.files.fopen( os.path.join(self.admin_repo, 'top.sls'), 'w') as fp_: fp_.write(textwrap.dedent('''\ base: @@ -500,7 +500,7 @@ class GitPillarSSHTestBase(GitPillarTestBase, SSHDMixin): super(GitPillarSSHTestBase, self).setUp() self.sshd_proc = self.find_proc(name='sshd', search=self.sshd_config) - self.sshd_bin = salt.utils.which('sshd') + self.sshd_bin = salt.utils.path.which('sshd') if self.sshd_proc is None: self.spawn_server() diff --git a/tests/support/helpers.py b/tests/support/helpers.py index 48b8afb35e5..13c7158aaab 100644 --- a/tests/support/helpers.py +++ b/tests/support/helpers.py @@ -19,8 +19,10 @@ import functools import inspect import logging import os +import random import signal import socket +import string import sys import threading import time @@ -30,7 +32,7 @@ import types # Import 3rd-party libs import psutil # pylint: disable=3rd-party-module-not-gated -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import range, builtins # pylint: disable=import-error,redefined-builtin try: from pytestsalt.utils import get_unused_localhost_port # pylint: disable=unused-import @@ -216,11 +218,11 @@ class RedirectStdStreams(object): def __init__(self, stdout=None, stderr=None): # Late import - import salt.utils + import salt.utils.files if stdout is None: - stdout = salt.utils.fopen(os.devnull, 'w') # pylint: disable=resource-leakage + stdout = salt.utils.files.fopen(os.devnull, 'w') # pylint: disable=resource-leakage if stderr is None: - stderr = salt.utils.fopen(os.devnull, 'w') # pylint: disable=resource-leakage + stderr = salt.utils.files.fopen(os.devnull, 'w') # pylint: disable=resource-leakage self.__stdout = stdout self.__stderr = stderr @@ -1023,6 +1025,7 @@ def requires_salt_modules(*names): def skip_if_binaries_missing(*binaries, **kwargs): import salt.utils + import salt.utils.path if len(binaries) == 1: if isinstance(binaries[0], (list, tuple, set, frozenset)): binaries = binaries[0] @@ -1037,14 +1040,14 @@ def skip_if_binaries_missing(*binaries, **kwargs): ) if check_all: for binary in binaries: - if salt.utils.which(binary) is None: + if salt.utils.path.which(binary) is None: return skip( '{0}The {1!r} binary was not found'.format( message and '{0}. '.format(message) or '', binary ) ) - elif salt.utils.which_bin(binaries) is None: + elif salt.utils.path.which_bin(binaries) is None: return skip( '{0}None of the following binaries was found: {1}'.format( message and '{0}. '.format(message) or '', @@ -1331,6 +1334,25 @@ def http_basic_auth(login_cb=lambda username, password: False): return wrapper +def generate_random_name(prefix, size=6): + ''' + Generates a random name by combining the provided prefix with a randomly generated + ascii string. + + .. versionadded:: Oxygen + + prefix + The string to prefix onto the randomly generated ascii string. + + size + The number of characters to generate. Default: 6. + ''' + return prefix + ''.join( + random.choice(string.ascii_uppercase + string.digits) + for x in range(size) + ) + + class Webserver(object): ''' Starts a tornado webserver on 127.0.0.1 on a random available port diff --git a/tests/support/mixins.py b/tests/support/mixins.py index 38601d81ff1..64a4bd4fbcd 100644 --- a/tests/support/mixins.py +++ b/tests/support/mixins.py @@ -30,8 +30,12 @@ from tests.support.runtests import RUNTIME_VARS from tests.support.paths import CODE_DIR # Import salt libs -#import salt.config -import salt.utils +import salt.config +import salt.utils # Can be removed once namespaced_function is moved +import salt.utils.event +import salt.utils.files +import salt.utils.path +import salt.utils.stringutils import salt.version import salt.exceptions from salt.utils.verify import verify_env @@ -40,7 +44,7 @@ from salt._compat import ElementTree as etree # Import 3rd-party libs import yaml -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import zip # pylint: disable=import-error,redefined-builtin log = logging.getLogger(__name__) @@ -111,7 +115,7 @@ class AdaptedConfigurationTestCaseMixin(object): rdict['config_dir'] = conf_dir rdict['conf_file'] = os.path.join(conf_dir, config_for) - with salt.utils.fopen(rdict['conf_file'], 'w') as wfh: + with salt.utils.files.fopen(rdict['conf_file'], 'w') as wfh: wfh.write(yaml.dump(rdict, default_flow_style=False)) return rdict @@ -244,9 +248,8 @@ class ShellCaseCommonTestsMixin(CheckShellBinaryNameAndVersionMixin): def test_salt_with_git_version(self): if getattr(self, '_call_binary_', None) is None: self.skipTest('\'_call_binary_\' not defined.') - from salt.utils import which from salt.version import __version_info__, SaltStackVersion - git = which('git') + git = salt.utils.path.which('git') if not git: self.skipTest('The git binary is not available') @@ -272,7 +275,7 @@ class ShellCaseCommonTestsMixin(CheckShellBinaryNameAndVersionMixin): self.skipTest( 'Failed to get the output of \'git describe\'. ' 'Error: \'{0}\''.format( - salt.utils.to_str(err) + salt.utils.stringutils.to_str(err) ) ) diff --git a/tests/support/mock.py b/tests/support/mock.py index 40e3d1f9993..4046f2e05e8 100644 --- a/tests/support/mock.py +++ b/tests/support/mock.py @@ -15,7 +15,7 @@ from __future__ import absolute_import import sys # Import salt libs -import salt.ext.six as six +from salt.ext import six try: if sys.version_info >= (3,): @@ -68,7 +68,7 @@ except ImportError as exc: class MagicMock(object): - __name__ = '{0}.fakemock'.format(__name__) + __name__ = '{0}.fakemock'.format(__name__) # future lint: disable=non-unicode-string def __init__(self, *args, **kwargs): pass diff --git a/tests/support/parser/__init__.py b/tests/support/parser/__init__.py index 75841639262..429ad4f0cf3 100644 --- a/tests/support/parser/__init__.py +++ b/tests/support/parser/__init__.py @@ -32,7 +32,7 @@ from tests.support.unit import TestLoader, TextTestRunner from tests.support.xmlunit import HAS_XMLRUNNER, XMLTestRunner # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six try: from tests.support.ext import console WIDTH, HEIGHT = console.getTerminalSize() diff --git a/tests/support/paths.py b/tests/support/paths.py index da93c8e1e33..46bc1a3cbb6 100644 --- a/tests/support/paths.py +++ b/tests/support/paths.py @@ -81,9 +81,9 @@ SCRIPT_TEMPLATES = { ], 'common': [ 'from salt.scripts import salt_{0}\n', - 'from salt.utils import is_windows\n\n', + 'import salt.utils.platform\n\n', 'if __name__ == \'__main__\':\n', - ' if is_windows():\n', + ' if salt.utils.platform.is_windows():\n', ' import os.path\n', ' import py_compile\n', ' cfile = os.path.splitext(__file__)[0] + ".pyc"\n', @@ -110,9 +110,9 @@ class ScriptPathMixin(object): log.info('Generating {0}'.format(script_path)) # Late import - import salt.utils + import salt.utils.files - with salt.utils.fopen(script_path, 'w') as sfh: + with salt.utils.files.fopen(script_path, 'w') as sfh: script_template = SCRIPT_TEMPLATES.get(script_name, None) if script_template is None: script_template = SCRIPT_TEMPLATES.get('common', None) diff --git a/tests/support/runtests.py b/tests/support/runtests.py index 6bee7d4c547..2e8bc9b0a2b 100644 --- a/tests/support/runtests.py +++ b/tests/support/runtests.py @@ -59,7 +59,7 @@ import multiprocessing import tests.support.paths as paths # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six try: import coverage # pylint: disable=import-error HAS_COVERAGE = True diff --git a/tests/support/unit.py b/tests/support/unit.py index faaf986f837..841e714e327 100644 --- a/tests/support/unit.py +++ b/tests/support/unit.py @@ -26,7 +26,7 @@ from __future__ import absolute_import import os import sys import logging -import salt.ext.six as six +from salt.ext import six try: import psutil HAS_PSUTIL = True @@ -177,6 +177,7 @@ class TestCase(_TestCase): def run(self, result=None): self._prerun_instance_attributes = dir(self) + self.maxDiff = None outcome = super(TestCase, self).run(result=result) for attr in dir(self): if attr == '_prerun_instance_attributes': @@ -305,6 +306,16 @@ class TestCase(_TestCase): ) # return _TestCase.failIfAlmostEqual(self, *args, **kwargs) + @staticmethod + def assert_called_once(mock): + ''' + mock.assert_called_once only exists in PY3 in 3.6 and newer + ''' + try: + mock.assert_called_once() + except AttributeError: + log.warning('assert_called_once invoked, but not available') + if six.PY2: def assertRegexpMatches(self, *args, **kwds): raise DeprecationWarning( diff --git a/tests/support/xmlunit.py b/tests/support/xmlunit.py index 1196aa0f8c6..5c7d350a5f0 100644 --- a/tests/support/xmlunit.py +++ b/tests/support/xmlunit.py @@ -19,7 +19,7 @@ import sys import logging # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six log = logging.getLogger(__name__) diff --git a/tests/unit/beacons/test_adb_beacon.py b/tests/unit/beacons/test_adb_beacon.py index 1ae3c2e7c41..e0d9f738c12 100644 --- a/tests/unit/beacons/test_adb_beacon.py +++ b/tests/unit/beacons/test_adb_beacon.py @@ -27,7 +27,7 @@ class ADBBeaconTestCase(TestCase, LoaderModuleMockMixin): } def test_no_adb_command(self): - with patch('salt.utils.which') as mock: + with patch('salt.utils.path.which') as mock: mock.return_value = None ret = adb.__virtual__() @@ -36,7 +36,7 @@ class ADBBeaconTestCase(TestCase, LoaderModuleMockMixin): self.assertFalse(ret) def test_with_adb_command(self): - with patch('salt.utils.which') as mock: + with patch('salt.utils.path.which') as mock: mock.return_value = '/usr/bin/adb' ret = adb.__virtual__() @@ -44,50 +44,49 @@ class ADBBeaconTestCase(TestCase, LoaderModuleMockMixin): mock.assert_called_once_with('adb') self.assertEqual(ret, 'adb') - def test_non_dict_config(self): - config = [] - - log_mock = Mock() - with patch.object(adb, 'log', log_mock): - - ret = adb.beacon(config) - - self.assertEqual(ret, []) - log_mock.info.assert_called_once_with('Configuration for adb beacon must be a dict.') - - def test_empty_config(self): + def test_non_list_config(self): config = {} - log_mock = Mock() - with patch.object(adb, 'log', log_mock): - ret = adb.beacon(config) + ret = adb.validate(config) - self.assertEqual(ret, []) - log_mock.info.assert_called_once_with('Configuration for adb beacon must include a states array.') + self.assertEqual(ret, (False, 'Configuration for adb beacon must' + ' be a list.')) + + def test_empty_config(self): + config = [{}] + + ret = adb.validate(config) + + self.assertEqual(ret, (False, 'Configuration for adb beacon must' + ' include a states array.')) def test_invalid_states(self): - config = {'states': ['Random', 'Failings']} + config = [{'states': ['Random', 'Failings']}] - log_mock = Mock() - with patch.object(adb, 'log', log_mock): + ret = adb.validate(config) - ret = adb.beacon(config) - - self.assertEqual(ret, []) - log_mock.info.assert_called_once_with('Need a one of the following adb states:' - ' offline, bootloader, device, host, recovery, ' - 'no permissions, sideload, unauthorized, unknown, missing') + self.assertEqual(ret, (False, 'Need a one of the following' + ' adb states: offline, bootloader,' + ' device, host, recovery, no' + ' permissions, sideload,' + ' unauthorized, unknown, missing')) def test_device_state(self): - config = {'states': ['device']} + config = [{'states': ['device']}] mock = Mock(return_value='List of devices attached\nHTC\tdevice',) with patch.dict(adb.__salt__, {'cmd.run': mock}): + + ret = adb.validate(config) + self.assertEqual(ret, (True, 'Valid beacon configuration')) + ret = adb.beacon(config) - self.assertEqual(ret, [{'device': 'HTC', 'state': 'device', 'tag': 'device'}]) + self.assertEqual(ret, [{'device': 'HTC', + 'state': 'device', + 'tag': 'device'}]) def test_device_state_change(self): - config = {'states': ['offline']} + config = [{'states': ['offline']}] out = [ 'List of devices attached\nHTC\tdevice', @@ -97,14 +96,19 @@ class ADBBeaconTestCase(TestCase, LoaderModuleMockMixin): mock = Mock(side_effect=out) with patch.dict(adb.__salt__, {'cmd.run': mock}): + ret = adb.validate(config) + self.assertEqual(ret, (True, 'Valid beacon configuration')) + ret = adb.beacon(config) self.assertEqual(ret, []) ret = adb.beacon(config) - self.assertEqual(ret, [{'device': 'HTC', 'state': 'offline', 'tag': 'offline'}]) + self.assertEqual(ret, [{'device': 'HTC', + 'state': 'offline', + 'tag': 'offline'}]) def test_multiple_devices(self): - config = {'states': ['offline', 'device']} + config = [{'states': ['offline', 'device']}] out = [ 'List of devices attached\nHTC\tdevice', @@ -114,8 +118,13 @@ class ADBBeaconTestCase(TestCase, LoaderModuleMockMixin): mock = Mock(side_effect=out) with patch.dict(adb.__salt__, {'cmd.run': mock}): + ret = adb.validate(config) + self.assertEqual(ret, (True, 'Valid beacon configuration')) + ret = adb.beacon(config) - self.assertEqual(ret, [{'device': 'HTC', 'state': 'device', 'tag': 'device'}]) + self.assertEqual(ret, [{'device': 'HTC', + 'state': 'device', + 'tag': 'device'}]) ret = adb.beacon(config) self.assertEqual(ret, [ @@ -124,16 +133,19 @@ class ADBBeaconTestCase(TestCase, LoaderModuleMockMixin): ]) def test_no_devices_with_different_states(self): - config = {'states': ['offline'], 'no_devices_event': True} + config = [{'states': ['offline'], 'no_devices_event': True}] mock = Mock(return_value='List of devices attached\nHTC\tdevice') with patch.dict(adb.__salt__, {'cmd.run': mock}): + ret = adb.validate(config) + self.assertEqual(ret, (True, 'Valid beacon configuration')) + ret = adb.beacon(config) self.assertEqual(ret, []) def test_no_devices_no_repeat(self): - config = {'states': ['offline', 'device'], 'no_devices_event': True} + config = [{'states': ['offline', 'device'], 'no_devices_event': True}] out = [ 'List of devices attached\nHTC\tdevice', @@ -144,8 +156,13 @@ class ADBBeaconTestCase(TestCase, LoaderModuleMockMixin): mock = Mock(side_effect=out) with patch.dict(adb.__salt__, {'cmd.run': mock}): + ret = adb.validate(config) + self.assertEqual(ret, (True, 'Valid beacon configuration')) + ret = adb.beacon(config) - self.assertEqual(ret, [{'device': 'HTC', 'state': 'device', 'tag': 'device'}]) + self.assertEqual(ret, [{'device': 'HTC', + 'state': 'device', + 'tag': 'device'}]) ret = adb.beacon(config) self.assertEqual(ret, [{'tag': 'no_devices'}]) @@ -154,7 +171,7 @@ class ADBBeaconTestCase(TestCase, LoaderModuleMockMixin): self.assertEqual(ret, []) def test_no_devices(self): - config = {'states': ['offline', 'device'], 'no_devices_event': True} + config = [{'states': ['offline', 'device'], 'no_devices_event': True}] out = [ 'List of devices attached', @@ -164,6 +181,9 @@ class ADBBeaconTestCase(TestCase, LoaderModuleMockMixin): mock = Mock(side_effect=out) with patch.dict(adb.__salt__, {'cmd.run': mock}): + ret = adb.validate(config) + self.assertEqual(ret, (True, 'Valid beacon configuration')) + ret = adb.beacon(config) self.assertEqual(ret, [{'tag': 'no_devices'}]) @@ -171,7 +191,7 @@ class ADBBeaconTestCase(TestCase, LoaderModuleMockMixin): self.assertEqual(ret, []) def test_device_missing(self): - config = {'states': ['device', 'missing']} + config = [{'states': ['device', 'missing']}] out = [ 'List of devices attached\nHTC\tdevice', @@ -183,37 +203,56 @@ class ADBBeaconTestCase(TestCase, LoaderModuleMockMixin): mock = Mock(side_effect=out) with patch.dict(adb.__salt__, {'cmd.run': mock}): - ret = adb.beacon(config) - self.assertEqual(ret, [{'device': 'HTC', 'state': 'device', 'tag': 'device'}]) + ret = adb.validate(config) + self.assertEqual(ret, (True, 'Valid beacon configuration')) ret = adb.beacon(config) - self.assertEqual(ret, [{'device': 'HTC', 'state': 'missing', 'tag': 'missing'}]) + self.assertEqual(ret, [{'device': 'HTC', + 'state': 'device', + 'tag': 'device'}]) ret = adb.beacon(config) - self.assertEqual(ret, [{'device': 'HTC', 'state': 'device', 'tag': 'device'}]) + self.assertEqual(ret, [{'device': 'HTC', + 'state': 'missing', + 'tag': 'missing'}]) + + ret = adb.beacon(config) + self.assertEqual(ret, [{'device': 'HTC', + 'state': 'device', + 'tag': 'device'}]) ret = adb.beacon(config) self.assertEqual(ret, []) def test_with_startup(self): - config = {'states': ['device']} + config = [{'states': ['device']}] mock = Mock(return_value='* daemon started successfully *\nList of devices attached\nHTC\tdevice',) with patch.dict(adb.__salt__, {'cmd.run': mock}): + ret = adb.validate(config) + self.assertEqual(ret, (True, 'Valid beacon configuration')) + ret = adb.beacon(config) - self.assertEqual(ret, [{'device': 'HTC', 'state': 'device', 'tag': 'device'}]) + self.assertEqual(ret, [{'device': 'HTC', + 'state': 'device', + 'tag': 'device'}]) def test_with_user(self): - config = {'states': ['device'], 'user': 'fred'} + config = [{'states': ['device'], 'user': 'fred'}] mock = Mock(return_value='* daemon started successfully *\nList of devices attached\nHTC\tdevice') with patch.dict(adb.__salt__, {'cmd.run': mock}): + ret = adb.validate(config) + self.assertEqual(ret, (True, 'Valid beacon configuration')) + ret = adb.beacon(config) mock.assert_called_once_with('adb devices', runas='fred') - self.assertEqual(ret, [{'device': 'HTC', 'state': 'device', 'tag': 'device'}]) + self.assertEqual(ret, [{'device': 'HTC', + 'state': 'device', + 'tag': 'device'}]) def test_device_low_battery(self): - config = {'states': ['device'], 'battery_low': 30} + config = [{'states': ['device'], 'battery_low': 30}] out = [ 'List of devices attached\nHTC\tdevice', @@ -221,12 +260,15 @@ class ADBBeaconTestCase(TestCase, LoaderModuleMockMixin): ] mock = Mock(side_effect=out) with patch.dict(adb.__salt__, {'cmd.run': mock}): + ret = adb.validate(config) + self.assertEqual(ret, (True, 'Valid beacon configuration')) + ret = adb.beacon(config) self.assertEqual(ret, [{'device': 'HTC', 'state': 'device', 'tag': 'device'}, {'device': 'HTC', 'battery_level': 25, 'tag': 'battery_low'}]) def test_device_no_repeat(self): - config = {'states': ['device'], 'battery_low': 30} + config = [{'states': ['device'], 'battery_low': 30}] out = [ 'List of devices attached\nHTC\tdevice', @@ -236,6 +278,9 @@ class ADBBeaconTestCase(TestCase, LoaderModuleMockMixin): ] mock = Mock(side_effect=out) with patch.dict(adb.__salt__, {'cmd.run': mock}): + ret = adb.validate(config) + self.assertEqual(ret, (True, 'Valid beacon configuration')) + ret = adb.beacon(config) self.assertEqual(ret, [{'device': 'HTC', 'state': 'device', 'tag': 'device'}, {'device': 'HTC', 'battery_level': 25, 'tag': 'battery_low'}]) @@ -244,7 +289,7 @@ class ADBBeaconTestCase(TestCase, LoaderModuleMockMixin): self.assertEqual(ret, []) def test_device_no_repeat_capacity_increase(self): - config = {'states': ['device'], 'battery_low': 75} + config = [{'states': ['device'], 'battery_low': 75}] out = [ 'List of devices attached\nHTC\tdevice', @@ -254,6 +299,9 @@ class ADBBeaconTestCase(TestCase, LoaderModuleMockMixin): ] mock = Mock(side_effect=out) with patch.dict(adb.__salt__, {'cmd.run': mock}): + ret = adb.validate(config) + self.assertEqual(ret, (True, 'Valid beacon configuration')) + ret = adb.beacon(config) self.assertEqual(ret, [{'device': 'HTC', 'state': 'device', 'tag': 'device'}, {'device': 'HTC', 'battery_level': 25, 'tag': 'battery_low'}]) @@ -262,7 +310,7 @@ class ADBBeaconTestCase(TestCase, LoaderModuleMockMixin): self.assertEqual(ret, []) def test_device_no_repeat_with_not_found_state(self): - config = {'states': ['offline'], 'battery_low': 30} + config = [{'states': ['offline'], 'battery_low': 30}] out = [ 'List of devices attached\nHTC\tdevice', @@ -272,6 +320,9 @@ class ADBBeaconTestCase(TestCase, LoaderModuleMockMixin): ] mock = Mock(side_effect=out) with patch.dict(adb.__salt__, {'cmd.run': mock}): + ret = adb.validate(config) + self.assertEqual(ret, (True, 'Valid beacon configuration')) + ret = adb.beacon(config) self.assertEqual(ret, [{'device': 'HTC', 'battery_level': 25, 'tag': 'battery_low'}]) @@ -279,7 +330,7 @@ class ADBBeaconTestCase(TestCase, LoaderModuleMockMixin): self.assertEqual(ret, []) def test_device_battery_charged(self): - config = {'states': ['device'], 'battery_low': 30} + config = [{'states': ['device'], 'battery_low': 30}] out = [ 'List of devices attached\nHTC\tdevice', @@ -287,11 +338,16 @@ class ADBBeaconTestCase(TestCase, LoaderModuleMockMixin): ] mock = Mock(side_effect=out) with patch.dict(adb.__salt__, {'cmd.run': mock}): + ret = adb.validate(config) + self.assertEqual(ret, (True, 'Valid beacon configuration')) + ret = adb.beacon(config) - self.assertEqual(ret, [{'device': 'HTC', 'state': 'device', 'tag': 'device'}]) + self.assertEqual(ret, [{'device': 'HTC', + 'state': 'device', + 'tag': 'device'}]) def test_device_low_battery_equal(self): - config = {'states': ['device'], 'battery_low': 25} + config = [{'states': ['device'], 'battery_low': 25}] out = [ 'List of devices attached\nHTC\tdevice', @@ -299,12 +355,15 @@ class ADBBeaconTestCase(TestCase, LoaderModuleMockMixin): ] mock = Mock(side_effect=out) with patch.dict(adb.__salt__, {'cmd.run': mock}): + ret = adb.validate(config) + self.assertEqual(ret, (True, 'Valid beacon configuration')) + ret = adb.beacon(config) self.assertEqual(ret, [{'device': 'HTC', 'state': 'device', 'tag': 'device'}, {'device': 'HTC', 'battery_level': 25, 'tag': 'battery_low'}]) def test_device_battery_not_found(self): - config = {'states': ['device'], 'battery_low': 25} + config = [{'states': ['device'], 'battery_low': 25}] out = [ 'List of devices attached\nHTC\tdevice', @@ -312,11 +371,14 @@ class ADBBeaconTestCase(TestCase, LoaderModuleMockMixin): ] mock = Mock(side_effect=out) with patch.dict(adb.__salt__, {'cmd.run': mock}): + ret = adb.validate(config) + self.assertEqual(ret, (True, 'Valid beacon configuration')) + ret = adb.beacon(config) self.assertEqual(ret, [{'device': 'HTC', 'state': 'device', 'tag': 'device'}]) def test_device_repeat_multi(self): - config = {'states': ['offline'], 'battery_low': 35} + config = [{'states': ['offline'], 'battery_low': 35}] out = [ 'List of devices attached\nHTC\tdevice', @@ -330,6 +392,9 @@ class ADBBeaconTestCase(TestCase, LoaderModuleMockMixin): ] mock = Mock(side_effect=out) with patch.dict(adb.__salt__, {'cmd.run': mock}): + ret = adb.validate(config) + self.assertEqual(ret, (True, 'Valid beacon configuration')) + ret = adb.beacon(config) self.assertEqual(ret, [{'device': 'HTC', 'battery_level': 25, 'tag': 'battery_low'}]) @@ -343,7 +408,7 @@ class ADBBeaconTestCase(TestCase, LoaderModuleMockMixin): self.assertEqual(ret, []) def test_weird_batteries(self): - config = {'states': ['device'], 'battery_low': 25} + config = [{'states': ['device'], 'battery_low': 25}] out = [ 'List of devices attached\nHTC\tdevice', @@ -351,11 +416,14 @@ class ADBBeaconTestCase(TestCase, LoaderModuleMockMixin): ] mock = Mock(side_effect=out) with patch.dict(adb.__salt__, {'cmd.run': mock}): + ret = adb.validate(config) + self.assertEqual(ret, (True, 'Valid beacon configuration')) + ret = adb.beacon(config) self.assertEqual(ret, [{'device': 'HTC', 'state': 'device', 'tag': 'device'}]) def test_multiple_batteries(self): - config = {'states': ['device'], 'battery_low': 30} + config = [{'states': ['device'], 'battery_low': 30}] out = [ 'List of devices attached\nHTC\tdevice', @@ -363,12 +431,15 @@ class ADBBeaconTestCase(TestCase, LoaderModuleMockMixin): ] mock = Mock(side_effect=out) with patch.dict(adb.__salt__, {'cmd.run': mock}): + ret = adb.validate(config) + self.assertEqual(ret, (True, 'Valid beacon configuration')) + ret = adb.beacon(config) self.assertEqual(ret, [{'device': 'HTC', 'state': 'device', 'tag': 'device'}, {'device': 'HTC', 'battery_level': 25, 'tag': 'battery_low'}]) def test_multiple_low_batteries(self): - config = {'states': ['device'], 'battery_low': 30} + config = [{'states': ['device'], 'battery_low': 30}] out = [ 'List of devices attached\nHTC\tdevice', @@ -376,6 +447,9 @@ class ADBBeaconTestCase(TestCase, LoaderModuleMockMixin): ] mock = Mock(side_effect=out) with patch.dict(adb.__salt__, {'cmd.run': mock}): + ret = adb.validate(config) + self.assertEqual(ret, (True, 'Valid beacon configuration')) + ret = adb.beacon(config) self.assertEqual(ret, [{'device': 'HTC', 'state': 'device', 'tag': 'device'}, {'device': 'HTC', 'battery_level': 25, 'tag': 'battery_low'}]) diff --git a/tests/unit/beacons/test_avahi_announce_beacon.py b/tests/unit/beacons/test_avahi_announce_beacon.py new file mode 100644 index 00000000000..baf5d0cb3e0 --- /dev/null +++ b/tests/unit/beacons/test_avahi_announce_beacon.py @@ -0,0 +1,44 @@ +# coding: utf-8 + +# Python libs +from __future__ import absolute_import + +# Salt testing libs +from tests.support.unit import skipIf, TestCase +from tests.support.mock import NO_MOCK, NO_MOCK_REASON +from tests.support.mixins import LoaderModuleMockMixin + +# Salt libs +import salt.beacons.avahi_announce as avahi_announce + + +@skipIf(NO_MOCK, NO_MOCK_REASON) +class AvahiAnnounceBeaconTestCase(TestCase, LoaderModuleMockMixin): + ''' + Test case for salt.beacons.avahi_announce + ''' + + def setup_loader_modules(self): + return { + avahi_announce: { + 'last_state': {}, + 'last_state_extra': {'no_devices': False} + } + } + + def test_non_list_config(self): + config = {} + + ret = avahi_announce.validate(config) + + self.assertEqual(ret, (False, 'Configuration for avahi_announce' + ' beacon must be a list.')) + + def test_empty_config(self): + config = [{}] + + ret = avahi_announce.validate(config) + + self.assertEqual(ret, (False, 'Configuration for avahi_announce' + ' beacon must contain servicetype, port' + ' and txt items.')) diff --git a/tests/unit/beacons/test_bonjour_announce_beacon.py b/tests/unit/beacons/test_bonjour_announce_beacon.py new file mode 100644 index 00000000000..f10aa2a42ef --- /dev/null +++ b/tests/unit/beacons/test_bonjour_announce_beacon.py @@ -0,0 +1,44 @@ +# coding: utf-8 + +# Python libs +from __future__ import absolute_import + +# Salt testing libs +from tests.support.unit import skipIf, TestCase +from tests.support.mock import NO_MOCK, NO_MOCK_REASON +from tests.support.mixins import LoaderModuleMockMixin + +# Salt libs +import salt.beacons.bonjour_announce as bonjour_announce + + +@skipIf(NO_MOCK, NO_MOCK_REASON) +class BonjourAnnounceBeaconTestCase(TestCase, LoaderModuleMockMixin): + ''' + Test case for salt.beacons.avahi_announce + ''' + + def setup_loader_modules(self): + return { + bonjour_announce: { + 'last_state': {}, + 'last_state_extra': {'no_devices': False} + } + } + + def test_non_list_config(self): + config = {} + + ret = bonjour_announce.validate(config) + + self.assertEqual(ret, (False, 'Configuration for bonjour_announce' + ' beacon must be a list.')) + + def test_empty_config(self): + config = [{}] + + ret = bonjour_announce.validate(config) + + self.assertEqual(ret, (False, 'Configuration for bonjour_announce' + ' beacon must contain servicetype, port' + ' and txt items.')) diff --git a/tests/unit/beacons/test_diskusage_beacon.py b/tests/unit/beacons/test_diskusage_beacon.py new file mode 100644 index 00000000000..b771220dd00 --- /dev/null +++ b/tests/unit/beacons/test_diskusage_beacon.py @@ -0,0 +1,88 @@ +# coding: utf-8 + +# Python libs +from __future__ import absolute_import +from collections import namedtuple + +# Salt testing libs +from tests.support.unit import skipIf, TestCase +from tests.support.mock import NO_MOCK, NO_MOCK_REASON, patch, MagicMock +from tests.support.mixins import LoaderModuleMockMixin + +# Salt libs +import salt.beacons.diskusage as diskusage + +STUB_DISK_PARTITION = namedtuple( + 'partition', + 'device mountpoint fstype, opts')( + '/dev/disk0s2', '/', 'hfs', + 'rw,local,rootfs,dovolfs,journaled,multilabel') +STUB_DISK_USAGE = namedtuple('usage', + 'total used free percent')(1000, 500, 500, 50) + + +@skipIf(NO_MOCK, NO_MOCK_REASON) +class DiskUsageBeaconTestCase(TestCase, LoaderModuleMockMixin): + ''' + Test case for salt.beacons.adb + ''' + + def setup_loader_modules(self): + return {} + + def test_non_list_config(self): + config = {} + + ret = diskusage.validate(config) + + self.assertEqual(ret, (False, 'Configuration for diskusage beacon must' + ' be a list.')) + + def test_empty_config(self): + config = [{}] + + ret = diskusage.validate(config) + + self.assertEqual(ret, (True, 'Valid beacon configuration')) + + def test_diskusage_match(self): + with patch('psutil.disk_partitions', + MagicMock(return_value=[STUB_DISK_PARTITION])), \ + patch('psutil.disk_usage', + MagicMock(return_value=STUB_DISK_USAGE)): + config = [{'/': '50%'}] + + ret = diskusage.validate(config) + + self.assertEqual(ret, (True, 'Valid beacon configuration')) + + ret = diskusage.beacon(config) + self.assertEqual(ret, [{'diskusage': 50, 'mount': '/'}]) + + def test_diskusage_nomatch(self): + with patch('psutil.disk_partitions', + MagicMock(return_value=[STUB_DISK_PARTITION])), \ + patch('psutil.disk_usage', + MagicMock(return_value=STUB_DISK_USAGE)): + config = [{'/': '70%'}] + + ret = diskusage.validate(config) + + self.assertEqual(ret, (True, 'Valid beacon configuration')) + + ret = diskusage.beacon(config) + self.assertNotEqual(ret, [{'diskusage': 50, 'mount': '/'}]) + + def test_diskusage_match_regex(self): + with patch('psutil.disk_partitions', + MagicMock(return_value=[STUB_DISK_PARTITION])), \ + patch('psutil.disk_usage', + MagicMock(return_value=STUB_DISK_USAGE)): + config = [{r'^\/': '50%'}] + + ret = diskusage.validate(config) + + self.assertEqual(ret, (True, 'Valid beacon configuration')) + + ret = diskusage.beacon(config) + self.assertEqual(ret, [{'diskusage': 50, 'mount': '/'}]) diff --git a/tests/unit/beacons/test_glxinfo.py b/tests/unit/beacons/test_glxinfo.py deleted file mode 100644 index fae753894d0..00000000000 --- a/tests/unit/beacons/test_glxinfo.py +++ /dev/null @@ -1,98 +0,0 @@ -# coding: utf-8 - -# Python libs -from __future__ import absolute_import - -# Salt libs -import salt.beacons.glxinfo as glxinfo - -# Salt testing libs -from tests.support.unit import skipIf, TestCase -from tests.support.mock import NO_MOCK, NO_MOCK_REASON, patch, Mock - -# Import test suite libs -from tests.support.mixins import LoaderModuleMockMixin - - -@skipIf(NO_MOCK, NO_MOCK_REASON) -class GLXInfoBeaconTestCase(TestCase, LoaderModuleMockMixin): - ''' - Test case for salt.beacons.glxinfo - ''' - - def setup_loader_modules(self): - return {glxinfo: {'last_state': {}}} - - def test_no_adb_command(self): - with patch('salt.utils.which') as mock: - mock.return_value = None - - ret = glxinfo.__virtual__() - - mock.assert_called_once_with('glxinfo') - self.assertFalse(ret) - - def test_with_adb_command(self): - with patch('salt.utils.which') as mock: - mock.return_value = '/usr/bin/glxinfo' - - ret = glxinfo.__virtual__() - - mock.assert_called_once_with('glxinfo') - self.assertEqual(ret, 'glxinfo') - - def test_non_dict_config(self): - config = [] - - log_mock = Mock() - with patch.object(glxinfo, 'log', log_mock): - ret = glxinfo.beacon(config) - - self.assertEqual(ret, []) - - def test_no_user(self): - config = {'screen_event': True} - - log_mock = Mock() - with patch.object(glxinfo, 'log', log_mock): - ret = glxinfo.beacon(config) - self.assertEqual(ret, []) - - def test_screen_state(self): - config = {'screen_event': True, 'user': 'frank'} - - mock = Mock(return_value=0) - with patch.dict(glxinfo.__salt__, {'cmd.retcode': mock}): - ret = glxinfo.beacon(config) - self.assertEqual(ret, [{'tag': 'screen_event', 'screen_available': True}]) - mock.assert_called_once_with('DISPLAY=:0 glxinfo', runas='frank', python_shell=True) - - def test_screen_state_missing(self): - config = {'screen_event': True, 'user': 'frank'} - - mock = Mock(return_value=255) - with patch.dict(glxinfo.__salt__, {'cmd.retcode': mock}): - ret = glxinfo.beacon(config) - self.assertEqual(ret, [{'tag': 'screen_event', 'screen_available': False}]) - - def test_screen_state_no_repeat(self): - config = {'screen_event': True, 'user': 'frank'} - - mock = Mock(return_value=255) - with patch.dict(glxinfo.__salt__, {'cmd.retcode': mock}): - ret = glxinfo.beacon(config) - self.assertEqual(ret, [{'tag': 'screen_event', 'screen_available': False}]) - - ret = glxinfo.beacon(config) - self.assertEqual(ret, []) - - def test_screen_state_change(self): - config = {'screen_event': True, 'user': 'frank'} - - mock = Mock(side_effect=[255, 0]) - with patch.dict(glxinfo.__salt__, {'cmd.retcode': mock}): - ret = glxinfo.beacon(config) - self.assertEqual(ret, [{'tag': 'screen_event', 'screen_available': False}]) - - ret = glxinfo.beacon(config) - self.assertEqual(ret, [{'tag': 'screen_event', 'screen_available': True}]) diff --git a/tests/unit/beacons/test_glxinfo_beacon.py b/tests/unit/beacons/test_glxinfo_beacon.py new file mode 100644 index 00000000000..3457b067439 --- /dev/null +++ b/tests/unit/beacons/test_glxinfo_beacon.py @@ -0,0 +1,118 @@ +# coding: utf-8 + +# Python libs +from __future__ import absolute_import + +# Salt libs +import salt.beacons.glxinfo as glxinfo + +# Salt testing libs +from tests.support.unit import skipIf, TestCase +from tests.support.mock import NO_MOCK, NO_MOCK_REASON, patch, Mock + +# Import test suite libs +from tests.support.mixins import LoaderModuleMockMixin + + +@skipIf(NO_MOCK, NO_MOCK_REASON) +class GLXInfoBeaconTestCase(TestCase, LoaderModuleMockMixin): + ''' + Test case for salt.beacons.glxinfo + ''' + + def setup_loader_modules(self): + return {glxinfo: {'last_state': {}}} + + def test_no_glxinfo_command(self): + with patch('salt.utils.path.which') as mock: + mock.return_value = None + + ret = glxinfo.__virtual__() + + mock.assert_called_once_with('glxinfo') + self.assertFalse(ret) + + def test_with_glxinfo_command(self): + with patch('salt.utils.path.which') as mock: + mock.return_value = '/usr/bin/glxinfo' + + ret = glxinfo.__virtual__() + + mock.assert_called_once_with('glxinfo') + self.assertEqual(ret, 'glxinfo') + + def test_non_list_config(self): + config = {} + + ret = glxinfo.validate(config) + + self.assertEqual(ret, (False, 'Configuration for glxinfo ' + 'beacon must be a list.')) + + def test_no_user(self): + config = [{'screen_event': True}] + + _expected = (False, 'Configuration for glxinfo beacon must ' + 'include a user as glxinfo is not ' + 'available to root.') + ret = glxinfo.validate(config) + self.assertEqual(ret, _expected) + + def test_screen_state(self): + config = [{'screen_event': True, + 'user': 'frank'}] + + mock = Mock(return_value=0) + with patch.dict(glxinfo.__salt__, {'cmd.retcode': mock}): + + ret = glxinfo.validate(config) + self.assertEqual(ret, (True, 'Valid beacon configuration')) + + ret = glxinfo.beacon(config) + self.assertEqual(ret, [{'tag': 'screen_event', + 'screen_available': True}]) + mock.assert_called_once_with('DISPLAY=:0 glxinfo', + runas='frank', python_shell=True) + + def test_screen_state_missing(self): + config = [{'screen_event': True, 'user': 'frank'}] + + mock = Mock(return_value=255) + with patch.dict(glxinfo.__salt__, {'cmd.retcode': mock}): + ret = glxinfo.validate(config) + self.assertEqual(ret, (True, 'Valid beacon configuration')) + + ret = glxinfo.beacon(config) + self.assertEqual(ret, [{'tag': 'screen_event', + 'screen_available': False}]) + + def test_screen_state_no_repeat(self): + config = [{'screen_event': True, 'user': 'frank'}] + + mock = Mock(return_value=255) + with patch.dict(glxinfo.__salt__, {'cmd.retcode': mock}): + ret = glxinfo.validate(config) + self.assertEqual(ret, (True, 'Valid beacon configuration')) + + ret = glxinfo.beacon(config) + self.assertEqual(ret, [{'tag': 'screen_event', + 'screen_available': False}]) + + ret = glxinfo.beacon(config) + self.assertEqual(ret, []) + + def test_screen_state_change(self): + config = [{'screen_event': True, 'user': 'frank'}] + + mock = Mock(side_effect=[255, 0]) + with patch.dict(glxinfo.__salt__, {'cmd.retcode': mock}): + ret = glxinfo.validate(config) + self.assertEqual(ret, (True, 'Valid beacon configuration')) + + ret = glxinfo.beacon(config) + self.assertEqual(ret, [{'tag': 'screen_event', + 'screen_available': False}]) + + ret = glxinfo.beacon(config) + self.assertEqual(ret, [{'tag': 'screen_event', + 'screen_available': True}]) diff --git a/tests/unit/beacons/test_haproxy_beacon.py b/tests/unit/beacons/test_haproxy_beacon.py new file mode 100644 index 00000000000..34a5a46eecb --- /dev/null +++ b/tests/unit/beacons/test_haproxy_beacon.py @@ -0,0 +1,78 @@ +# coding: utf-8 + +# Python libs +from __future__ import absolute_import + +# Salt testing libs +from tests.support.unit import skipIf, TestCase +from tests.support.mock import NO_MOCK, NO_MOCK_REASON, patch, MagicMock +from tests.support.mixins import LoaderModuleMockMixin + +# Salt libs +import salt.beacons.haproxy as haproxy + + +@skipIf(NO_MOCK, NO_MOCK_REASON) +class HAProxyBeaconTestCase(TestCase, LoaderModuleMockMixin): + ''' + Test case for salt.beacons.haproxy + ''' + + def setup_loader_modules(self): + return { + haproxy: { + '__context__': {}, + '__salt__': {}, + } + } + + def test_non_list_config(self): + config = {} + + ret = haproxy.validate(config) + + self.assertEqual(ret, (False, 'Configuration for haproxy beacon must' + ' be a list.')) + + def test_empty_config(self): + config = [{}] + + ret = haproxy.validate(config) + + self.assertEqual(ret, (False, 'Configuration for haproxy beacon ' + 'requires backends.')) + + def test_no_servers(self): + config = [{'backends': {'www-backend': {'threshold': 45}}}] + ret = haproxy.validate(config) + + self.assertEqual(ret, (False, 'Backends for haproxy ' + 'beacon require servers.')) + + def test_threshold_reached(self): + config = [{'backends': {'www-backend': {'threshold': 45, + 'servers': ['web1'] + }}}] + ret = haproxy.validate(config) + + self.assertEqual(ret, (True, 'Valid beacon configuration')) + + mock = MagicMock(return_value=46) + with patch.dict(haproxy.__salt__, {'haproxy.get_sessions': mock}): + ret = haproxy.beacon(config) + self.assertEqual(ret, [{'threshold': 45, + 'scur': 46, + 'server': 'web1'}]) + + def test_threshold_not_reached(self): + config = [{'backends': {'www-backend': {'threshold': 100, + 'servers': ['web1'] + }}}] + ret = haproxy.validate(config) + + self.assertEqual(ret, (True, 'Valid beacon configuration')) + + mock = MagicMock(return_value=50) + with patch.dict(haproxy.__salt__, {'haproxy.get_sessions': mock}): + ret = haproxy.beacon(config) + self.assertEqual(ret, []) diff --git a/tests/unit/beacons/test_inotify_beacon.py b/tests/unit/beacons/test_inotify_beacon.py index 8f91c565c74..c071d4fde3b 100644 --- a/tests/unit/beacons/test_inotify_beacon.py +++ b/tests/unit/beacons/test_inotify_beacon.py @@ -7,7 +7,7 @@ import shutil import tempfile # Salt libs -import salt.utils +import salt.utils.files from salt.beacons import inotify # Salt testing libs @@ -20,6 +20,9 @@ try: except ImportError: HAS_PYINOTIFY = False +import logging +log = logging.getLogger(__name__) + @skipIf(not HAS_PYINOTIFY, 'pyinotify is not available') class INotifyBeaconTestCase(TestCase, LoaderModuleMockMixin): @@ -37,17 +40,20 @@ class INotifyBeaconTestCase(TestCase, LoaderModuleMockMixin): shutil.rmtree(self.tmpdir, ignore_errors=True) def test_empty_config(self): - config = {} + config = [{}] ret = inotify.beacon(config) self.assertEqual(ret, []) def test_file_open(self): path = os.path.realpath(__file__) - config = {path: {'mask': ['open']}} + config = [{'files': {path: {'mask': ['open']}}}] + ret = inotify.validate(config) + self.assertEqual(ret, (True, 'Valid beacon configuration')) + ret = inotify.beacon(config) self.assertEqual(ret, []) - with salt.utils.fopen(path, 'r') as f: + with salt.utils.files.fopen(path, 'r') as f: pass ret = inotify.beacon(config) self.assertEqual(len(ret), 1) @@ -55,27 +61,33 @@ class INotifyBeaconTestCase(TestCase, LoaderModuleMockMixin): self.assertEqual(ret[0]['change'], 'IN_OPEN') def test_dir_no_auto_add(self): - config = {self.tmpdir: {'mask': ['create']}} + config = [{'files': {self.tmpdir: {'mask': ['create']}}}] + ret = inotify.validate(config) + self.assertEqual(ret, (True, 'Valid beacon configuration')) + ret = inotify.beacon(config) self.assertEqual(ret, []) fp = os.path.join(self.tmpdir, 'tmpfile') - with salt.utils.fopen(fp, 'w') as f: + with salt.utils.files.fopen(fp, 'w') as f: pass ret = inotify.beacon(config) self.assertEqual(len(ret), 1) self.assertEqual(ret[0]['path'], fp) self.assertEqual(ret[0]['change'], 'IN_CREATE') - with salt.utils.fopen(fp, 'r') as f: + with salt.utils.files.fopen(fp, 'r') as f: pass ret = inotify.beacon(config) self.assertEqual(ret, []) def test_dir_auto_add(self): - config = {self.tmpdir: {'mask': ['create', 'open'], 'auto_add': True}} + config = [{'files': {self.tmpdir: {'mask': ['create', 'open'], 'auto_add': True}}}] + ret = inotify.validate(config) + self.assertEqual(ret, (True, 'Valid beacon configuration')) + ret = inotify.beacon(config) self.assertEqual(ret, []) fp = os.path.join(self.tmpdir, 'tmpfile') - with salt.utils.fopen(fp, 'w') as f: + with salt.utils.files.fopen(fp, 'w') as f: pass ret = inotify.beacon(config) self.assertEqual(len(ret), 2) @@ -83,7 +95,7 @@ class INotifyBeaconTestCase(TestCase, LoaderModuleMockMixin): self.assertEqual(ret[0]['change'], 'IN_CREATE') self.assertEqual(ret[1]['path'], fp) self.assertEqual(ret[1]['change'], 'IN_OPEN') - with salt.utils.fopen(fp, 'r') as f: + with salt.utils.files.fopen(fp, 'r') as f: pass ret = inotify.beacon(config) self.assertEqual(len(ret), 1) @@ -96,12 +108,15 @@ class INotifyBeaconTestCase(TestCase, LoaderModuleMockMixin): dp2 = os.path.join(dp1, 'subdir2') os.mkdir(dp2) fp = os.path.join(dp2, 'tmpfile') - with salt.utils.fopen(fp, 'w') as f: + with salt.utils.files.fopen(fp, 'w') as f: pass - config = {self.tmpdir: {'mask': ['open'], 'recurse': True}} + config = [{'files': {self.tmpdir: {'mask': ['open'], 'recurse': True}}}] + ret = inotify.validate(config) + self.assertEqual(ret, (True, 'Valid beacon configuration')) + ret = inotify.beacon(config) self.assertEqual(ret, []) - with salt.utils.fopen(fp) as f: + with salt.utils.files.fopen(fp) as f: pass ret = inotify.beacon(config) self.assertEqual(len(ret), 3) @@ -115,9 +130,12 @@ class INotifyBeaconTestCase(TestCase, LoaderModuleMockMixin): def test_dir_recurse_auto_add(self): dp1 = os.path.join(self.tmpdir, 'subdir1') os.mkdir(dp1) - config = {self.tmpdir: {'mask': ['create', 'delete'], - 'recurse': True, - 'auto_add': True}} + config = [{'files': {self.tmpdir: {'mask': ['create', 'delete'], + 'recurse': True, + 'auto_add': True}}}] + ret = inotify.validate(config) + self.assertEqual(ret, (True, 'Valid beacon configuration')) + ret = inotify.beacon(config) self.assertEqual(ret, []) dp2 = os.path.join(dp1, 'subdir2') @@ -127,7 +145,7 @@ class INotifyBeaconTestCase(TestCase, LoaderModuleMockMixin): self.assertEqual(ret[0]['path'], dp2) self.assertEqual(ret[0]['change'], 'IN_CREATE|IN_ISDIR') fp = os.path.join(dp2, 'tmpfile') - with salt.utils.fopen(fp, 'w') as f: + with salt.utils.files.fopen(fp, 'w') as f: pass ret = inotify.beacon(config) self.assertEqual(len(ret), 1) diff --git a/tests/unit/beacons/test_journald_beacon.py b/tests/unit/beacons/test_journald_beacon.py new file mode 100644 index 00000000000..7b9b109bf0f --- /dev/null +++ b/tests/unit/beacons/test_journald_beacon.py @@ -0,0 +1,120 @@ +# coding: utf-8 + +# Python libs +from __future__ import absolute_import +import datetime +from uuid import UUID + +# Salt testing libs +from tests.support.unit import skipIf, TestCase +from tests.support.mock import NO_MOCK, NO_MOCK_REASON, Mock +from tests.support.mixins import LoaderModuleMockMixin + +# Salt libs +import salt.beacons.journald as journald +import salt.utils + +import logging +log = logging.getLogger(__name__) + +_STUB_JOURNALD_ENTRY = {'_BOOT_ID': UUID('ad3915a5-9008-4fec-a635-140606525497'), + '__MONOTONIC_TIMESTAMP': (datetime.timedelta(4, 28586, 703069), + UUID('ad3915a5-9008-4fec-a635-140606525497')), + '_AUDIT_LOGINUID': 1000, 'SYSLOG_FACILITY': 10, + '_SYSTEMD_SLICE': u'system.slice', + '_GID': 0, + '__REALTIME_TIMESTAMP': datetime.datetime(2017, 6, 27, + 20, 8, 16, + 468468), + '_AUDIT_SESSION': 351, 'PRIORITY': 6, + '_TRANSPORT': u'syslog', + '_HOSTNAME': u'hostname', + '_CAP_EFFECTIVE': u'3fffffffff', + '_SYSTEMD_UNIT': u'ssh.service', + '_MACHINE_ID': UUID('14fab5bb-228d-414b-bdf4-cbc62cb7ba54'), + '_PID': 15091, + 'SYSLOG_IDENTIFIER': u'sshd', + '_SOURCE_REALTIME_TIMESTAMP': datetime.datetime(2017, + 6, + 27, + 20, + 8, + 16, + 468454), + '_SYSTEMD_CGROUP': u'/system.slice/ssh.service', + '__CURSOR': 's=7711ee01b03446309383870171dd5839;i=a74e;b=ad3915a590084feca635140606525497;m=571f43f8 dd;t=552fc7ed1cdf4;x=4ca0a3d4f1905736', + '_COMM': u'sshd', + '_CMDLINE': u'sshd: gareth [priv]', + '_SYSTEMD_INVOCATION_ID': u'38a5d5aad292426d93bfaab72a69c2ab', + '_EXE': u'/usr/sbin/sshd', + '_UID': 0, + 'SYSLOG_PID': 15091, + 'MESSAGE': u'pam_unix(sshd:session): session opened for user username by (uid=0)'} + + +class SystemdJournaldMock(Mock): + ''' Request Mock''' + + returned_once = False + + def get_next(self, *args, **kwargs): + if not self.returned_once: + self.returned_once = True + return _STUB_JOURNALD_ENTRY + else: + return None + + def seek_tail(self, *args, **kwargs): + return {} + + def get_previous(self, *args, **kwargs): + return {} + + +SYSTEMD_MOCK = SystemdJournaldMock() + + +@skipIf(NO_MOCK, NO_MOCK_REASON) +class JournaldBeaconTestCase(TestCase, LoaderModuleMockMixin): + ''' + Test case for salt.beacons.journald + ''' + + def setup_loader_modules(self): + return { + journald: { + '__context__': { + 'systemd.journald': SYSTEMD_MOCK, + }, + '__salt__': {}, + } + } + + def test_non_list_config(self): + config = {} + + ret = journald.validate(config) + + self.assertEqual(ret, (False, 'Configuration for journald beacon must' + ' be a list.')) + + def test_empty_config(self): + config = [{}] + + ret = journald.validate(config) + + self.assertEqual(ret, (True, 'Valid beacon configuration')) + + def test_journald_match(self): + config = [{'services': {'sshd': {'SYSLOG_IDENTIFIER': 'sshd', + 'PRIORITY': 6}}}] + + ret = journald.validate(config) + + self.assertEqual(ret, (True, 'Valid beacon configuration')) + + _expected_return = salt.utils.simple_types_filter(_STUB_JOURNALD_ENTRY) + _expected_return['tag'] = 'sshd' + + ret = journald.beacon(config) + self.assertEqual(ret, [_expected_return]) diff --git a/tests/unit/beacons/test_load_beacon.py b/tests/unit/beacons/test_load_beacon.py new file mode 100644 index 00000000000..37288b53ad1 --- /dev/null +++ b/tests/unit/beacons/test_load_beacon.py @@ -0,0 +1,61 @@ +# coding: utf-8 + +# Python libs +from __future__ import absolute_import + +# Salt testing libs +from tests.support.unit import skipIf, TestCase +from tests.support.mock import NO_MOCK, NO_MOCK_REASON, patch, MagicMock +from tests.support.mixins import LoaderModuleMockMixin + +# Salt libs +import salt.beacons.load as load + +import logging +log = logging.getLogger(__name__) + + +@skipIf(NO_MOCK, NO_MOCK_REASON) +class LoadBeaconTestCase(TestCase, LoaderModuleMockMixin): + ''' + Test case for salt.beacons.load + ''' + + def setup_loader_modules(self): + return { + load: { + '__context__': {}, + '__salt__': {}, + } + } + + def test_non_list_config(self): + config = {} + + ret = load.validate(config) + + self.assertEqual(ret, (False, 'Configuration for load beacon must' + ' be a list.')) + + def test_empty_config(self): + config = [{}] + + ret = load.validate(config) + + self.assertEqual(ret, (False, 'Averages configuration is required' + ' for load beacon.')) + + def test_load_match(self): + with patch('os.getloadavg', + MagicMock(return_value=(1.82, 1.84, 1.56))): + config = [{'averages': {'1m': [0.0, 2.0], + '5m': [0.0, 1.5], + '15m': [0.0, 1.0]}}] + + ret = load.validate(config) + + self.assertEqual(ret, (True, 'Valid beacon configuration')) + + _expected_return = [{'15m': 1.56, '1m': 1.82, '5m': 1.84}] + ret = load.beacon(config) + self.assertEqual(ret, _expected_return) diff --git a/tests/unit/beacons/test_log_beacon.py b/tests/unit/beacons/test_log_beacon.py new file mode 100644 index 00000000000..8551c486734 --- /dev/null +++ b/tests/unit/beacons/test_log_beacon.py @@ -0,0 +1,70 @@ +# coding: utf-8 + +# Python libs +from __future__ import absolute_import + +# Salt testing libs +from tests.support.unit import skipIf, TestCase +from tests.support.mock import NO_MOCK, NO_MOCK_REASON, patch, mock_open +from tests.support.mixins import LoaderModuleMockMixin + +# Salt libs +import salt.beacons.log as log + +import logging +_log = logging.getLogger(__name__) + + +_STUB_LOG_ENTRY = 'Jun 29 12:58:51 hostname sshd[6536]: ' \ + 'pam_unix(sshd:session): session opened ' \ + 'for user username by (uid=0)\n' + + +@skipIf(NO_MOCK, NO_MOCK_REASON) +class LogBeaconTestCase(TestCase, LoaderModuleMockMixin): + ''' + Test case for salt.beacons.log + ''' + + def setup_loader_modules(self): + return { + log: { + '__context__': {'log.loc': 2}, + '__salt__': {}, + } + } + + def test_non_list_config(self): + config = {} + + ret = log.validate(config) + + self.assertEqual(ret, (False, 'Configuration for log beacon must' + ' be a list.')) + + def test_empty_config(self): + config = [{}] + + ret = log.validate(config) + + self.assertEqual(ret, (False, 'Configuration for log beacon ' + 'must contain file option.')) + + def test_log_match(self): + with patch('salt.utils.files.fopen', + mock_open(read_data=_STUB_LOG_ENTRY)): + config = [{'file': '/var/log/auth.log', + 'tags': {'sshd': {'regex': '.*sshd.*'}} + }] + + ret = log.validate(config) + + self.assertEqual(ret, (True, 'Valid beacon configuration')) + + _expected_return = [{'error': '', + 'match': 'yes', + 'raw': _STUB_LOG_ENTRY.rstrip('\n'), + 'tag': 'sshd' + }] + ret = log.beacon(config) + self.assertEqual(ret, _expected_return) diff --git a/tests/unit/beacons/test_memusage_beacon.py b/tests/unit/beacons/test_memusage_beacon.py new file mode 100644 index 00000000000..e3219073d7a --- /dev/null +++ b/tests/unit/beacons/test_memusage_beacon.py @@ -0,0 +1,70 @@ +# coding: utf-8 + +# Python libs +from __future__ import absolute_import +from collections import namedtuple + +# Salt testing libs +from tests.support.unit import skipIf, TestCase +from tests.support.mock import NO_MOCK, NO_MOCK_REASON, patch, MagicMock +from tests.support.mixins import LoaderModuleMockMixin + +# Salt libs +import salt.beacons.memusage as memusage + +STUB_MEMORY_USAGE = namedtuple('vmem', 'total available percent used free active inactive buffers cached shared')( + 15722012672, 9329594368, 40.7, 5137018880, + 4678086656, 6991405056, 2078953472, + 1156378624, 4750528512, 898908160) + + +@skipIf(NO_MOCK, NO_MOCK_REASON) +class MemUsageBeaconTestCase(TestCase, LoaderModuleMockMixin): + ''' + Test case for salt.beacons.memusage + ''' + + def setup_loader_modules(self): + return {} + + def test_non_list_config(self): + config = {} + + ret = memusage.validate(config) + + self.assertEqual(ret, (False, 'Configuration for memusage beacon must' + ' be a list.')) + + def test_empty_config(self): + config = [{}] + + ret = memusage.validate(config) + + self.assertEqual(ret, (False, 'Configuration for memusage beacon ' + 'requires percent.')) + + def test_memusage_match(self): + with patch('psutil.virtual_memory', + MagicMock(return_value=STUB_MEMORY_USAGE)): + + config = [{'percent': '40%'}, {'interval': 30}] + + ret = memusage.validate(config) + + self.assertEqual(ret, (True, 'Valid beacon configuration')) + + ret = memusage.beacon(config) + self.assertEqual(ret, [{'memusage': 40.7}]) + + def test_memusage_nomatch(self): + with patch('psutil.virtual_memory', + MagicMock(return_value=STUB_MEMORY_USAGE)): + + config = [{'percent': '70%'}] + + ret = memusage.validate(config) + + self.assertEqual(ret, (True, 'Valid beacon configuration')) + + ret = memusage.beacon(config) + self.assertNotEqual(ret, [{'memusage': 50}]) diff --git a/tests/unit/beacons/test_network_info_beacon.py b/tests/unit/beacons/test_network_info_beacon.py new file mode 100644 index 00000000000..0b76c50ecd7 --- /dev/null +++ b/tests/unit/beacons/test_network_info_beacon.py @@ -0,0 +1,115 @@ +# coding: utf-8 + +# Python libs +from __future__ import absolute_import +from collections import namedtuple + +# Salt testing libs +from tests.support.unit import skipIf, TestCase +from tests.support.mock import NO_MOCK, NO_MOCK_REASON, patch, MagicMock +from tests.support.mixins import LoaderModuleMockMixin + +# Salt libs +import salt.beacons.network_info as network_info + +import logging +log = logging.getLogger(__name__) + +STUB_NET_IO_COUNTERS = {'eth0': namedtuple('snetio', + 'bytes_sent bytes_recv \ + packets_sent packets_recv \ + errin errout \ + dropin \ + dropout')(93662618, 914626664, + 465694, 903802, 0, + 0, 0, 0)} + + +@skipIf(NO_MOCK, NO_MOCK_REASON) +class NetworkInfoBeaconTestCase(TestCase, LoaderModuleMockMixin): + ''' + Test case for salt.beacons.network_info + ''' + + def setup_loader_modules(self): + return { + network_info: { + '__context__': {}, + '__salt__': {}, + } + } + + def test_non_list_config(self): + config = {} + + ret = network_info.validate(config) + + self.assertEqual(ret, (False, 'Configuration for network_info beacon' + ' must be a list.')) + + def test_empty_config(self): + config = [{}] + + ret = network_info.validate(config) + + self.assertEqual(ret, (True, 'Valid beacon configuration')) + + def test_network_info_equal(self): + with patch('salt.utils.psutil_compat.net_io_counters', + MagicMock(return_value=STUB_NET_IO_COUNTERS)): + config = [{'interfaces': {'eth0': {'type': 'equal', + 'bytes_sent': 914626664, + 'bytes_recv': 93662618, + 'packets_sent': 465694, + 'packets_recv': 903802, + 'errin': 0, + 'errout': 0, + 'dropin': 0, + 'dropout': 0}}}] + + ret = network_info.validate(config) + + self.assertEqual(ret, (True, 'Valid beacon configuration')) + + _expected_return = [{'interface': 'eth0', + 'network_info': {'bytes_recv': 914626664, + 'bytes_sent': 93662618, + 'dropin': 0, + 'dropout': 0, + 'errin': 0, + 'errout': 0, + 'packets_recv': 903802, + 'packets_sent': 465694}}] + + ret = network_info.beacon(config) + self.assertEqual(ret, _expected_return) + + def test_network_info_greater_than(self): + with patch('salt.utils.psutil_compat.net_io_counters', + MagicMock(return_value=STUB_NET_IO_COUNTERS)): + config = [{'interfaces': {'eth0': {'type': 'greater', + 'bytes_sent': 100000, + 'bytes_recv': 100000, + 'packets_sent': 100000, + 'packets_recv': 100000, + 'errin': 0, + 'errout': 0, + 'dropin': 0, + 'dropout': 0}}}] + + ret = network_info.validate(config) + + self.assertEqual(ret, (True, 'Valid beacon configuration')) + + _expected_return = [{'interface': 'eth0', + 'network_info': {'bytes_recv': 914626664, + 'bytes_sent': 93662618, + 'dropin': 0, + 'dropout': 0, + 'errin': 0, + 'errout': 0, + 'packets_recv': 903802, + 'packets_sent': 465694}}] + + ret = network_info.beacon(config) + self.assertEqual(ret, _expected_return) diff --git a/tests/unit/beacons/test_network_settings_beacon.py b/tests/unit/beacons/test_network_settings_beacon.py new file mode 100644 index 00000000000..c09e2f33fb2 --- /dev/null +++ b/tests/unit/beacons/test_network_settings_beacon.py @@ -0,0 +1,45 @@ +# coding: utf-8 + +# Python libs +from __future__ import absolute_import + +# Salt testing libs +from tests.support.unit import skipIf, TestCase +from tests.support.mock import NO_MOCK, NO_MOCK_REASON +from tests.support.mixins import LoaderModuleMockMixin + +# Salt libs +import salt.beacons.network_settings as network_settings + +import logging +log = logging.getLogger(__name__) + + +@skipIf(NO_MOCK, NO_MOCK_REASON) +class NetworkSettingsBeaconTestCase(TestCase, LoaderModuleMockMixin): + ''' + Test case for salt.beacons.network_settings + ''' + + def setup_loader_modules(self): + return { + network_settings: { + '__context__': {}, + '__salt__': {}, + } + } + + def test_non_list_config(self): + config = {} + + ret = network_settings.validate(config) + + self.assertEqual(ret, (False, 'Configuration for network_settings' + ' beacon must be a list.')) + + def test_empty_config(self): + config = [{}] + + ret = network_settings.validate(config) + + self.assertEqual(ret, (True, 'Valid beacon configuration')) diff --git a/tests/unit/beacons/test_ps_beacon.py b/tests/unit/beacons/test_ps_beacon.py new file mode 100644 index 00000000000..2ae5a5735bc --- /dev/null +++ b/tests/unit/beacons/test_ps_beacon.py @@ -0,0 +1,76 @@ +# coding: utf-8 + +# Python libs +from __future__ import absolute_import + +# Salt testing libs +from tests.support.unit import skipIf, TestCase +from tests.support.mock import NO_MOCK, NO_MOCK_REASON, patch +from tests.support.mixins import LoaderModuleMockMixin + +# Salt libs +import salt.beacons.ps as ps + +PATCH_OPTS = dict(autospec=True, spec_set=True) + + +class FakeProcess(object): + + def __init__(self, _name, pid): + self._name = _name + self.pid = pid + + def name(self): + return self._name + + +@skipIf(NO_MOCK, NO_MOCK_REASON) +class PSBeaconTestCase(TestCase, LoaderModuleMockMixin): + ''' + Test case for salt.beacons.[s] + ''' + + def setup_loader_modules(self): + return {} + + def test_non_list_config(self): + config = {} + + ret = ps.validate(config) + + self.assertEqual(ret, (False, 'Configuration for ps beacon must' + ' be a list.')) + + def test_empty_config(self): + config = [{}] + + ret = ps.validate(config) + + self.assertEqual(ret, (False, 'Configuration for ps ' + 'beacon requires processes.')) + + def test_ps_running(self): + with patch('salt.utils.psutil_compat.process_iter', **PATCH_OPTS) as mock_process_iter: + mock_process_iter.return_value = [FakeProcess(_name='salt-master', pid=3), + FakeProcess(_name='salt-minion', pid=4)] + config = [{'processes': {'salt-master': 'running'}}] + + ret = ps.validate(config) + + self.assertEqual(ret, (True, 'Valid beacon configuration')) + + ret = ps.beacon(config) + self.assertEqual(ret, [{'salt-master': 'Running'}]) + + def test_ps_not_running(self): + with patch('salt.utils.psutil_compat.process_iter', **PATCH_OPTS) as mock_process_iter: + mock_process_iter.return_value = [FakeProcess(_name='salt-master', pid=3), + FakeProcess(_name='salt-minion', pid=4)] + config = [{'processes': {'mysql': 'stopped'}}] + + ret = ps.validate(config) + + self.assertEqual(ret, (True, 'Valid beacon configuration')) + + ret = ps.beacon(config) + self.assertEqual(ret, [{'mysql': 'Stopped'}]) diff --git a/tests/unit/beacons/test_salt_proxy_beacon.py b/tests/unit/beacons/test_salt_proxy_beacon.py new file mode 100644 index 00000000000..d02c75cdb88 --- /dev/null +++ b/tests/unit/beacons/test_salt_proxy_beacon.py @@ -0,0 +1,80 @@ +# coding: utf-8 + +# Python libs +from __future__ import absolute_import +from collections import namedtuple + +# Salt testing libs +from tests.support.unit import skipIf, TestCase +from tests.support.mock import NO_MOCK, NO_MOCK_REASON, patch, MagicMock +from tests.support.mixins import LoaderModuleMockMixin + +# Salt libs +import salt.beacons.salt_proxy as salt_proxy + +PATCH_OPTS = dict(autospec=True, spec_set=True) + +FakeProcess = namedtuple('Process', 'cmdline pid') + + +@skipIf(NO_MOCK, NO_MOCK_REASON) +class SaltProxyBeaconTestCase(TestCase, LoaderModuleMockMixin): + ''' + Test case for salt.beacons.[s] + ''' + + def setup_loader_modules(self): + return { + salt_proxy: { + '__context__': {}, + '__salt__': {}, + } + } + + def test_non_list_config(self): + config = {} + + ret = salt_proxy.validate(config) + + self.assertEqual(ret, (False, 'Configuration for salt_proxy beacon' + ' must be a list.')) + + def test_empty_config(self): + config = [{}] + + ret = salt_proxy.validate(config) + + self.assertEqual(ret, (False, 'Configuration for salt_proxy ' + 'beacon requires proxies.')) + + def test_salt_proxy_running(self): + mock = MagicMock(return_value={'result': True}) + with patch.dict(salt_proxy.__salt__, {'salt_proxy.is_running': mock}): + config = [{'proxies': {'p8000': ''}}] + + ret = salt_proxy.validate(config) + + ret = salt_proxy.beacon(config) + self.assertEqual(ret, [{'p8000': 'Proxy p8000 is already running'}]) + + def test_salt_proxy_not_running(self): + is_running_mock = MagicMock(return_value={'result': False}) + configure_mock = MagicMock(return_value={'result': True, + 'changes': {'new': 'Salt Proxy: Started proxy process for p8000', + 'old': []}}) + cmd_run_mock = MagicMock(return_value={'pid': 1000, + 'retcode': 0, + 'stderr': '', + 'stdout': ''}) + with patch.dict(salt_proxy.__salt__, + {'salt_proxy.is_running': is_running_mock}), \ + patch.dict(salt_proxy.__salt__, + {'salt_proxy.configure_proxy': configure_mock}), \ + patch.dict(salt_proxy.__salt__, + {'cmd.run_all': cmd_run_mock}): + config = [{'proxies': {'p8000': ''}}] + + ret = salt_proxy.validate(config) + + ret = salt_proxy.beacon(config) + self.assertEqual(ret, [{'p8000': 'Proxy p8000 was started'}]) diff --git a/tests/unit/beacons/test_sensehat_beacon.py b/tests/unit/beacons/test_sensehat_beacon.py new file mode 100644 index 00000000000..bd36ee500c0 --- /dev/null +++ b/tests/unit/beacons/test_sensehat_beacon.py @@ -0,0 +1,104 @@ +# coding: utf-8 + +# Python libs +from __future__ import absolute_import + +# Salt testing libs +from tests.support.unit import skipIf, TestCase +from tests.support.mock import NO_MOCK, NO_MOCK_REASON, MagicMock +from tests.support.mixins import LoaderModuleMockMixin + +# Salt libs +import salt.beacons.sensehat as sensehat + + +@skipIf(NO_MOCK, NO_MOCK_REASON) +class SensehatBeaconTestCase(TestCase, LoaderModuleMockMixin): + ''' + Test case for salt.beacons.[s] + ''' + + def setup_loader_modules(self): + + self.HUMIDITY_MOCK = MagicMock(return_value=80) + self.TEMPERATURE_MOCK = MagicMock(return_value=30) + self.PRESSURE_MOCK = MagicMock(return_value=1500) + + self.addCleanup(delattr, self, 'HUMIDITY_MOCK') + self.addCleanup(delattr, self, 'TEMPERATURE_MOCK') + self.addCleanup(delattr, self, 'PRESSURE_MOCK') + + return { + sensehat: { + '__context__': {}, + '__salt__': {'sensehat.get_humidity': self.HUMIDITY_MOCK, + 'sensehat.get_temperature': self.TEMPERATURE_MOCK, + 'sensehat.get_pressure': self.PRESSURE_MOCK + }, + } + } + + def test_non_list_config(self): + config = {} + + ret = sensehat.validate(config) + + self.assertEqual(ret, (False, 'Configuration for sensehat beacon' + ' must be a list.')) + + def test_empty_config(self): + config = [{}] + + ret = sensehat.validate(config) + + self.assertEqual(ret, (False, 'Configuration for sensehat ' + 'beacon requires sensors.')) + + def test_sensehat_humidity_match(self): + + config = [{'sensors': {'humidity': '70%'}}] + + ret = sensehat.validate(config) + + ret = sensehat.beacon(config) + self.assertEqual(ret, [{'tag': 'sensehat/humidity', + 'humidity': 80}]) + + def test_sensehat_temperature_match(self): + + config = [{'sensors': {'temperature': 20}}] + + ret = sensehat.validate(config) + + ret = sensehat.beacon(config) + self.assertEqual(ret, [{'tag': 'sensehat/temperature', + 'temperature': 30}]) + + def test_sensehat_temperature_match_range(self): + + config = [{'sensors': {'temperature': [20, 29]}}] + + ret = sensehat.validate(config) + + ret = sensehat.beacon(config) + self.assertEqual(ret, [{'tag': 'sensehat/temperature', + 'temperature': 30}]) + + def test_sensehat_pressure_match(self): + + config = [{'sensors': {'pressure': '1400'}}] + + ret = sensehat.validate(config) + + ret = sensehat.beacon(config) + self.assertEqual(ret, [{'tag': 'sensehat/pressure', + 'pressure': 1500}]) + + def test_sensehat_no_match(self): + + config = [{'sensors': {'pressure': '1600'}}] + + ret = sensehat.validate(config) + + ret = sensehat.beacon(config) + self.assertEqual(ret, []) diff --git a/tests/unit/beacons/test_service_beacon.py b/tests/unit/beacons/test_service_beacon.py new file mode 100644 index 00000000000..3f9ac318dc3 --- /dev/null +++ b/tests/unit/beacons/test_service_beacon.py @@ -0,0 +1,76 @@ +# coding: utf-8 + +# Python libs +from __future__ import absolute_import +from collections import namedtuple + +# Salt testing libs +from tests.support.unit import skipIf, TestCase +from tests.support.mock import NO_MOCK, NO_MOCK_REASON, patch, MagicMock +from tests.support.mixins import LoaderModuleMockMixin + +# Salt libs +import salt.beacons.service as service_beacon + +PATCH_OPTS = dict(autospec=True, spec_set=True) + +FakeProcess = namedtuple('Process', 'cmdline pid') + + +@skipIf(NO_MOCK, NO_MOCK_REASON) +class ServiceBeaconTestCase(TestCase, LoaderModuleMockMixin): + ''' + Test case for salt.beacons.service + ''' + + def setup_loader_modules(self): + return { + service_beacon: { + '__context__': {}, + '__salt__': {}, + } + } + + def test_non_list_config(self): + config = {} + + ret = service_beacon.validate(config) + + self.assertEqual(ret, (False, 'Configuration for service beacon must' + ' be a list.')) + + def test_empty_config(self): + config = [{}] + + ret = service_beacon.validate(config) + + self.assertEqual(ret, (False, 'Configuration for service ' + 'beacon requires services.')) + + def test_service_running(self): + with patch.dict(service_beacon.__salt__, + {'service.status': MagicMock(return_value=True)}): + config = [{'services': {'salt-master': {}}}] + + ret = service_beacon.validate(config) + + self.assertEqual(ret, (True, 'Valid beacon configuration')) + + ret = service_beacon.beacon(config) + self.assertEqual(ret, [{'service_name': 'salt-master', + 'tag': 'salt-master', + 'salt-master': {'running': True}}]) + + def test_service_not_running(self): + with patch.dict(service_beacon.__salt__, + {'service.status': MagicMock(return_value=False)}): + config = [{'services': {'salt-master': {}}}] + + ret = service_beacon.validate(config) + + self.assertEqual(ret, (True, 'Valid beacon configuration')) + + ret = service_beacon.beacon(config) + self.assertEqual(ret, [{'service_name': 'salt-master', + 'tag': 'salt-master', + 'salt-master': {'running': False}}]) diff --git a/tests/unit/beacons/test_status.py b/tests/unit/beacons/test_status_beacon.py similarity index 100% rename from tests/unit/beacons/test_status.py rename to tests/unit/beacons/test_status_beacon.py diff --git a/tests/unit/beacons/test_telegram_bot_msg_beacon.py b/tests/unit/beacons/test_telegram_bot_msg_beacon.py index 06efe6333ec..837f0ac7bc5 100644 --- a/tests/unit/beacons/test_telegram_bot_msg_beacon.py +++ b/tests/unit/beacons/test_telegram_bot_msg_beacon.py @@ -19,6 +19,9 @@ try: except ImportError: HAS_TELEGRAM = False +import logging +log = logging.getLogger(__name__) + @skipIf(not HAS_TELEGRAM, 'telegram is not available') @skipIf(NO_MOCK, NO_MOCK_REASON) @@ -30,49 +33,49 @@ class TelegramBotMsgBeaconTestCase(TestCase, LoaderModuleMockMixin): return {telegram_bot_msg: {}} def test_validate_empty_config(self, *args, **kwargs): - ret = telegram_bot_msg.__validate__(None) + ret = telegram_bot_msg.validate(None) self.assertEqual(ret, (False, ('Configuration for telegram_bot_msg ' - 'beacon must be a dictionary.'))) + 'beacon must be a list.'))) def test_validate_missing_accept_from_config(self, *args, **kwargs): - ret = telegram_bot_msg.__validate__({ + ret = telegram_bot_msg.validate([{ 'token': 'bcd' - }) + }]) self.assertEqual(ret, (False, ('Not all required configuration for ' 'telegram_bot_msg are set.'))) def test_validate_missing_token_config(self, *args, **kwargs): - ret = telegram_bot_msg.__validate__({ + ret = telegram_bot_msg.validate([{ 'accept_from': [] - }) + }]) self.assertEqual(ret, (False, ('Not all required configuration for ' 'telegram_bot_msg are set.'))) def test_validate_config_not_list_in_accept_from(self, *args, **kwargs): - ret = telegram_bot_msg.__validate__({ + ret = telegram_bot_msg.validate([{ 'token': 'bcd', 'accept_from': {'nodict': "1"} - }) + }]) self.assertEqual(ret, (False, ('Configuration for telegram_bot_msg, ' 'accept_from must be a list of ' 'usernames.'))) def test_validate_valid_config(self, *args, **kwargs): - ret = telegram_bot_msg.__validate__({ + ret = telegram_bot_msg.validate([{ 'token': 'bcd', 'accept_from': [ 'username' ] - }) + }]) self.assertEqual(ret, (True, 'Valid beacon configuration.')) def test_call_no_updates(self): with patch("salt.beacons.telegram_bot_msg.telegram") as telegram_api: token = 'abc' - config = { + config = [{ 'token': token, 'accept_from': ['tester'] - } + }] inst = MagicMock(name='telegram.Bot()') telegram_api.Bot = MagicMock(name='telegram', return_value=inst) inst.get_updates.return_value = [] @@ -86,18 +89,19 @@ class TelegramBotMsgBeaconTestCase(TestCase, LoaderModuleMockMixin): with patch("salt.beacons.telegram_bot_msg.telegram") as telegram_api: token = 'abc' username = 'tester' - config = { + config = [{ 'token': token, 'accept_from': [username] - } + }] inst = MagicMock(name='telegram.Bot()') telegram_api.Bot = MagicMock(name='telegram', return_value=inst) + log.debug('telegram {}'.format(telegram)) username = 'different_user' - user = telegram.User(id=1, first_name='', username=username) - chat = telegram.Chat(1, 'private', username=username) + user = telegram.user.User(id=1, first_name='', username=username) + chat = telegram.chat.Chat(1, 'private', username=username) date = datetime.datetime(2016, 12, 18, 0, 0) - message = telegram.Message(1, user, date=date, chat=chat) + message = telegram.message.Message(1, user, date=date, chat=chat) update = telegram.update.Update(update_id=1, message=message) inst.get_updates.return_value = [update] @@ -111,10 +115,10 @@ class TelegramBotMsgBeaconTestCase(TestCase, LoaderModuleMockMixin): with patch("salt.beacons.telegram_bot_msg.telegram") as telegram_api: token = 'abc' username = 'tester' - config = { + config = [{ 'token': token, 'accept_from': [username] - } + }] inst = MagicMock(name='telegram.Bot()') telegram_api.Bot = MagicMock(name='telegram', return_value=inst) diff --git a/tests/unit/beacons/test_twilio_txt_msg_beacon.py b/tests/unit/beacons/test_twilio_txt_msg_beacon.py new file mode 100644 index 00000000000..5223e06d28d --- /dev/null +++ b/tests/unit/beacons/test_twilio_txt_msg_beacon.py @@ -0,0 +1,165 @@ +# coding: utf-8 + +# Python libs +from __future__ import absolute_import + +# Salt libs +from salt.beacons import twilio_txt_msg + +# Salt testing libs +from tests.support.mixins import LoaderModuleMockMixin +from tests.support.unit import TestCase, skipIf +from tests.support.mock import NO_MOCK, NO_MOCK_REASON, MagicMock, patch + +# Import 3rd Party libs +try: + import twilio + # Grab version, ensure elements are ints + twilio_version = tuple([int(x) for x in twilio.__version_info__]) + if twilio_version > (5, ): + TWILIO_5 = False + else: + TWILIO_5 = True + HAS_TWILIO = True +except ImportError: + HAS_TWILIO = False + +import logging +log = logging.getLogger(__name__) + + +class MockTwilioRestException(Exception): + ''' + Mock TwilioRestException class + ''' + def __init__(self): + self.code = 'error code' + self.msg = 'Exception error' + self.status = 'Not send' + super(MockTwilioRestException, self).__init__(self.msg) + + +class MockMessages(object): + ''' + Mock SMS class + ''' + flag = None + + def __init__(self): + self.sid = '011' + self.price = '200' + self.price_unit = '1' + self.status = 'Sent' + self.num_segments = '2' + self.num_media = '0' + self.body = None + self.date_sent = '01-01-2015' + self.date_created = '01-01-2015' + self.to = None + self.from_ = None + + def create(self, body, to, from_): + ''' + Mock create method + ''' + msg = MockMessages() + if self.flag == 1: + raise MockTwilioRestException() + msg.body = body + msg.to = to + msg.from_ = from_ + return msg + + def list(self, to): + ''' + Mock list method + ''' + msg = MockMessages() + return [msg] + + def delete(self): + ''' + Mock delete method + ''' + return None + + +class MockSMS(object): + ''' + Mock SMS class + ''' + def __init__(self): + self.messages = MockMessages() + + +class MockTwilioRestClient(object): + ''' + Mock TwilioRestClient class + ''' + def __init__(self): + if TWILIO_5: + self.sms = MockSMS() + else: + self.messages = MockMessages() + + +@skipIf(not HAS_TWILIO, 'twilio.rest is not available') +@skipIf(NO_MOCK, NO_MOCK_REASON) +class TwilioMsgTxtBeaconTestCase(TestCase, LoaderModuleMockMixin): + ''' + Test case for salt.beacons.twilio_txt_msg + ''' + def setup_loader_modules(self): + return {twilio_txt_msg: {}} + + def test_validate_dictionary_config(self): + ''' + Test empty configuration + ''' + config = {} + ret = twilio_txt_msg.validate(config) + self.assertEqual(ret, (False, ('Configuration for twilio_txt_msg ' + 'beacon must be a list.'))) + + def test_validate_empty_config(self): + ''' + Test empty configuration + ''' + config = [{}] + ret = twilio_txt_msg.validate(config) + self.assertEqual(ret, (False, ('Configuration for twilio_txt_msg ' + 'beacon must contain account_sid, ' + 'auth_token and twilio_number items.'))) + + def test_validate_missing_config_item(self): + ''' + Test empty configuration + ''' + config = [{'account_sid': 'ACXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', + 'twilio_number': '+15555555555'}] + + ret = twilio_txt_msg.validate(config) + self.assertEqual(ret, (False, ('Configuration for twilio_txt_msg ' + 'beacon must contain account_sid, ' + 'auth_token and twilio_number items.'))) + + def test_receive_message(self): + ''' + Test receive a message + ''' + config = [{'account_sid': 'ACXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX', + 'auth_token': 'my_token', + 'twilio_number': '+15555555555'}] + + ret = twilio_txt_msg.validate(config) + self.assertEqual(ret, (True, 'Valid beacon configuration')) + + _expected_return = [{'texts': [{'body': 'None', + 'images': [], + 'from': 'None', + 'id': '011', + 'sent': '01-01-2015'}]}] + mock = MagicMock(return_value=MockTwilioRestClient()) + with patch.object(twilio_txt_msg, 'TwilioRestClient', mock): + ret = twilio_txt_msg.beacon(config) + self.assertEqual(ret, _expected_return) diff --git a/tests/unit/cache/test_localfs.py b/tests/unit/cache/test_localfs.py index 3aa2608a79c..dcaae493e02 100644 --- a/tests/unit/cache/test_localfs.py +++ b/tests/unit/cache/test_localfs.py @@ -21,12 +21,12 @@ from tests.support.mock import ( # Import Salt libs import salt.payload -import salt.utils +import salt.utils.files import salt.cache.localfs as localfs from salt.exceptions import SaltCacheError # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six @skipIf(NO_MOCK, NO_MOCK_REASON) @@ -62,7 +62,7 @@ class LocalFSTest(TestCase, LoaderModuleMockMixin): def test_store_close_mkstemp_file_handle(self): ''' Tests that the file descriptor that is opened by os.open during the mkstemp call - in localfs.store is closed before calling salt.utils.fopen on the filename. + in localfs.store is closed before calling salt.utils.files.fopen on the filename. This test mocks the call to mkstemp, but forces an OSError to be raised when the close() function is called on a file descriptor that doesn't exist. @@ -79,7 +79,7 @@ class LocalFSTest(TestCase, LoaderModuleMockMixin): with patch('os.path.isdir', MagicMock(return_value=True)): with patch('tempfile.mkstemp', MagicMock(return_value=('one', 'two'))): with patch('os.close', MagicMock(return_value=None)): - with patch('salt.utils.fopen', MagicMock(side_effect=IOError)): + with patch('salt.utils.files.fopen', MagicMock(side_effect=IOError)): self.assertRaises(SaltCacheError, localfs.store, bank='', key='', data='', cachedir='') def test_store_success(self): @@ -93,7 +93,7 @@ class LocalFSTest(TestCase, LoaderModuleMockMixin): self._create_tmp_cache_file(tmp_dir, salt.payload.Serial(self)) # Read in the contents of the key.p file and assert "payload data" was written - with salt.utils.fopen(tmp_dir + '/bank/key.p', 'rb') as fh_: + with salt.utils.files.fopen(tmp_dir + '/bank/key.p', 'rb') as fh_: for line in fh_: self.assertIn(six.b('payload data'), line) @@ -113,7 +113,7 @@ class LocalFSTest(TestCase, LoaderModuleMockMixin): file. ''' with patch('os.path.isfile', MagicMock(return_value=True)): - with patch('salt.utils.fopen', MagicMock(side_effect=IOError)): + with patch('salt.utils.files.fopen', MagicMock(side_effect=IOError)): self.assertRaises(SaltCacheError, localfs.fetch, bank='', key='', cachedir='') def test_fetch_success(self): @@ -209,26 +209,26 @@ class LocalFSTest(TestCase, LoaderModuleMockMixin): with patch('os.remove', MagicMock(side_effect=OSError)): self.assertRaises(SaltCacheError, localfs.flush, bank='', key='key', cachedir='/var/cache/salt') - # 'ls' function tests: 3 + # 'list' function tests: 3 - def test_ls_no_base_dir(self): + def test_list_no_base_dir(self): ''' Tests that the ls function returns an empty list if the bank directory doesn't exist. ''' with patch('os.path.isdir', MagicMock(return_value=False)): - self.assertEqual(localfs.ls(bank='', cachedir=''), []) + self.assertEqual(localfs.list_(bank='', cachedir=''), []) - def test_ls_error_raised_no_bank_directory_access(self): + def test_list_error_raised_no_bank_directory_access(self): ''' Tests that a SaltCacheError is raised when there is a problem accessing the cache bank directory. ''' with patch('os.path.isdir', MagicMock(return_value=True)): with patch('os.listdir', MagicMock(side_effect=OSError)): - self.assertRaises(SaltCacheError, localfs.ls, bank='', cachedir='') + self.assertRaises(SaltCacheError, localfs.list_, bank='', cachedir='') - def test_ls_success(self): + def test_list_success(self): ''' Tests the return of the ls function containing bank entries. ''' @@ -240,7 +240,7 @@ class LocalFSTest(TestCase, LoaderModuleMockMixin): # Now test the return of the ls function with patch.dict(localfs.__opts__, {'cachedir': tmp_dir}): - self.assertEqual(localfs.ls(bank='bank', cachedir=tmp_dir), ['key']) + self.assertEqual(localfs.list_(bank='bank', cachedir=tmp_dir), ['key']) # 'contains' function tests: 1 diff --git a/tests/unit/cloud/clouds/test_ec2.py b/tests/unit/cloud/clouds/test_ec2.py index 9ffd74d47b1..4f77b14a1b4 100644 --- a/tests/unit/cloud/clouds/test_ec2.py +++ b/tests/unit/cloud/clouds/test_ec2.py @@ -2,42 +2,39 @@ # Import Python libs from __future__ import absolute_import -import os -import tempfile # Import Salt Libs from salt.cloud.clouds import ec2 from salt.exceptions import SaltCloudSystemExit # Import Salt Testing Libs -from tests.support.mixins import LoaderModuleMockMixin from tests.support.unit import TestCase, skipIf -from tests.support.mock import NO_MOCK, NO_MOCK_REASON +from tests.support.mock import NO_MOCK, NO_MOCK_REASON, patch, PropertyMock @skipIf(NO_MOCK, NO_MOCK_REASON) -class EC2TestCase(TestCase, LoaderModuleMockMixin): +class EC2TestCase(TestCase): ''' Unit TestCase for salt.cloud.clouds.ec2 module. ''' - def setup_loader_modules(self): - return {ec2: {}} - def test__validate_key_path_and_mode(self): - with tempfile.NamedTemporaryFile() as f: - key_file = f.name - os.chmod(key_file, 0o644) - self.assertRaises(SaltCloudSystemExit, - ec2._validate_key_path_and_mode, - key_file) - os.chmod(key_file, 0o600) - self.assertTrue(ec2._validate_key_path_and_mode(key_file)) - os.chmod(key_file, 0o400) - self.assertTrue(ec2._validate_key_path_and_mode(key_file)) + # Key file exists + with patch('os.path.exists', return_value=True): + with patch('os.stat') as patched_stat: - # tmp file removed - self.assertRaises(SaltCloudSystemExit, - ec2._validate_key_path_and_mode, - key_file) + type(patched_stat.return_value).st_mode = PropertyMock(return_value=0o644) + self.assertRaises( + SaltCloudSystemExit, ec2._validate_key_path_and_mode, 'key_file') + + type(patched_stat.return_value).st_mode = PropertyMock(return_value=0o600) + self.assertTrue(ec2._validate_key_path_and_mode('key_file')) + + type(patched_stat.return_value).st_mode = PropertyMock(return_value=0o400) + self.assertTrue(ec2._validate_key_path_and_mode('key_file')) + + # Key file does not exist + with patch('os.path.exists', return_value=False): + self.assertRaises( + SaltCloudSystemExit, ec2._validate_key_path_and_mode, 'key_file') diff --git a/tests/unit/cloud/clouds/test_gce.py b/tests/unit/cloud/clouds/test_gce.py index e81c24373bd..773190aa757 100644 --- a/tests/unit/cloud/clouds/test_gce.py +++ b/tests/unit/cloud/clouds/test_gce.py @@ -105,7 +105,7 @@ class GCETestCase(TestCase, LoaderModuleMockMixin): get_deps = gce.get_dependencies() self.assertEqual(get_deps, True) if LooseVersion(mock_version) >= LooseVersion('2.0.0'): - p.assert_called_once() + self.assert_called_once(p) def test_provider_matches(self): """ diff --git a/tests/unit/config/test_api.py b/tests/unit/config/test_api.py index 9cc629d2ff1..b7e2d5d7a76 100644 --- a/tests/unit/config/test_api.py +++ b/tests/unit/config/test_api.py @@ -18,12 +18,22 @@ from tests.support.mock import ( # Import Salt libs import salt.config +import salt.utils.platform +import salt.syspaths MOCK_MASTER_DEFAULT_OPTS = { - 'log_file': '/var/log/salt/master', - 'pidfile': '/var/run/salt-master.pid', - 'root_dir': '/' + 'log_file': '{0}/var/log/salt/master'.format(salt.syspaths.ROOT_DIR), + 'pidfile': '{0}/var/run/salt-master.pid'.format(salt.syspaths.ROOT_DIR), + 'root_dir': format(salt.syspaths.ROOT_DIR) } +if salt.utils.platform.is_windows(): + MOCK_MASTER_DEFAULT_OPTS = { + 'log_file': '{0}\\var\\log\\salt\\master'.format( + salt.syspaths.ROOT_DIR), + 'pidfile': '{0}\\var\\run\\salt-master.pid'.format( + salt.syspaths.ROOT_DIR), + 'root_dir': format(salt.syspaths.ROOT_DIR) + } @skipIf(NO_MOCK, NO_MOCK_REASON) @@ -46,8 +56,15 @@ class APIConfigTestCase(TestCase): the DEFAULT_API_OPTS 'api_logfile' value. ''' with patch('salt.config.client_config', MagicMock(return_value=MOCK_MASTER_DEFAULT_OPTS)): + + expected = '{0}/var/log/salt/api'.format( + salt.syspaths.ROOT_DIR if salt.syspaths.ROOT_DIR != '/' else '') + if salt.utils.platform.is_windows(): + expected = '{0}\\var\\log\\salt\\api'.format( + salt.syspaths.ROOT_DIR) + ret = salt.config.api_config('/some/fake/path') - self.assertEqual(ret['log_file'], '/var/log/salt/api') + self.assertEqual(ret['log_file'], expected) def test_api_config_pidfile_values(self): ''' @@ -56,8 +73,15 @@ class APIConfigTestCase(TestCase): the DEFAULT_API_OPTS 'api_pidfile' value. ''' with patch('salt.config.client_config', MagicMock(return_value=MOCK_MASTER_DEFAULT_OPTS)): + + expected = '{0}/var/run/salt-api.pid'.format( + salt.syspaths.ROOT_DIR if salt.syspaths.ROOT_DIR != '/' else '') + if salt.utils.platform.is_windows(): + expected = '{0}\\var\\run\\salt-api.pid'.format( + salt.syspaths.ROOT_DIR) + ret = salt.config.api_config('/some/fake/path') - self.assertEqual(ret['pidfile'], '/var/run/salt-api.pid') + self.assertEqual(ret['pidfile'], expected) @destructiveTest def test_master_config_file_overrides_defaults(self): @@ -68,6 +92,10 @@ class APIConfigTestCase(TestCase): ''' foo_dir = '/foo/bar/baz' hello_dir = '/hello/world' + if salt.utils.platform.is_windows(): + foo_dir = 'c:\\foo\\bar\\baz' + hello_dir = 'c:\\hello\\world' + mock_master_config = { 'api_pidfile': foo_dir, 'api_logfile': hello_dir, @@ -98,6 +126,11 @@ class APIConfigTestCase(TestCase): mock_master_config = MOCK_MASTER_DEFAULT_OPTS.copy() mock_master_config['root_dir'] = '/mock/root/' + if salt.utils.platform.is_windows(): + mock_log = 'c:\\mock\\root\\var\\log\\salt\\api' + mock_pid = 'c:\\mock\\root\\var\\run\\salt-api.pid' + mock_master_config['root_dir'] = 'c:\\mock\\root' + with patch('salt.config.client_config', MagicMock(return_value=mock_master_config)): ret = salt.config.api_config('/some/fake/path') diff --git a/tests/unit/config/test_config.py b/tests/unit/config/test_config.py index 42bdaf29971..948d2ee35ec 100644 --- a/tests/unit/config/test_config.py +++ b/tests/unit/config/test_config.py @@ -23,6 +23,7 @@ from tests.support.mock import NO_MOCK, NO_MOCK_REASON, MagicMock, patch # Import Salt libs import salt.minion import salt.utils +import salt.utils.files import salt.utils.network from salt.syspaths import CONFIG_DIR from salt import config as sconfig @@ -62,7 +63,7 @@ DEFAULT = {'default_include': PATH} def _unhandled_mock_read(filename): ''' - Raise an error because we should not be calling salt.utils.fopen() + Raise an error because we should not be calling salt.utils.files.fopen() ''' raise CommandExecutionError('Unhandled mock read for {0}'.format(filename)) @@ -79,7 +80,7 @@ class ConfigTestCase(TestCase, AdaptedConfigurationTestCaseMixin): def test_sha256_is_default_for_master(self): fpath = tempfile.mktemp() try: - with salt.utils.fopen(fpath, 'w') as wfh: + with salt.utils.files.fopen(fpath, 'w') as wfh: wfh.write( "root_dir: /\n" "key_logfile: key\n" @@ -93,7 +94,7 @@ class ConfigTestCase(TestCase, AdaptedConfigurationTestCaseMixin): def test_sha256_is_default_for_minion(self): fpath = tempfile.mktemp() try: - with salt.utils.fopen(fpath, 'w') as wfh: + with salt.utils.files.fopen(fpath, 'w') as wfh: wfh.write( "root_dir: /\n" "key_logfile: key\n" @@ -106,17 +107,26 @@ class ConfigTestCase(TestCase, AdaptedConfigurationTestCaseMixin): def test_proper_path_joining(self): fpath = tempfile.mktemp() + temp_config = 'root_dir: /\n'\ + 'key_logfile: key\n' + if salt.utils.platform.is_windows(): + temp_config = 'root_dir: c:\\\n'\ + 'key_logfile: key\n' try: - with salt.utils.fopen(fpath, 'w') as fp_: - fp_.write( - 'root_dir: /\n' - 'key_logfile: key\n' - ) + with salt.utils.files.fopen(fpath, 'w') as fp_: + fp_.write(temp_config) + config = sconfig.master_config(fpath) + expect_path_join = os.path.join('/', 'key') + expect_sep_join = '//key' + if salt.utils.platform.is_windows(): + expect_path_join = os.path.join('c:\\', 'key') + expect_sep_join = 'c:\\\\key' + # os.path.join behavior - self.assertEqual(config['key_logfile'], os.path.join('/', 'key')) + self.assertEqual(config['key_logfile'], expect_path_join) # os.sep.join behavior - self.assertNotEqual(config['key_logfile'], '//key') + self.assertNotEqual(config['key_logfile'], expect_sep_join) finally: if os.path.isfile(fpath): os.unlink(fpath) @@ -127,7 +137,7 @@ class ConfigTestCase(TestCase, AdaptedConfigurationTestCaseMixin): root_dir = os.path.join(tempdir, 'foo', 'bar') os.makedirs(root_dir) fpath = os.path.join(root_dir, 'config') - with salt.utils.fopen(fpath, 'w') as fp_: + with salt.utils.files.fopen(fpath, 'w') as fp_: fp_.write( 'root_dir: {0}\n' 'log_file: {1}\n'.format(root_dir, fpath) @@ -145,7 +155,7 @@ class ConfigTestCase(TestCase, AdaptedConfigurationTestCaseMixin): root_dir = os.path.join(tempdir, 'foo', 'bar') os.makedirs(root_dir) fpath = os.path.join(root_dir, 'config') - with salt.utils.fopen(fpath, 'w') as fp_: + with salt.utils.files.fopen(fpath, 'w') as fp_: fp_.write( 'root_dir: {0}\n' 'log_file: {1}\n'.format(root_dir, fpath) @@ -157,6 +167,9 @@ class ConfigTestCase(TestCase, AdaptedConfigurationTestCaseMixin): if os.path.isdir(tempdir): shutil.rmtree(tempdir) + @skipIf( + salt.utils.platform.is_windows(), + 'You can\'t set an environment dynamically in Windows') def test_load_master_config_from_environ_var(self): original_environ = os.environ.copy() @@ -166,7 +179,7 @@ class ConfigTestCase(TestCase, AdaptedConfigurationTestCaseMixin): os.makedirs(env_root_dir) env_fpath = os.path.join(env_root_dir, 'config-env') - with salt.utils.fopen(env_fpath, 'w') as fp_: + with salt.utils.files.fopen(env_fpath, 'w') as fp_: fp_.write( 'root_dir: {0}\n' 'log_file: {1}\n'.format(env_root_dir, env_fpath) @@ -182,7 +195,7 @@ class ConfigTestCase(TestCase, AdaptedConfigurationTestCaseMixin): root_dir = os.path.join(tempdir, 'foo', 'bar') os.makedirs(root_dir) fpath = os.path.join(root_dir, 'config') - with salt.utils.fopen(fpath, 'w') as fp_: + with salt.utils.files.fopen(fpath, 'w') as fp_: fp_.write( 'root_dir: {0}\n' 'log_file: {1}\n'.format(root_dir, fpath) @@ -201,6 +214,9 @@ class ConfigTestCase(TestCase, AdaptedConfigurationTestCaseMixin): if os.path.isdir(tempdir): shutil.rmtree(tempdir) + @skipIf( + salt.utils.platform.is_windows(), + 'You can\'t set an environment dynamically in Windows') def test_load_minion_config_from_environ_var(self): original_environ = os.environ.copy() @@ -210,7 +226,7 @@ class ConfigTestCase(TestCase, AdaptedConfigurationTestCaseMixin): os.makedirs(env_root_dir) env_fpath = os.path.join(env_root_dir, 'config-env') - with salt.utils.fopen(env_fpath, 'w') as fp_: + with salt.utils.files.fopen(env_fpath, 'w') as fp_: fp_.write( 'root_dir: {0}\n' 'log_file: {1}\n'.format(env_root_dir, env_fpath) @@ -226,7 +242,7 @@ class ConfigTestCase(TestCase, AdaptedConfigurationTestCaseMixin): root_dir = os.path.join(tempdir, 'foo', 'bar') os.makedirs(root_dir) fpath = os.path.join(root_dir, 'config') - with salt.utils.fopen(fpath, 'w') as fp_: + with salt.utils.files.fopen(fpath, 'w') as fp_: fp_.write( 'root_dir: {0}\n' 'log_file: {1}\n'.format(root_dir, fpath) @@ -256,7 +272,7 @@ class ConfigTestCase(TestCase, AdaptedConfigurationTestCaseMixin): # configuration settings using the provided client configuration # file master_config = os.path.join(env_root_dir, 'master') - with salt.utils.fopen(master_config, 'w') as fp_: + with salt.utils.files.fopen(master_config, 'w') as fp_: fp_.write( 'blah: true\n' 'root_dir: {0}\n' @@ -266,7 +282,7 @@ class ConfigTestCase(TestCase, AdaptedConfigurationTestCaseMixin): # Now the client configuration file env_fpath = os.path.join(env_root_dir, 'config-env') - with salt.utils.fopen(env_fpath, 'w') as fp_: + with salt.utils.files.fopen(env_fpath, 'w') as fp_: fp_.write( 'root_dir: {0}\n' 'log_file: {1}\n'.format(env_root_dir, env_fpath) @@ -283,7 +299,7 @@ class ConfigTestCase(TestCase, AdaptedConfigurationTestCaseMixin): root_dir = os.path.join(tempdir, 'foo', 'bar') os.makedirs(root_dir) fpath = os.path.join(root_dir, 'config') - with salt.utils.fopen(fpath, 'w') as fp_: + with salt.utils.files.fopen(fpath, 'w') as fp_: fp_.write( 'root_dir: {0}\n' 'log_file: {1}\n'.format(root_dir, fpath) @@ -311,7 +327,7 @@ class ConfigTestCase(TestCase, AdaptedConfigurationTestCaseMixin): # Let's populate a minion configuration file with some basic # settings - with salt.utils.fopen(minion_config, 'w') as fp_: + with salt.utils.files.fopen(minion_config, 'w') as fp_: fp_.write( 'blah: false\n' 'root_dir: {0}\n' @@ -324,7 +340,7 @@ class ConfigTestCase(TestCase, AdaptedConfigurationTestCaseMixin): # file so overrides can happen, the final value of blah should be # True. extra_config = os.path.join(minion_confd, 'extra.conf') - with salt.utils.fopen(extra_config, 'w') as fp_: + with salt.utils.files.fopen(extra_config, 'w') as fp_: fp_.write('blah: true\n') # Let's load the configuration @@ -346,7 +362,7 @@ class ConfigTestCase(TestCase, AdaptedConfigurationTestCaseMixin): # Let's populate a master configuration file with some basic # settings - with salt.utils.fopen(master_config, 'w') as fp_: + with salt.utils.files.fopen(master_config, 'w') as fp_: fp_.write( 'blah: false\n' 'root_dir: {0}\n' @@ -359,7 +375,7 @@ class ConfigTestCase(TestCase, AdaptedConfigurationTestCaseMixin): # file so overrides can happen, the final value of blah should be # True. extra_config = os.path.join(master_confd, 'extra.conf') - with salt.utils.fopen(extra_config, 'w') as fp_: + with salt.utils.files.fopen(extra_config, 'w') as fp_: fp_.write('blah: true\n') # Let's load the configuration @@ -380,10 +396,10 @@ class ConfigTestCase(TestCase, AdaptedConfigurationTestCaseMixin): # Create some kown files. for f in 'abc': fpath = os.path.join(tempdir, f) - with salt.utils.fopen(fpath, 'w') as wfh: + with salt.utils.files.fopen(fpath, 'w') as wfh: wfh.write(f) - with salt.utils.fopen(fpath, 'w') as wfh: + with salt.utils.files.fopen(fpath, 'w') as wfh: wfh.write( 'file_roots:\n' ' base:\n' @@ -410,10 +426,10 @@ class ConfigTestCase(TestCase, AdaptedConfigurationTestCaseMixin): # Create some kown files. for f in 'abc': fpath = os.path.join(tempdir, f) - with salt.utils.fopen(fpath, 'w') as wfh: + with salt.utils.files.fopen(fpath, 'w') as wfh: wfh.write(f) - with salt.utils.fopen(fpath, 'w') as wfh: + with salt.utils.files.fopen(fpath, 'w') as wfh: wfh.write( 'pillar_roots:\n' ' base:\n' @@ -432,6 +448,30 @@ class ConfigTestCase(TestCase, AdaptedConfigurationTestCaseMixin): if os.path.isdir(tempdir): shutil.rmtree(tempdir) + def test_master_id_function(self): + try: + tempdir = tempfile.mkdtemp(dir=TMP) + master_config = os.path.join(tempdir, 'master') + + with salt.utils.fopen(master_config, 'w') as fp_: + fp_.write( + 'id_function:\n' + ' test.echo:\n' + ' text: hello_world\n' + 'root_dir: {0}\n' + 'log_file: {1}\n'.format(tempdir, master_config) + ) + + # Let's load the configuration + config = sconfig.master_config(master_config) + + self.assertEqual(config['log_file'], master_config) + # 'master_config' appends '_master' to the ID + self.assertEqual(config['id'], 'hello_world_master') + finally: + if os.path.isdir(tempdir): + shutil.rmtree(tempdir) + def test_minion_file_roots_glob(self): # Config file and stub file_roots. fpath = tempfile.mktemp() @@ -440,10 +480,10 @@ class ConfigTestCase(TestCase, AdaptedConfigurationTestCaseMixin): # Create some kown files. for f in 'abc': fpath = os.path.join(tempdir, f) - with salt.utils.fopen(fpath, 'w') as wfh: + with salt.utils.files.fopen(fpath, 'w') as wfh: wfh.write(f) - with salt.utils.fopen(fpath, 'w') as wfh: + with salt.utils.files.fopen(fpath, 'w') as wfh: wfh.write( 'file_roots:\n' ' base:\n' @@ -470,10 +510,10 @@ class ConfigTestCase(TestCase, AdaptedConfigurationTestCaseMixin): # Create some kown files. for f in 'abc': fpath = os.path.join(tempdir, f) - with salt.utils.fopen(fpath, 'w') as wfh: + with salt.utils.files.fopen(fpath, 'w') as wfh: wfh.write(f) - with salt.utils.fopen(fpath, 'w') as wfh: + with salt.utils.files.fopen(fpath, 'w') as wfh: wfh.write( 'pillar_roots:\n' ' base:\n' @@ -492,6 +532,29 @@ class ConfigTestCase(TestCase, AdaptedConfigurationTestCaseMixin): if os.path.isdir(tempdir): shutil.rmtree(tempdir) + def test_minion_id_function(self): + try: + tempdir = tempfile.mkdtemp(dir=TMP) + minion_config = os.path.join(tempdir, 'minion') + + with salt.utils.fopen(minion_config, 'w') as fp_: + fp_.write( + 'id_function:\n' + ' test.echo:\n' + ' text: hello_world\n' + 'root_dir: {0}\n' + 'log_file: {1}\n'.format(tempdir, minion_config) + ) + + # Let's load the configuration + config = sconfig.minion_config(minion_config) + + self.assertEqual(config['log_file'], minion_config) + self.assertEqual(config['id'], 'hello_world') + finally: + if os.path.isdir(tempdir): + shutil.rmtree(tempdir) + def test_syndic_config(self): syndic_conf_path = self.get_config_file_path('syndic') minion_conf_path = self.get_config_file_path('minion') @@ -589,6 +652,9 @@ class ConfigTestCase(TestCase, AdaptedConfigurationTestCaseMixin): search_paths = sconfig.cloud_config('/etc/salt/cloud').get('deploy_scripts_search_path') etc_deploy_path = '/salt/cloud.deploy.d' deploy_path = '/salt/cloud/deploy' + if salt.utils.platform.is_windows(): + etc_deploy_path = '/salt\\cloud.deploy.d' + deploy_path = '\\salt\\cloud\\deploy' # Check cloud.deploy.d path is the first element in the search_paths tuple self.assertTrue(search_paths[0].endswith(etc_deploy_path)) @@ -661,8 +727,8 @@ class ConfigTestCase(TestCase, AdaptedConfigurationTestCaseMixin): Tests passing in valid provider and profile config files successfully ''' providers = {'test-provider': - {'digital_ocean': - {'driver': 'digital_ocean', 'profiles': {}}}} + {'digitalocean': + {'driver': 'digitalocean', 'profiles': {}}}} overrides = {'test-profile': {'provider': 'test-provider', 'image': 'Ubuntu 12.10 x64', @@ -670,7 +736,7 @@ class ConfigTestCase(TestCase, AdaptedConfigurationTestCaseMixin): 'conf_file': PATH} ret = {'test-profile': {'profile': 'test-profile', - 'provider': 'test-provider:digital_ocean', + 'provider': 'test-provider:digitalocean', 'image': 'Ubuntu 12.10 x64', 'size': '512MB'}} self.assertEqual(sconfig.apply_vm_profiles_config(providers, @@ -1007,6 +1073,9 @@ class ConfigTestCase(TestCase, AdaptedConfigurationTestCaseMixin): # other cloud configuration tests + @skipIf( + salt.utils.platform.is_windows(), + 'You can\'t set an environment dynamically in Windows') def test_load_cloud_config_from_environ_var(self): original_environ = os.environ.copy() @@ -1016,7 +1085,7 @@ class ConfigTestCase(TestCase, AdaptedConfigurationTestCaseMixin): os.makedirs(env_root_dir) env_fpath = os.path.join(env_root_dir, 'config-env') - with salt.utils.fopen(env_fpath, 'w') as fp_: + with salt.utils.files.fopen(env_fpath, 'w') as fp_: fp_.write( 'root_dir: {0}\n' 'log_file: {1}\n'.format(env_root_dir, env_fpath) @@ -1032,7 +1101,7 @@ class ConfigTestCase(TestCase, AdaptedConfigurationTestCaseMixin): root_dir = os.path.join(tempdir, 'foo', 'bar') os.makedirs(root_dir) fpath = os.path.join(root_dir, 'config') - with salt.utils.fopen(fpath, 'w') as fp_: + with salt.utils.files.fopen(fpath, 'w') as fp_: fp_.write( 'root_dir: {0}\n' 'log_file: {1}\n'.format(root_dir, fpath) @@ -1063,8 +1132,8 @@ class ConfigTestCase(TestCase, AdaptedConfigurationTestCaseMixin): default_config = sconfig.cloud_config(config_file_path) default_config['deploy_scripts_search_path'] = deploy_dir_path - with salt.utils.fopen(config_file_path, 'w') as cfd: - cfd.write(yaml.dump(default_config)) + with salt.utils.files.fopen(config_file_path, 'w') as cfd: + cfd.write(yaml.dump(default_config, default_flow_style=False)) default_config = sconfig.cloud_config(config_file_path) diff --git a/tests/unit/daemons/__init__.py b/tests/unit/daemons/__init__.py new file mode 100644 index 00000000000..40a96afc6ff --- /dev/null +++ b/tests/unit/daemons/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/tests/unit/daemons/test_masterapi.py b/tests/unit/daemons/test_masterapi.py new file mode 100644 index 00000000000..29ea37ecd47 --- /dev/null +++ b/tests/unit/daemons/test_masterapi.py @@ -0,0 +1,201 @@ +# -*- coding: utf-8 -*- + +# Import Python libs +from __future__ import absolute_import + +# Import Salt libs +import salt.config +import salt.daemons.masterapi as masterapi + +# Import Salt Testing Libs +from tests.support.unit import TestCase +from tests.support.mock import ( + patch, + MagicMock, +) + + +class LocalFuncsTestCase(TestCase): + ''' + TestCase for salt.daemons.masterapi.LocalFuncs class + ''' + + def setUp(self): + opts = salt.config.master_config(None) + self.local_funcs = masterapi.LocalFuncs(opts, 'test-key') + + def test_runner_token_not_authenticated(self): + ''' + Asserts that a TokenAuthenticationError is returned when the token can't authenticate. + ''' + mock_ret = {u'error': {u'name': u'TokenAuthenticationError', + u'message': u'Authentication failure of type "token" occurred.'}} + ret = self.local_funcs.runner({u'token': u'asdfasdfasdfasdf'}) + self.assertDictEqual(mock_ret, ret) + + def test_runner_token_authorization_error(self): + ''' + Asserts that a TokenAuthenticationError is returned when the token authenticates, but is + not authorized. + ''' + token = u'asdfasdfasdfasdf' + load = {u'token': token, u'fun': u'test.arg', u'kwarg': {}} + mock_token = {u'token': token, u'eauth': u'foo', u'name': u'test'} + mock_ret = {u'error': {u'name': u'TokenAuthenticationError', + u'message': u'Authentication failure of type "token" occurred ' + u'for user test.'}} + + with patch('salt.auth.LoadAuth.authenticate_token', MagicMock(return_value=mock_token)), \ + patch('salt.auth.LoadAuth.get_auth_list', MagicMock(return_value=[])): + ret = self.local_funcs.runner(load) + + self.assertDictEqual(mock_ret, ret) + + def test_runner_token_salt_invocation_error(self): + ''' + Asserts that a SaltInvocationError is returned when the token authenticates, but the + command is malformed. + ''' + token = u'asdfasdfasdfasdf' + load = {u'token': token, u'fun': u'badtestarg', u'kwarg': {}} + mock_token = {u'token': token, u'eauth': u'foo', u'name': u'test'} + mock_ret = {u'error': {u'name': u'SaltInvocationError', + u'message': u'A command invocation error occurred: Check syntax.'}} + + with patch('salt.auth.LoadAuth.authenticate_token', MagicMock(return_value=mock_token)), \ + patch('salt.auth.LoadAuth.get_auth_list', MagicMock(return_value=[])): + ret = self.local_funcs.runner(load) + + self.assertDictEqual(mock_ret, ret) + + def test_runner_eauth_not_authenticated(self): + ''' + Asserts that an EauthAuthenticationError is returned when the user can't authenticate. + ''' + mock_ret = {u'error': {u'name': u'EauthAuthenticationError', + u'message': u'Authentication failure of type "eauth" occurred for ' + u'user UNKNOWN.'}} + ret = self.local_funcs.runner({u'eauth': u'foo'}) + self.assertDictEqual(mock_ret, ret) + + def test_runner_eauth_authorization_error(self): + ''' + Asserts that an EauthAuthenticationError is returned when the user authenticates, but is + not authorized. + ''' + load = {u'eauth': u'foo', u'username': u'test', u'fun': u'test.arg', u'kwarg': {}} + mock_ret = {u'error': {u'name': u'EauthAuthenticationError', + u'message': u'Authentication failure of type "eauth" occurred for ' + u'user test.'}} + with patch('salt.auth.LoadAuth.authenticate_eauth', MagicMock(return_value=True)), \ + patch('salt.auth.LoadAuth.get_auth_list', MagicMock(return_value=[])): + ret = self.local_funcs.runner(load) + + self.assertDictEqual(mock_ret, ret) + + def test_runner_eauth_salt_invocation_errpr(self): + ''' + Asserts that an EauthAuthenticationError is returned when the user authenticates, but the + command is malformed. + ''' + load = {u'eauth': u'foo', u'username': u'test', u'fun': u'bad.test.arg.func', u'kwarg': {}} + mock_ret = {u'error': {u'name': u'SaltInvocationError', + u'message': u'A command invocation error occurred: Check syntax.'}} + with patch('salt.auth.LoadAuth.authenticate_eauth', MagicMock(return_value=True)), \ + patch('salt.auth.LoadAuth.get_auth_list', MagicMock(return_value=[])): + ret = self.local_funcs.runner(load) + + self.assertDictEqual(mock_ret, ret) + + def test_wheel_token_not_authenticated(self): + ''' + Asserts that a TokenAuthenticationError is returned when the token can't authenticate. + ''' + mock_ret = {u'error': {u'name': u'TokenAuthenticationError', + u'message': u'Authentication failure of type "token" occurred.'}} + ret = self.local_funcs.wheel({u'token': u'asdfasdfasdfasdf'}) + self.assertDictEqual(mock_ret, ret) + + def test_wheel_token_authorization_error(self): + ''' + Asserts that a TokenAuthenticationError is returned when the token authenticates, but is + not authorized. + ''' + token = u'asdfasdfasdfasdf' + load = {u'token': token, u'fun': u'test.arg', u'kwarg': {}} + mock_token = {u'token': token, u'eauth': u'foo', u'name': u'test'} + mock_ret = {u'error': {u'name': u'TokenAuthenticationError', + u'message': u'Authentication failure of type "token" occurred ' + u'for user test.'}} + + with patch('salt.auth.LoadAuth.authenticate_token', MagicMock(return_value=mock_token)), \ + patch('salt.auth.LoadAuth.get_auth_list', MagicMock(return_value=[])): + ret = self.local_funcs.wheel(load) + + self.assertDictEqual(mock_ret, ret) + + def test_wheel_token_salt_invocation_error(self): + ''' + Asserts that a SaltInvocationError is returned when the token authenticates, but the + command is malformed. + ''' + token = u'asdfasdfasdfasdf' + load = {u'token': token, u'fun': u'badtestarg', u'kwarg': {}} + mock_token = {u'token': token, u'eauth': u'foo', u'name': u'test'} + mock_ret = {u'error': {u'name': u'SaltInvocationError', + u'message': u'A command invocation error occurred: Check syntax.'}} + + with patch('salt.auth.LoadAuth.authenticate_token', MagicMock(return_value=mock_token)), \ + patch('salt.auth.LoadAuth.get_auth_list', MagicMock(return_value=[])): + ret = self.local_funcs.wheel(load) + + self.assertDictEqual(mock_ret, ret) + + def test_wheel_eauth_not_authenticated(self): + ''' + Asserts that an EauthAuthenticationError is returned when the user can't authenticate. + ''' + mock_ret = {u'error': {u'name': u'EauthAuthenticationError', + u'message': u'Authentication failure of type "eauth" occurred for ' + u'user UNKNOWN.'}} + ret = self.local_funcs.wheel({u'eauth': u'foo'}) + self.assertDictEqual(mock_ret, ret) + + def test_wheel_eauth_authorization_error(self): + ''' + Asserts that an EauthAuthenticationError is returned when the user authenticates, but is + not authorized. + ''' + load = {u'eauth': u'foo', u'username': u'test', u'fun': u'test.arg', u'kwarg': {}} + mock_ret = {u'error': {u'name': u'EauthAuthenticationError', + u'message': u'Authentication failure of type "eauth" occurred for ' + u'user test.'}} + with patch('salt.auth.LoadAuth.authenticate_eauth', MagicMock(return_value=True)), \ + patch('salt.auth.LoadAuth.get_auth_list', MagicMock(return_value=[])): + ret = self.local_funcs.wheel(load) + + self.assertDictEqual(mock_ret, ret) + + def test_wheel_eauth_salt_invocation_errpr(self): + ''' + Asserts that an EauthAuthenticationError is returned when the user authenticates, but the + command is malformed. + ''' + load = {u'eauth': u'foo', u'username': u'test', u'fun': u'bad.test.arg.func', u'kwarg': {}} + mock_ret = {u'error': {u'name': u'SaltInvocationError', + u'message': u'A command invocation error occurred: Check syntax.'}} + with patch('salt.auth.LoadAuth.authenticate_eauth', MagicMock(return_value=True)), \ + patch('salt.auth.LoadAuth.get_auth_list', MagicMock(return_value=[])): + ret = self.local_funcs.wheel(load) + + self.assertDictEqual(mock_ret, ret) + + def test_wheel_user_not_authenticated(self): + ''' + Asserts that an UserAuthenticationError is returned when the user can't authenticate. + ''' + mock_ret = {u'error': {u'name': u'UserAuthenticationError', + u'message': u'Authentication failure of type "user" occurred for ' + u'user UNKNOWN.'}} + ret = self.local_funcs.wheel({}) + self.assertDictEqual(mock_ret, ret) diff --git a/tests/unit/fileserver/test_fileclient.py b/tests/unit/fileserver/test_fileclient.py index eda65540264..38db095b980 100644 --- a/tests/unit/fileserver/test_fileclient.py +++ b/tests/unit/fileserver/test_fileclient.py @@ -17,9 +17,9 @@ from tests.support.unit import TestCase, skipIf from tests.support.mock import MagicMock, patch, NO_MOCK, NO_MOCK_REASON # Import salt libs -import salt.utils +import salt.utils.files from salt import fileclient -import salt.ext.six as six +from salt.ext import six log = logging.getLogger(__name__) @@ -118,7 +118,7 @@ class FileclientCacheTest(TestCase, AdaptedConfigurationTestCaseMixin, LoaderMod _new_dir(saltenv_root) path = os.path.join(saltenv_root, 'foo.txt') - with salt.utils.fopen(path, 'w') as fp_: + with salt.utils.files.fopen(path, 'w') as fp_: fp_.write( 'This is a test file in the \'{0}\' saltenv.\n' .format(saltenv) @@ -128,7 +128,7 @@ class FileclientCacheTest(TestCase, AdaptedConfigurationTestCaseMixin, LoaderMod os.makedirs(subdir_abspath) for subdir_file in SUBDIR_FILES: path = os.path.join(subdir_abspath, subdir_file) - with salt.utils.fopen(path, 'w') as fp_: + with salt.utils.files.fopen(path, 'w') as fp_: fp_.write( 'This is file \'{0}\' in subdir \'{1} from saltenv ' '\'{2}\''.format(subdir_file, SUBDIR, saltenv) @@ -174,7 +174,7 @@ class FileclientCacheTest(TestCase, AdaptedConfigurationTestCaseMixin, LoaderMod # sufficient here. If opening the file raises an exception, # this is a problem, so we are not catching the exception # and letting it be raised so that the test fails. - with salt.utils.fopen(cache_loc) as fp_: + with salt.utils.files.fopen(cache_loc) as fp_: content = fp_.read() log.debug('cache_loc = %s', cache_loc) log.debug('content = %s', content) @@ -214,7 +214,7 @@ class FileclientCacheTest(TestCase, AdaptedConfigurationTestCaseMixin, LoaderMod # sufficient here. If opening the file raises an exception, # this is a problem, so we are not catching the exception # and letting it be raised so that the test fails. - with salt.utils.fopen(cache_loc) as fp_: + with salt.utils.files.fopen(cache_loc) as fp_: content = fp_.read() log.debug('cache_loc = %s', cache_loc) log.debug('content = %s', content) @@ -255,7 +255,7 @@ class FileclientCacheTest(TestCase, AdaptedConfigurationTestCaseMixin, LoaderMod # sufficient here. If opening the file raises an exception, # this is a problem, so we are not catching the exception # and letting it be raised so that the test fails. - with salt.utils.fopen(cache_loc) as fp_: + with salt.utils.files.fopen(cache_loc) as fp_: content = fp_.read() log.debug('cache_loc = %s', cache_loc) log.debug('content = %s', content) @@ -285,7 +285,7 @@ class FileclientCacheTest(TestCase, AdaptedConfigurationTestCaseMixin, LoaderMod # opening the file raises an exception, this is a problem, so # we are not catching the exception and letting it be raised so # that the test fails. - with salt.utils.fopen(cache_loc) as fp_: + with salt.utils.files.fopen(cache_loc) as fp_: content = fp_.read() log.debug('cache_loc = %s', cache_loc) log.debug('content = %s', content) @@ -319,7 +319,7 @@ class FileclientCacheTest(TestCase, AdaptedConfigurationTestCaseMixin, LoaderMod # opening the file raises an exception, this is a problem, so # we are not catching the exception and letting it be raised so # that the test fails. - with salt.utils.fopen(cache_loc) as fp_: + with salt.utils.files.fopen(cache_loc) as fp_: content = fp_.read() log.debug('cache_loc = %s', cache_loc) log.debug('content = %s', content) @@ -354,7 +354,7 @@ class FileclientCacheTest(TestCase, AdaptedConfigurationTestCaseMixin, LoaderMod # opening the file raises an exception, this is a problem, so # we are not catching the exception and letting it be raised so # that the test fails. - with salt.utils.fopen(cache_loc) as fp_: + with salt.utils.files.fopen(cache_loc) as fp_: content = fp_.read() log.debug('cache_loc = %s', cache_loc) log.debug('content = %s', content) diff --git a/tests/unit/fileserver/test_gitfs.py b/tests/unit/fileserver/test_gitfs.py index 64532ecf3b3..64d8ca52844 100644 --- a/tests/unit/fileserver/test_gitfs.py +++ b/tests/unit/fileserver/test_gitfs.py @@ -9,8 +9,12 @@ import os import shutil import tempfile import textwrap -import pwd import logging +import stat +try: + import pwd +except ImportError: + pass # Import 3rd-party libs import yaml @@ -29,8 +33,10 @@ from tests.support.mock import NO_MOCK, NO_MOCK_REASON, patch from tests.support.paths import TMP, FILES # Import salt libs -import salt.utils.gitfs import salt.fileserver.gitfs as gitfs +import salt.utils.gitfs +import salt.utils.platform +import salt.utils.win_functions log = logging.getLogger(__name__) @@ -43,31 +49,34 @@ class GitfsConfigTestCase(TestCase, LoaderModuleMockMixin): self.tmp_sock_dir = tempfile.mkdtemp(dir=TMP) return { gitfs: { - '__opts__': {'cachedir': self.tmp_cachedir, - 'sock_dir': self.tmp_sock_dir, - 'gitfs_root': 'salt', - 'fileserver_backend': ['git'], - 'gitfs_base': 'master', - 'fileserver_events': True, - 'transport': 'zeromq', - 'gitfs_mountpoint': '', - 'gitfs_saltenv': [], - 'gitfs_env_whitelist': [], - 'gitfs_env_blacklist': [], - 'gitfs_saltenv_whitelist': [], - 'gitfs_saltenv_blacklist': [], - 'gitfs_user': '', - 'gitfs_password': '', - 'gitfs_insecure_auth': False, - 'gitfs_privkey': '', - 'gitfs_pubkey': '', - 'gitfs_passphrase': '', - 'gitfs_refspecs': [ - '+refs/heads/*:refs/remotes/origin/*', - '+refs/tags/*:refs/tags/*' - ], - 'gitfs_ssl_verify': True, - '__role': 'master' + '__opts__': { + 'cachedir': self.tmp_cachedir, + 'sock_dir': self.tmp_sock_dir, + 'gitfs_root': 'salt', + 'fileserver_backend': ['git'], + 'gitfs_base': 'master', + 'fileserver_events': True, + 'transport': 'zeromq', + 'gitfs_mountpoint': '', + 'gitfs_saltenv': [], + 'gitfs_env_whitelist': [], + 'gitfs_env_blacklist': [], + 'gitfs_saltenv_whitelist': [], + 'gitfs_saltenv_blacklist': [], + 'gitfs_user': '', + 'gitfs_password': '', + 'gitfs_insecure_auth': False, + 'gitfs_privkey': '', + 'gitfs_pubkey': '', + 'gitfs_passphrase': '', + 'gitfs_refspecs': [ + '+refs/heads/*:refs/remotes/origin/*', + '+refs/tags/*:refs/tags/*' + ], + 'gitfs_ssl_verify': True, + 'gitfs_disable_saltenv_mapping': False, + 'gitfs_ref_types': ['branch', 'tag', 'sha'], + '__role': 'master', } } } @@ -164,32 +173,35 @@ class GitFSTest(TestCase, LoaderModuleMockMixin): self.tmp_repo_dir = os.path.join(TMP, 'gitfs_root') return { gitfs: { - '__opts__': {'cachedir': self.tmp_cachedir, - 'sock_dir': self.tmp_sock_dir, - 'gitfs_remotes': ['file://' + self.tmp_repo_dir], - 'gitfs_root': '', - 'fileserver_backend': ['git'], - 'gitfs_base': 'master', - 'fileserver_events': True, - 'transport': 'zeromq', - 'gitfs_mountpoint': '', - 'gitfs_saltenv': [], - 'gitfs_env_whitelist': [], - 'gitfs_env_blacklist': [], - 'gitfs_saltenv_whitelist': [], - 'gitfs_saltenv_blacklist': [], - 'gitfs_user': '', - 'gitfs_password': '', - 'gitfs_insecure_auth': False, - 'gitfs_privkey': '', - 'gitfs_pubkey': '', - 'gitfs_passphrase': '', - 'gitfs_refspecs': [ - '+refs/heads/*:refs/remotes/origin/*', - '+refs/tags/*:refs/tags/*' - ], - 'gitfs_ssl_verify': True, - '__role': 'master' + '__opts__': { + 'cachedir': self.tmp_cachedir, + 'sock_dir': self.tmp_sock_dir, + 'gitfs_remotes': ['file://' + self.tmp_repo_dir], + 'gitfs_root': '', + 'fileserver_backend': ['git'], + 'gitfs_base': 'master', + 'fileserver_events': True, + 'transport': 'zeromq', + 'gitfs_mountpoint': '', + 'gitfs_saltenv': [], + 'gitfs_env_whitelist': [], + 'gitfs_env_blacklist': [], + 'gitfs_saltenv_whitelist': [], + 'gitfs_saltenv_blacklist': [], + 'gitfs_user': '', + 'gitfs_password': '', + 'gitfs_insecure_auth': False, + 'gitfs_privkey': '', + 'gitfs_pubkey': '', + 'gitfs_passphrase': '', + 'gitfs_refspecs': [ + '+refs/heads/*:refs/remotes/origin/*', + '+refs/tags/*:refs/tags/*' + ], + 'gitfs_ssl_verify': True, + 'gitfs_disable_saltenv_mapping': False, + 'gitfs_ref_types': ['branch', 'tag', 'sha'], + '__role': 'master', } } } @@ -204,7 +216,6 @@ class GitFSTest(TestCase, LoaderModuleMockMixin): self.integration_base_files = os.path.join(FILES, 'file', 'base') # Create the dir if it doesn't already exist - try: shutil.copytree(self.integration_base_files, self.tmp_repo_dir + '/') except OSError: @@ -218,7 +229,10 @@ class GitFSTest(TestCase, LoaderModuleMockMixin): if 'USERNAME' not in os.environ: try: - os.environ['USERNAME'] = pwd.getpwuid(os.geteuid()).pw_name + if salt.utils.platform.is_windows(): + os.environ['USERNAME'] = salt.utils.win_functions.get_current_user() + else: + os.environ['USERNAME'] = pwd.getpwuid(os.geteuid()).pw_name except AttributeError: log.error('Unable to get effective username, falling back to ' '\'root\'.') @@ -234,14 +248,18 @@ class GitFSTest(TestCase, LoaderModuleMockMixin): Remove the temporary git repository and gitfs cache directory to ensure a clean environment for each test. ''' - shutil.rmtree(self.tmp_repo_dir) - shutil.rmtree(self.tmp_cachedir) - shutil.rmtree(self.tmp_sock_dir) + shutil.rmtree(self.tmp_repo_dir, onerror=self._rmtree_error) + shutil.rmtree(self.tmp_cachedir, onerror=self._rmtree_error) + shutil.rmtree(self.tmp_sock_dir, onerror=self._rmtree_error) del self.tmp_repo_dir del self.tmp_cachedir del self.tmp_sock_dir del self.integration_base_files + def _rmtree_error(self, func, path, excinfo): + os.chmod(path, stat.S_IWRITE) + func(path) + def test_file_list(self): ret = gitfs.file_list(LOAD) self.assertIn('testfile', ret) diff --git a/tests/unit/fileserver/test_roots.py b/tests/unit/fileserver/test_roots.py index 66adaa87580..4956fdac7a3 100644 --- a/tests/unit/fileserver/test_roots.py +++ b/tests/unit/fileserver/test_roots.py @@ -15,10 +15,11 @@ from tests.support.paths import FILES, TMP, TMP_STATE_TREE from tests.support.unit import TestCase, skipIf from tests.support.mock import patch, NO_MOCK, NO_MOCK_REASON -# Import salt libs -from salt.fileserver import roots -from salt import fileclient -import salt.utils +# Import Salt libs +import salt.fileserver.roots +import salt.fileclient +import salt.utils.files +import salt.utils.platform try: import win32file @@ -36,17 +37,17 @@ class RootsTest(TestCase, AdaptedConfigurationTestCaseMixin, LoaderModuleMockMix empty_dir = os.path.join(TMP_STATE_TREE, 'empty_dir') if not os.path.isdir(empty_dir): os.makedirs(empty_dir) - return {roots: {'__opts__': self.opts}} + return {salt.fileserver.roots: {'__opts__': self.opts}} @classmethod def setUpClass(cls): ''' Create special file_roots for symlink test on Windows ''' - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): root_dir = tempfile.mkdtemp(dir=TMP) source_sym = os.path.join(root_dir, 'source_sym') - with salt.utils.fopen(source_sym, 'w') as fp_: + with salt.utils.files.fopen(source_sym, 'w') as fp_: fp_.write('hello world!\n') cwd = os.getcwd() try: @@ -63,9 +64,9 @@ class RootsTest(TestCase, AdaptedConfigurationTestCaseMixin, LoaderModuleMockMix ''' Remove special file_roots for symlink test ''' - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): try: - salt.utils.rm_rf(cls.test_symlink_list_file_roots['base'][0]) + salt.utils.files.rm_rf(cls.test_symlink_list_file_roots['base'][0]) except OSError: pass @@ -73,25 +74,25 @@ class RootsTest(TestCase, AdaptedConfigurationTestCaseMixin, LoaderModuleMockMix del self.opts def test_file_list(self): - ret = roots.file_list({'saltenv': 'base'}) + ret = salt.fileserver.roots.file_list({'saltenv': 'base'}) self.assertIn('testfile', ret) def test_find_file(self): - ret = roots.find_file('testfile') + ret = salt.fileserver.roots.find_file('testfile') self.assertEqual('testfile', ret['rel']) full_path_to_file = os.path.join(FILES, 'file', 'base', 'testfile') self.assertEqual(full_path_to_file, ret['path']) def test_serve_file(self): - with patch.dict(roots.__opts__, {'file_buffer_size': 262144}): + with patch.dict(salt.fileserver.roots.__opts__, {'file_buffer_size': 262144}): load = {'saltenv': 'base', 'path': os.path.join(FILES, 'file', 'base', 'testfile'), 'loc': 0 } fnd = {'path': os.path.join(FILES, 'file', 'base', 'testfile'), 'rel': 'testfile'} - ret = roots.serve_file(load, fnd) + ret = salt.fileserver.roots.serve_file(load, fnd) data = 'Scene 24\n\n \n OLD MAN: Ah, hee he he ha!\n ' \ 'ARTHUR: And this enchanter of whom you speak, he ' \ @@ -106,7 +107,7 @@ class RootsTest(TestCase, AdaptedConfigurationTestCaseMixin, LoaderModuleMockMix 'OLD MAN: Seek you the Bridge of Death.\n ARTHUR: ' \ 'The Bridge of Death, which leads to the Grail?\n ' \ 'OLD MAN: Hee hee ha ha!\n\n' - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): data = 'Scene 24\r\n\r\n \r\n OLD MAN: Ah, hee he he ' \ 'ha!\r\n ARTHUR: And this enchanter of whom you ' \ 'speak, he has seen the grail?\r\n OLD MAN: Ha ha ' \ @@ -140,12 +141,12 @@ class RootsTest(TestCase, AdaptedConfigurationTestCaseMixin, LoaderModuleMockMix 'path': os.path.join(FILES, 'file', 'base', 'testfile'), 'rel': 'testfile' } - ret = roots.file_hash(load, fnd) + ret = salt.fileserver.roots.file_hash(load, fnd) # Hashes are different in Windows. May be how git translates line # endings hsum = 'baba5791276eb99a7cc498fb1acfbc3b4bd96d24cfe984b4ed6b5be2418731df' - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): hsum = '754aa260e1f3e70f43aaf92149c7d1bad37f708c53304c37660e628d7553f687' self.assertDictEqual( @@ -157,17 +158,17 @@ class RootsTest(TestCase, AdaptedConfigurationTestCaseMixin, LoaderModuleMockMix ) def test_file_list_emptydirs(self): - ret = roots.file_list_emptydirs({'saltenv': 'base'}) + ret = salt.fileserver.roots.file_list_emptydirs({'saltenv': 'base'}) self.assertIn('empty_dir', ret) def test_dir_list(self): - ret = roots.dir_list({'saltenv': 'base'}) + ret = salt.fileserver.roots.dir_list({'saltenv': 'base'}) self.assertIn('empty_dir', ret) def test_symlink_list(self): - file_roots = self.test_symlink_list_file_roots \ - or self.opts['file_roots'] - ret = roots.symlink_list({'saltenv': 'base'}) + if self.test_symlink_list_file_roots: + self.opts['file_roots'] = self.test_symlink_list_file_roots + ret = salt.fileserver.roots.symlink_list({'saltenv': 'base'}) self.assertDictEqual(ret, {'dest_sym': 'source_sym'}) @@ -183,7 +184,7 @@ class RootsLimitTraversalTest(TestCase, AdaptedConfigurationTestCaseMixin): file_client_opts = self.get_temp_config('master') file_client_opts['fileserver_limit_traversal'] = True - ret = fileclient.Client(file_client_opts).list_states('base') + ret = salt.fileclient.Client(file_client_opts).list_states('base') self.assertIn('test_deep.test', ret) self.assertIn('test_deep.a.test', ret) self.assertNotIn('test_deep.b.2.test', ret) diff --git a/tests/unit/grains/test_core.py b/tests/unit/grains/test_core.py index b899d4d110d..5fb2d855318 100644 --- a/tests/unit/grains/test_core.py +++ b/tests/unit/grains/test_core.py @@ -20,10 +20,11 @@ from tests.support.mock import ( # Import Salt Libs import salt.utils +import salt.utils.platform import salt.grains.core as core # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six @skipIf(NO_MOCK, NO_MOCK_REASON) @@ -34,7 +35,7 @@ class CoreGrainsTestCase(TestCase, LoaderModuleMockMixin): def setup_loader_modules(self): return {core: {}} - @skipIf(not salt.utils.is_linux(), 'System is not Linux') + @skipIf(not salt.utils.platform.is_linux(), 'System is not Linux') def test_gnu_slash_linux_in_os_name(self): ''' Test to return a list of all enabled services @@ -68,7 +69,7 @@ class CoreGrainsTestCase(TestCase, LoaderModuleMockMixin): return orig_import(name, *args) # Skip the first if statement - with patch.object(salt.utils, 'is_proxy', + with patch.object(salt.utils.platform, 'is_proxy', MagicMock(return_value=False)): # Skip the selinux/systemd stuff (not pertinent) with patch.object(core, '_linux_bin_exists', @@ -124,7 +125,7 @@ class CoreGrainsTestCase(TestCase, LoaderModuleMockMixin): self.assertEqual(os_grains.get('os_family'), 'Debian') - @skipIf(not salt.utils.is_linux(), 'System is not Linux') + @skipIf(not salt.utils.platform.is_linux(), 'System is not Linux') def test_suse_os_from_cpe_data(self): ''' Test if 'os' grain is parsed from CPE_NAME of /etc/os-release @@ -165,7 +166,7 @@ class CoreGrainsTestCase(TestCase, LoaderModuleMockMixin): return orig_import(name, *args) # Skip the first if statement - with patch.object(salt.utils, 'is_proxy', + with patch.object(salt.utils.platform, 'is_proxy', MagicMock(return_value=False)): # Skip the selinux/systemd stuff (not pertinent) with patch.object(core, '_linux_bin_exists', @@ -213,7 +214,7 @@ class CoreGrainsTestCase(TestCase, LoaderModuleMockMixin): return orig_import(name, *args) # Skip the first if statement - with patch.object(salt.utils, 'is_proxy', + with patch.object(salt.utils.platform, 'is_proxy', MagicMock(return_value=False)): # Skip the selinux/systemd stuff (not pertinent) with patch.object(core, '_linux_bin_exists', @@ -231,7 +232,7 @@ class CoreGrainsTestCase(TestCase, LoaderModuleMockMixin): distro_mock = MagicMock( return_value=('SUSE test', 'version', 'arch') ) - with patch("salt.utils.fopen", mock_open()) as suse_release_file: + with patch('salt.utils.files.fopen', mock_open()) as suse_release_file: suse_release_file.return_value.__iter__.return_value = os_release_map.get('suse_release_file', '').splitlines() with patch.object(core, 'linux_distribution', distro_mock): with patch.object(core, '_linux_gpu_data', empty_mock): @@ -249,7 +250,7 @@ class CoreGrainsTestCase(TestCase, LoaderModuleMockMixin): self.assertListEqual(list(os_grains.get('osrelease_info')), os_release_map['osrelease_info']) self.assertEqual(os_grains.get('osmajorrelease'), os_release_map['osmajorrelease']) - @skipIf(not salt.utils.is_linux(), 'System is not Linux') + @skipIf(not salt.utils.platform.is_linux(), 'System is not Linux') def test_suse_os_grains_sles11sp3(self): ''' Test if OS grains are parsed correctly in SLES 11 SP3 @@ -268,7 +269,7 @@ PATCHLEVEL = 3 } self._run_suse_os_grains_tests(_os_release_map) - @skipIf(not salt.utils.is_linux(), 'System is not Linux') + @skipIf(not salt.utils.platform.is_linux(), 'System is not Linux') def test_suse_os_grains_sles11sp4(self): ''' Test if OS grains are parsed correctly in SLES 11 SP4 @@ -292,7 +293,7 @@ PATCHLEVEL = 3 } self._run_suse_os_grains_tests(_os_release_map) - @skipIf(not salt.utils.is_linux(), 'System is not Linux') + @skipIf(not salt.utils.platform.is_linux(), 'System is not Linux') def test_suse_os_grains_sles12(self): ''' Test if OS grains are parsed correctly in SLES 12 @@ -316,7 +317,7 @@ PATCHLEVEL = 3 } self._run_suse_os_grains_tests(_os_release_map) - @skipIf(not salt.utils.is_linux(), 'System is not Linux') + @skipIf(not salt.utils.platform.is_linux(), 'System is not Linux') def test_suse_os_grains_sles12sp1(self): ''' Test if OS grains are parsed correctly in SLES 12 SP1 @@ -340,7 +341,7 @@ PATCHLEVEL = 3 } self._run_suse_os_grains_tests(_os_release_map) - @skipIf(not salt.utils.is_linux(), 'System is not Linux') + @skipIf(not salt.utils.platform.is_linux(), 'System is not Linux') def test_suse_os_grains_opensuse_leap_42_1(self): ''' Test if OS grains are parsed correctly in openSUSE Leap 42.1 @@ -364,7 +365,7 @@ PATCHLEVEL = 3 } self._run_suse_os_grains_tests(_os_release_map) - @skipIf(not salt.utils.is_linux(), 'System is not Linux') + @skipIf(not salt.utils.platform.is_linux(), 'System is not Linux') def test_suse_os_grains_tumbleweed(self): ''' Test if OS grains are parsed correctly in openSUSE Tumbleweed @@ -388,7 +389,7 @@ PATCHLEVEL = 3 } self._run_suse_os_grains_tests(_os_release_map) - @skipIf(not salt.utils.is_linux(), 'System is not Linux') + @skipIf(not salt.utils.platform.is_linux(), 'System is not Linux') def test_ubuntu_os_grains(self): ''' Test if OS grains are parsed correctly in Ubuntu Xenial Xerus @@ -429,7 +430,7 @@ PATCHLEVEL = 3 return orig_import(name, *args) # Skip the first if statement - with patch.object(salt.utils, 'is_proxy', + with patch.object(salt.utils.platform, 'is_proxy', MagicMock(return_value=False)): # Skip the selinux/systemd stuff (not pertinent) with patch.object(core, '_linux_bin_exists', @@ -445,7 +446,7 @@ PATCHLEVEL = 3 # Mock linux_distribution to give us the OS # name that we want. distro_mock = MagicMock(return_value=('Ubuntu', '16.04', 'xenial')) - with patch("salt.utils.fopen", mock_open()) as suse_release_file: + with patch('salt.utils.files.fopen', mock_open()) as suse_release_file: suse_release_file.return_value.__iter__.return_value = os_release_map.get( 'suse_release_file', '').splitlines() with patch.object(core, 'linux_distribution', distro_mock): diff --git a/tests/unit/loader/test_globals.py b/tests/unit/loader/test_globals.py index ba2aef6b62b..9935188e46a 100644 --- a/tests/unit/loader/test_globals.py +++ b/tests/unit/loader/test_globals.py @@ -18,7 +18,7 @@ import inspect import yaml # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six class LoaderGlobalsTest(ModuleCase): diff --git a/tests/unit/loader/test_interfaces.py b/tests/unit/loader/test_interfaces.py index c2125b9303f..c589f271400 100644 --- a/tests/unit/loader/test_interfaces.py +++ b/tests/unit/loader/test_interfaces.py @@ -13,7 +13,7 @@ from __future__ import absolute_import from tests.support.unit import TestCase # Import Salt libs -import salt.ext.six as six +from salt.ext import six from salt.config import minion_config import salt.loader diff --git a/tests/unit/loader/test_loader.py b/tests/unit/loader/test_loader.py index 1a71a380065..2403dbba212 100644 --- a/tests/unit/loader/test_loader.py +++ b/tests/unit/loader/test_loader.py @@ -25,9 +25,10 @@ from tests.support.paths import TMP # Import Salt libs import salt.config -import salt.utils +import salt.utils.files +import salt.utils.stringutils # pylint: disable=import-error,no-name-in-module,redefined-builtin -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import range # pylint: enable=no-name-in-module,redefined-builtin @@ -81,7 +82,7 @@ class LazyLoaderTest(TestCase): self.module_dir = tempfile.mkdtemp(dir=TMP) self.module_file = os.path.join(self.module_dir, '{0}.py'.format(self.module_name)) - with salt.utils.fopen(self.module_file, 'w') as fh: + with salt.utils.files.fopen(self.module_file, 'w') as fh: fh.write(loader_template) fh.flush() os.fsync(fh.fileno()) @@ -346,9 +347,9 @@ class LazyLoaderReloadingTest(TestCase): def update_module(self): self.count += 1 - with salt.utils.fopen(self.module_path, 'wb') as fh: + with salt.utils.files.fopen(self.module_path, 'wb') as fh: fh.write( - salt.utils.to_bytes( + salt.utils.stringutils.to_bytes( module_template.format(count=self.count) ) ) @@ -484,8 +485,8 @@ class LazyLoaderVirtualAliasTest(TestCase): del cls.opts def update_module(self): - with salt.utils.fopen(self.module_path, 'wb') as fh: - fh.write(salt.utils.to_bytes(virtual_alias_module_template)) + with salt.utils.files.fopen(self.module_path, 'wb') as fh: + fh.write(salt.utils.stringutils.to_bytes(virtual_alias_module_template)) fh.flush() os.fsync(fh.fileno()) # flush to disk @@ -578,9 +579,9 @@ class LazyLoaderSubmodReloadingTest(TestCase): def update_module(self): self.count += 1 - with salt.utils.fopen(self.module_path, 'wb') as fh: + with salt.utils.files.fopen(self.module_path, 'wb') as fh: fh.write( - salt.utils.to_bytes( + salt.utils.stringutils.to_bytes( submodule_template.format(self.module_name, count=self.count) ) ) @@ -602,9 +603,9 @@ class LazyLoaderSubmodReloadingTest(TestCase): for modname in list(sys.modules): if modname.startswith(self.module_name): del sys.modules[modname] - with salt.utils.fopen(self.lib_path, 'wb') as fh: + with salt.utils.files.fopen(self.lib_path, 'wb') as fh: fh.write( - salt.utils.to_bytes( + salt.utils.stringutils.to_bytes( submodule_lib_template.format(count=self.lib_count) ) ) @@ -738,8 +739,8 @@ class LazyLoaderModulePackageTest(TestCase): dirname = os.path.dirname(pyfile) if not os.path.exists(dirname): os.makedirs(dirname) - with salt.utils.fopen(pyfile, 'wb') as fh: - fh.write(salt.utils.to_bytes(contents)) + with salt.utils.files.fopen(pyfile, 'wb') as fh: + fh.write(salt.utils.stringutils.to_bytes(contents)) fh.flush() os.fsync(fh.fileno()) # flush to disk @@ -827,7 +828,7 @@ class LazyLoaderDeepSubmodReloadingTest(TestCase): self.lib_count = collections.defaultdict(int) # mapping of path -> count # bootstrap libs - with salt.utils.fopen(os.path.join(self.module_dir, '__init__.py'), 'w') as fh: + with salt.utils.files.fopen(os.path.join(self.module_dir, '__init__.py'), 'w') as fh: # No .decode() needed here as deep_init_base is defined as str and # not bytes. fh.write(deep_init_base.format(self.module_name)) @@ -881,9 +882,9 @@ class LazyLoaderDeepSubmodReloadingTest(TestCase): del sys.modules[modname] path = os.path.join(self.lib_paths[lib_name], '__init__.py') self.lib_count[lib_name] += 1 - with salt.utils.fopen(path, 'wb') as fh: + with salt.utils.files.fopen(path, 'wb') as fh: fh.write( - salt.utils.to_bytes( + salt.utils.stringutils.to_bytes( submodule_lib_template.format(count=self.lib_count[lib_name]) ) ) diff --git a/tests/unit/modules/test_alternatives.py b/tests/unit/modules/test_alternatives.py index b018eafa803..8dd0cde28d6 100644 --- a/tests/unit/modules/test_alternatives.py +++ b/tests/unit/modules/test_alternatives.py @@ -66,30 +66,28 @@ class AlternativesTestCase(TestCase, LoaderModuleMockMixin): ) def test_show_current(self): - with patch('os.readlink') as os_readlink_mock: - os_readlink_mock.return_value = '/etc/alternatives/salt' + mock = MagicMock(return_value='/etc/alternatives/salt') + with patch('salt.utils.path.readlink', mock): ret = alternatives.show_current('better-world') self.assertEqual('/etc/alternatives/salt', ret) - os_readlink_mock.assert_called_once_with( - '/etc/alternatives/better-world' - ) + mock.assert_called_once_with('/etc/alternatives/better-world') with TestsLoggingHandler() as handler: - os_readlink_mock.side_effect = OSError('Hell was not found!!!') + mock.side_effect = OSError('Hell was not found!!!') self.assertFalse(alternatives.show_current('hell')) - os_readlink_mock.assert_called_with('/etc/alternatives/hell') + mock.assert_called_with('/etc/alternatives/hell') self.assertIn('ERROR:alternative: hell does not exist', handler.messages) def test_check_installed(self): - with patch('os.readlink') as os_readlink_mock: - os_readlink_mock.return_value = '/etc/alternatives/salt' + mock = MagicMock(return_value='/etc/alternatives/salt') + with patch('salt.utils.path.readlink', mock): self.assertTrue( alternatives.check_installed( 'better-world', '/etc/alternatives/salt' ) ) - os_readlink_mock.return_value = False + mock.return_value = False self.assertFalse( alternatives.check_installed( 'help', '/etc/alternatives/salt' diff --git a/tests/unit/modules/test_apache.py b/tests/unit/modules/test_apache.py index 8963443585f..14f409337fc 100644 --- a/tests/unit/modules/test_apache.py +++ b/tests/unit/modules/test_apache.py @@ -200,6 +200,6 @@ class ApacheTestCase(TestCase, LoaderModuleMockMixin): ''' with patch('salt.modules.apache._parse_config', MagicMock(return_value='Listen 22')): - with patch('salt.utils.fopen', mock_open()): + with patch('salt.utils.files.fopen', mock_open()): self.assertEqual(apache.config('/ports.conf', [{'Listen': '22'}]), 'Listen 22') diff --git a/tests/unit/modules/test_archive.py b/tests/unit/modules/test_archive.py index 0827881687d..edf5f7f7d3a 100644 --- a/tests/unit/modules/test_archive.py +++ b/tests/unit/modules/test_archive.py @@ -18,8 +18,8 @@ from tests.support.mock import NO_MOCK, NO_MOCK_REASON, MagicMock, patch # Import salt libs import salt.modules.archive as archive +import salt.utils.path from salt.exceptions import CommandNotFoundError -from salt.utils import which_bin # Import 3rd-party libs from salt.ext.six.moves import zip @@ -45,7 +45,7 @@ class ArchiveTestCase(TestCase, LoaderModuleMockMixin): def test_tar(self): with patch('glob.glob', lambda pathname: [pathname]): - with patch('salt.utils.which', lambda exe: exe): + with patch('salt.utils.path.which', lambda exe: exe): mock = MagicMock(return_value='salt') with patch.dict(archive.__salt__, {'cmd.run': mock}): ret = archive.tar( @@ -76,7 +76,7 @@ class ArchiveTestCase(TestCase, LoaderModuleMockMixin): ) def test_tar_raises_exception_if_not_found(self): - with patch('salt.utils.which', lambda exe: None): + with patch('salt.utils.path.which', lambda exe: None): mock = MagicMock(return_value='salt') with patch.dict(archive.__salt__, {'cmd.run': mock}): self.assertRaises( @@ -91,7 +91,7 @@ class ArchiveTestCase(TestCase, LoaderModuleMockMixin): def test_gzip(self): mock = MagicMock(return_value='salt') with patch.dict(archive.__salt__, {'cmd.run': mock}): - with patch('salt.utils.which', lambda exe: exe): + with patch('salt.utils.path.which', lambda exe: exe): ret = archive.gzip('/tmp/something-to-compress') self.assertEqual(['salt'], ret) mock.assert_called_once_with( @@ -102,7 +102,7 @@ class ArchiveTestCase(TestCase, LoaderModuleMockMixin): def test_gzip_raises_exception_if_not_found(self): mock = MagicMock(return_value='salt') with patch.dict(archive.__salt__, {'cmd.run': mock}): - with patch('salt.utils.which', lambda exe: None): + with patch('salt.utils.path.which', lambda exe: None): self.assertRaises( CommandNotFoundError, archive.gzip, '/tmp/something-to-compress' @@ -112,7 +112,7 @@ class ArchiveTestCase(TestCase, LoaderModuleMockMixin): def test_gunzip(self): mock = MagicMock(return_value='salt') with patch.dict(archive.__salt__, {'cmd.run': mock}): - with patch('salt.utils.which', lambda exe: exe): + with patch('salt.utils.path.which', lambda exe: exe): ret = archive.gunzip('/tmp/something-to-decompress.tar.gz') self.assertEqual(['salt'], ret) mock.assert_called_once_with( @@ -123,7 +123,7 @@ class ArchiveTestCase(TestCase, LoaderModuleMockMixin): def test_gunzip_raises_exception_if_not_found(self): mock = MagicMock(return_value='salt') with patch.dict(archive.__salt__, {'cmd.run': mock}): - with patch('salt.utils.which', lambda exe: None): + with patch('salt.utils.path.which', lambda exe: None): self.assertRaises( CommandNotFoundError, archive.gunzip, @@ -133,7 +133,7 @@ class ArchiveTestCase(TestCase, LoaderModuleMockMixin): def test_cmd_zip(self): with patch('glob.glob', lambda pathname: [pathname]): - with patch('salt.utils.which', lambda exe: exe): + with patch('salt.utils.path.which', lambda exe: exe): mock = MagicMock(return_value='salt') with patch.dict(archive.__salt__, {'cmd.run': mock}): ret = archive.cmd_zip( @@ -172,12 +172,14 @@ class ArchiveTestCase(TestCase, LoaderModuleMockMixin): '/tmp/tmpePe8yO,/tmp/tmpLeSw1A', template='jinja' ) - self.assertEqual(['tmp/tmpePe8yO', 'tmp/tmpLeSw1A'], ret) + expected = [os.path.join('tmp', 'tmpePe8yO'), + os.path.join('tmp', 'tmpLeSw1A')] + self.assertEqual(expected, ret) def test_zip_raises_exception_if_not_found(self): mock = MagicMock(return_value='salt') with patch.dict(archive.__salt__, {'cmd.run': mock}): - with patch('salt.utils.which', lambda exe: None): + with patch('salt.utils.path.which', lambda exe: None): self.assertRaises( CommandNotFoundError, archive.cmd_zip, @@ -199,7 +201,7 @@ class ArchiveTestCase(TestCase, LoaderModuleMockMixin): 'pid': 12345, 'retcode': 0}) - with patch('salt.utils.which', lambda exe: exe): + with patch('salt.utils.path.which', lambda exe: exe): mock = _get_mock() with patch.dict(archive.__salt__, {'cmd.run_all': mock}): @@ -315,7 +317,7 @@ class ArchiveTestCase(TestCase, LoaderModuleMockMixin): def test_unzip_raises_exception_if_not_found(self): mock = MagicMock(return_value='salt') with patch.dict(archive.__salt__, {'cmd.run': mock}): - with patch('salt.utils.which', lambda exe: None): + with patch('salt.utils.path.which', lambda exe: None): self.assertRaises( CommandNotFoundError, archive.cmd_unzip, @@ -328,7 +330,7 @@ class ArchiveTestCase(TestCase, LoaderModuleMockMixin): def test_rar(self): with patch('glob.glob', lambda pathname: [pathname]): - with patch('salt.utils.which', lambda exe: exe): + with patch('salt.utils.path.which', lambda exe: exe): mock = MagicMock(return_value='salt') with patch.dict(archive.__salt__, {'cmd.run': mock}): ret = archive.rar( @@ -358,7 +360,7 @@ class ArchiveTestCase(TestCase, LoaderModuleMockMixin): def test_rar_raises_exception_if_not_found(self): mock = MagicMock(return_value='salt') with patch.dict(archive.__salt__, {'cmd.run': mock}): - with patch('salt.utils.which', lambda exe: None): + with patch('salt.utils.path.which', lambda exe: None): self.assertRaises( CommandNotFoundError, archive.rar, @@ -367,10 +369,10 @@ class ArchiveTestCase(TestCase, LoaderModuleMockMixin): ) self.assertFalse(mock.called) - @skipIf(which_bin(('unrar', 'rar')) is None, 'unrar not installed') + @skipIf(salt.utils.path.which_bin(('unrar', 'rar')) is None, 'unrar not installed') def test_unrar(self): - with patch('salt.utils.which_bin', lambda exe: exe[0] if isinstance(exe, (list, tuple)) else exe): - with patch('salt.utils.which', lambda exe: exe[0] if isinstance(exe, (list, tuple)) else exe): + with patch('salt.utils.path.which_bin', lambda exe: exe[0] if isinstance(exe, (list, tuple)) else exe): + with patch('salt.utils.path.which', lambda exe: exe[0] if isinstance(exe, (list, tuple)) else exe): mock = MagicMock(return_value='salt') with patch.dict(archive.__salt__, {'cmd.run': mock}): ret = archive.unrar( @@ -400,7 +402,7 @@ class ArchiveTestCase(TestCase, LoaderModuleMockMixin): ) def test_unrar_raises_exception_if_not_found(self): - with patch('salt.utils.which_bin', lambda exe: None): + with patch('salt.utils.path.which_bin', lambda exe: None): mock = MagicMock(return_value='salt') with patch.dict(archive.__salt__, {'cmd.run': mock}): self.assertRaises( @@ -412,7 +414,7 @@ class ArchiveTestCase(TestCase, LoaderModuleMockMixin): self.assertFalse(mock.called) def test_trim_files(self): - with patch('salt.utils.which_bin', lambda exe: exe): + with patch('salt.utils.path.which_bin', lambda exe: exe): source = 'file.tar.gz' tmp_dir = 'temp' files = [ diff --git a/tests/unit/modules/test_at.py b/tests/unit/modules/test_at.py index b368aba88e6..f27d3be7a1c 100644 --- a/tests/unit/modules/test_at.py +++ b/tests/unit/modules/test_at.py @@ -17,7 +17,7 @@ from tests.support.mock import ( ) # Import Salt Libs -import salt.utils +import salt.utils.path import salt.modules.at as at @@ -99,21 +99,21 @@ class AtTestCase(TestCase, LoaderModuleMockMixin): Tests for remove jobs from the queue. """ with patch('salt.modules.at.atq', MagicMock(return_value=self.atq_output)): - with patch.object(salt.utils, 'which', return_value=None): + with patch.object(salt.utils.path, 'which', return_value=None): self.assertEqual(at.atrm(), "'at.atrm' is not available.") - with patch.object(salt.utils, 'which', return_value=True): + with patch.object(salt.utils.path, 'which', return_value=True): self.assertDictEqual(at.atrm(), {'jobs': {'removed': [], 'tag': None}}) with patch.object(at, '_cmd', return_value=True): - with patch.object(salt.utils, 'which', return_value=True): + with patch.object(salt.utils.path, 'which', return_value=True): self.assertDictEqual(at.atrm('all'), {'jobs': {'removed': ['101'], 'tag': None}}) with patch.object(at, '_cmd', return_value=True): - with patch.object(salt.utils, 'which', return_value=True): + with patch.object(salt.utils.path, 'which', return_value=True): self.assertDictEqual(at.atrm(101), {'jobs': {'removed': ['101'], 'tag': None}}) @@ -149,11 +149,11 @@ class AtTestCase(TestCase, LoaderModuleMockMixin): with patch('salt.modules.at.atq', MagicMock(return_value=self.atq_output)): self.assertDictEqual(at.at(), {'jobs': []}) - with patch.object(salt.utils, 'which', return_value=None): + with patch.object(salt.utils.path, 'which', return_value=None): self.assertEqual(at.at('12:05am', '/sbin/reboot', tag='reboot'), "'at.at' is not available.") - with patch.object(salt.utils, 'which', return_value=True): + with patch.object(salt.utils.path, 'which', return_value=True): with patch.dict(at.__grains__, {'os_family': 'RedHat'}): mock = MagicMock(return_value=None) with patch.dict(at.__salt__, {'cmd.run': mock}): diff --git a/tests/unit/modules/test_augeas_cfg.py b/tests/unit/modules/test_augeas_cfg.py index 268fbe51794..a2d40ac9867 100644 --- a/tests/unit/modules/test_augeas_cfg.py +++ b/tests/unit/modules/test_augeas_cfg.py @@ -15,6 +15,7 @@ from tests.support.mock import ( # Import Salt Libs import salt.modules.augeas_cfg as augeas_cfg from salt.exceptions import SaltInvocationError +from salt.ext import six # Make sure augeas python interface is installed if augeas_cfg.HAS_AUGEAS: from augeas import Augeas as _Augeas @@ -26,7 +27,7 @@ class AugeasCfgTestCase(TestCase): Test cases for salt.modules.augeas_cfg ''' # 'execute' function tests: 3 - + @skipIf(six.PY3, 'Disabled pending https://github.com/hercules-team/python-augeas/issues/30') def test_execute(self): ''' Test if it execute Augeas commands diff --git a/tests/unit/modules/test_beacons.py b/tests/unit/modules/test_beacons.py new file mode 100644 index 00000000000..6706866bb9c --- /dev/null +++ b/tests/unit/modules/test_beacons.py @@ -0,0 +1,127 @@ +# -*- coding: utf-8 -*- +''' + :codeauthor: :email:`Jayesh Kariya ` +''' + +# Import Python Libs +from __future__ import absolute_import +import os + +# Import Salt Testing Libs +from tests.support.mixins import LoaderModuleMockMixin +from tests.support.paths import TMP +from tests.support.unit import TestCase, skipIf +from tests.support.mock import ( + MagicMock, + patch, + NO_MOCK, + NO_MOCK_REASON +) + +# Import Salt Libs +import salt.modules.beacons as beacons +from salt.utils.event import SaltEvent + +SOCK_DIR = os.path.join(TMP, 'test-socks') + + +@skipIf(NO_MOCK, NO_MOCK_REASON) +class BeaconsTestCase(TestCase, LoaderModuleMockMixin): + ''' + Test cases for salt.modules.beacons + ''' + def setup_loader_modules(self): + return {beacons: {}} + + def test_delete(self): + ''' + Test deleting a beacon. + ''' + comm1 = 'Deleted beacon: ps.' + event_returns = [ + {'complete': True, + 'tag': '/salt/minion/minion_beacons_delete_complete', + 'beacons': {}}, + ] + + with patch.dict(beacons.__opts__, {'beacons': {'ps': [{'processes': {'salt-master': 'stopped', 'apache2': 'stopped'}}]}, 'sock_dir': SOCK_DIR}): + mock = MagicMock(return_value=True) + with patch.dict(beacons.__salt__, {'event.fire': mock}): + with patch.object(SaltEvent, 'get_event', side_effect=event_returns): + self.assertDictEqual(beacons.delete('ps'), + {'comment': comm1, 'result': True}) + + def test_add(self): + ''' + Test adding a beacon + ''' + comm1 = 'Added beacon: ps.' + event_returns = [{'complete': True, + 'tag': '/salt/minion/minion_beacons_list_complete', + 'beacons': {}}, + {'complete': True, + 'valid': True, + 'vcomment': '', + 'tag': '/salt/minion/minion_beacons_list_complete'}, + {'complete': True, + 'tag': '/salt/minion/minion_beacon_add_complete', + 'beacons': {'ps': [{'processes': {'salt-master': 'stopped', 'apache2': 'stopped'}}]}}] + + with patch.dict(beacons.__opts__, {'beacons': {}, 'sock_dir': SOCK_DIR}): + mock = MagicMock(return_value=True) + with patch.dict(beacons.__salt__, {'event.fire': mock}): + with patch.object(SaltEvent, 'get_event', side_effect=event_returns): + self.assertDictEqual(beacons.add('ps', [{'processes': {'salt-master': 'stopped', 'apache2': 'stopped'}}]), + {'comment': comm1, 'result': True}) + + def test_save(self): + ''' + Test saving beacons. + ''' + comm1 = 'Beacons saved to //tmp/beacons.conf.' + with patch.dict(beacons.__opts__, {'config_dir': '', 'beacons': {}, + 'default_include': '/tmp/', + 'sock_dir': SOCK_DIR}): + + mock = MagicMock(return_value=True) + with patch.dict(beacons.__salt__, {'event.fire': mock}): + _ret_value = {'complete': True, 'beacons': {}} + with patch.object(SaltEvent, 'get_event', return_value=_ret_value): + self.assertDictEqual(beacons.save(), + {'comment': comm1, 'result': True}) + + def test_disable(self): + ''' + Test disabling beacons + ''' + comm1 = 'Disabled beacons on minion.' + event_returns = [{'complete': True, + 'tag': '/salt/minion/minion_beacons_disabled_complete', + 'beacons': {'enabled': False, + 'ps': [{'processes': {'salt-master': 'stopped', + 'apache2': 'stopped'}}]}}] + + with patch.dict(beacons.__opts__, {'beacons': {}, 'sock_dir': SOCK_DIR}): + mock = MagicMock(return_value=True) + with patch.dict(beacons.__salt__, {'event.fire': mock}): + with patch.object(SaltEvent, 'get_event', side_effect=event_returns): + self.assertDictEqual(beacons.disable(), + {'comment': comm1, 'result': True}) + + def test_enable(self): + ''' + Test enabling beacons + ''' + comm1 = 'Enabled beacons on minion.' + event_returns = [{'complete': True, + 'tag': '/salt/minion/minion_beacon_enabled_complete', + 'beacons': {'enabled': True, + 'ps': [{'processes': {'salt-master': 'stopped', + 'apache2': 'stopped'}}]}}] + + with patch.dict(beacons.__opts__, {'beacons': {}, 'sock_dir': SOCK_DIR}): + mock = MagicMock(return_value=True) + with patch.dict(beacons.__salt__, {'event.fire': mock}): + with patch.object(SaltEvent, 'get_event', side_effect=event_returns): + self.assertDictEqual(beacons.enable(), + {'comment': comm1, 'result': True}) diff --git a/tests/unit/modules/test_boto_elasticsearch_domain.py b/tests/unit/modules/test_boto_elasticsearch_domain.py index 174bf6ebd5f..41565ecbc5f 100644 --- a/tests/unit/modules/test_boto_elasticsearch_domain.py +++ b/tests/unit/modules/test_boto_elasticsearch_domain.py @@ -18,7 +18,7 @@ from tests.support.mock import ( ) # Import Salt libs -import salt.ext.six as six +from salt.ext import six import salt.loader from salt.utils.versions import LooseVersion import salt.modules.boto_elasticsearch_domain as boto_elasticsearch_domain diff --git a/tests/unit/modules/test_boto_elb.py b/tests/unit/modules/test_boto_elb.py index de355627f68..8d6d517cc75 100644 --- a/tests/unit/modules/test_boto_elb.py +++ b/tests/unit/modules/test_boto_elb.py @@ -4,6 +4,7 @@ from __future__ import absolute_import import logging from copy import deepcopy +import pkg_resources # import Python Third Party Libs # pylint: disable=import-error @@ -15,28 +16,28 @@ except ImportError: HAS_BOTO = False try: - from moto import mock_ec2, mock_elb + from moto import mock_ec2_deprecated, mock_elb_deprecated HAS_MOTO = True except ImportError: HAS_MOTO = False - def mock_ec2(self): + def mock_ec2_deprecated(self): ''' - if the mock_ec2 function is not available due to import failure + if the mock_ec2_deprecated function is not available due to import failure this replaces the decorated function with stub_function. - Allows boto_vpc unit tests to use the @mock_ec2 decorator - without a "NameError: name 'mock_ec2' is not defined" error. + Allows boto_elb unit tests to use the @mock_ec2_deprecated decorator + without a "NameError: name 'mock_ec2_deprecated' is not defined" error. ''' def stub_function(self): pass return stub_function - def mock_elb(self): + def mock_elb_deprecated(self): ''' - if the mock_ec2 function is not available due to import failure + if the mock_elb_deprecated function is not available due to import failure this replaces the decorated function with stub_function. - Allows boto_vpc unit tests to use the @mock_ec2 decorator - without a "NameError: name 'mock_ec2' is not defined" error. + Allows boto_elb unit tests to use the @mock_elb_deprecated decorator + without a "NameError: name 'mock_elb_deprecated' is not defined" error. ''' def stub_function(self): pass @@ -45,9 +46,10 @@ except ImportError: # Import Salt Libs import salt.config -import salt.ext.six as six +from salt.ext import six import salt.loader import salt.modules.boto_elb as boto_elb +import salt.utils.versions # Import Salt Testing Libs from tests.support.mixins import LoaderModuleMockMixin @@ -64,12 +66,32 @@ conn_parameters = {'region': region, 'key': access_key, 'keyid': secret_key, boto_conn_parameters = {'aws_access_key_id': access_key, 'aws_secret_access_key': secret_key} instance_parameters = {'instance_type': 't1.micro'} +required_moto = '0.3.7' +required_moto_py3 = '1.0.1' + + +def _has_required_moto(): + ''' + Returns True or False depending on if ``moto`` is installed and at the correct version, + depending on what version of Python is running these tests. + ''' + if not HAS_MOTO: + return False + else: + moto_version = salt.utils.versions.LooseVersion(pkg_resources.get_distribution('moto').version) + if moto_version < required_moto: + return False + elif six.PY3 and moto_version < required_moto_py3: + return False + + return True @skipIf(NO_MOCK, NO_MOCK_REASON) -@skipIf(six.PY3, 'Running tests with Python 3. These tests need to be rewritten to support Py3.') @skipIf(HAS_BOTO is False, 'The boto module must be installed.') @skipIf(HAS_MOTO is False, 'The moto module must be installed.') +@skipIf(_has_required_moto() is False, 'The moto module must be >= to {0} for ' + 'PY2 or {1} for PY3.'.format(required_moto, required_moto_py3)) class BotoElbTestCase(TestCase, LoaderModuleMockMixin): ''' TestCase for salt.modules.boto_elb module @@ -92,8 +114,8 @@ class BotoElbTestCase(TestCase, LoaderModuleMockMixin): # __virtual__ must be caller in order for _get_conn to be injected boto_elb.__virtual__() - @mock_ec2 - @mock_elb + @mock_ec2_deprecated + @mock_elb_deprecated def test_register_instances_valid_id_result_true(self): ''' tests that given a valid instance id and valid ELB that @@ -111,8 +133,8 @@ class BotoElbTestCase(TestCase, LoaderModuleMockMixin): **conn_parameters) self.assertEqual(True, register_result) - @mock_ec2 - @mock_elb + @mock_ec2_deprecated + @mock_elb_deprecated def test_register_instances_valid_id_string(self): ''' tests that given a string containing a instance id and valid ELB that @@ -134,8 +156,8 @@ class BotoElbTestCase(TestCase, LoaderModuleMockMixin): log.debug(load_balancer_refreshed.instances) self.assertEqual([reservations.instances[0].id], registered_instance_ids) - @mock_ec2 - @mock_elb + @mock_ec2_deprecated + @mock_elb_deprecated def test_deregister_instances_valid_id_result_true(self): ''' tests that given an valid id the boto_elb deregister_instances method @@ -155,8 +177,8 @@ class BotoElbTestCase(TestCase, LoaderModuleMockMixin): **conn_parameters) self.assertEqual(True, deregister_result) - @mock_ec2 - @mock_elb + @mock_ec2_deprecated + @mock_elb_deprecated def test_deregister_instances_valid_id_string(self): ''' tests that given an valid id the boto_elb deregister_instances method @@ -181,8 +203,8 @@ class BotoElbTestCase(TestCase, LoaderModuleMockMixin): load_balancer_refreshed.instances] self.assertEqual(actual_instances, expected_instances) - @mock_ec2 - @mock_elb + @mock_ec2_deprecated + @mock_elb_deprecated def test_deregister_instances_valid_id_list(self): ''' tests that given an valid ids in the form of a list that the boto_elb diff --git a/tests/unit/modules/test_boto_lambda.py b/tests/unit/modules/test_boto_lambda.py index fe81c09871a..4c26a10a960 100644 --- a/tests/unit/modules/test_boto_lambda.py +++ b/tests/unit/modules/test_boto_lambda.py @@ -31,9 +31,10 @@ import salt.modules.boto_lambda as boto_lambda from salt.exceptions import SaltInvocationError from salt.utils.versions import LooseVersion import salt.utils +import salt.utils.stringutils # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import range # pylint: disable=import-error,redefined-builtin # pylint: disable=import-error,no-name-in-module try: @@ -155,7 +156,7 @@ class TempZipFile(object): suffix='.zip', prefix='salt_test_', delete=False) as tmp: to_write = '###\n' if six.PY3: - to_write = salt.utils.to_bytes(to_write) + to_write = salt.utils.stringutils.to_bytes(to_write) tmp.write(to_write) self.zipfile = tmp.name return self.zipfile diff --git a/tests/unit/modules/test_boto_s3_bucket.py b/tests/unit/modules/test_boto_s3_bucket.py index 1bafb3ea7a8..f3acbad0720 100644 --- a/tests/unit/modules/test_boto_s3_bucket.py +++ b/tests/unit/modules/test_boto_s3_bucket.py @@ -23,7 +23,7 @@ import salt.modules.boto_s3_bucket as boto_s3_bucket from salt.utils.versions import LooseVersion # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import range # pylint: disable=import-error,redefined-builtin # pylint: disable=import-error,no-name-in-module,unused-import diff --git a/tests/unit/modules/test_boto_secgroup.py b/tests/unit/modules/test_boto_secgroup.py index 7a1d9c01971..ef90958decc 100644 --- a/tests/unit/modules/test_boto_secgroup.py +++ b/tests/unit/modules/test_boto_secgroup.py @@ -29,17 +29,17 @@ except ImportError: HAS_BOTO = False try: - from moto import mock_ec2 + from moto import mock_ec2_deprecated HAS_MOTO = True except ImportError: HAS_MOTO = False - def mock_ec2(self): + def mock_ec2_deprecated(self): ''' - if the mock_ec2 function is not available due to import failure + if the mock_ec2_deprecated function is not available due to import failure this replaces the decorated function with stub_function. - Allows boto_secgroup unit tests to use the @mock_ec2 decorator - without a "NameError: name 'mock_ec2' is not defined" error. + Allows boto_secgroup unit tests to use the @mock_ec2_deprecated decorator + without a "NameError: name 'mock_ec2_deprecated' is not defined" error. ''' def stub_function(self): pass @@ -117,7 +117,7 @@ class BotoSecgroupTestCase(TestCase, LoaderModuleMockMixin): {'to_port': 80, 'from_port': 80, 'ip_protocol': u'tcp', 'cidr_ip': u'0.0.0.0/0'}] self.assertEqual(boto_secgroup._split_rules(rules), split_rules) - @mock_ec2 + @mock_ec2_deprecated def test_create_ec2_classic(self): ''' Test of creation of an EC2-Classic security group. The test ensures @@ -137,7 +137,7 @@ class BotoSecgroupTestCase(TestCase, LoaderModuleMockMixin): secgroup_created_group[0].vpc_id] self.assertEqual(expected_create_result, secgroup_create_result) - @mock_ec2 + @mock_ec2_deprecated def test_create_ec2_vpc(self): ''' test of creation of an EC2-VPC security group. The test ensures that a @@ -155,7 +155,7 @@ class BotoSecgroupTestCase(TestCase, LoaderModuleMockMixin): secgroup_create_result = [secgroup_created_group[0].name, secgroup_created_group[0].description, secgroup_created_group[0].vpc_id] self.assertEqual(expected_create_result, secgroup_create_result) - @mock_ec2 + @mock_ec2_deprecated def test_get_group_id_ec2_classic(self): ''' tests that given a name of a group in EC2-Classic that the correct @@ -177,7 +177,7 @@ class BotoSecgroupTestCase(TestCase, LoaderModuleMockMixin): @skipIf(True, 'test skipped because moto does not yet support group' ' filters https://github.com/spulec/moto/issues/154') - @mock_ec2 + @mock_ec2_deprecated def test_get_group_id_ec2_vpc(self): ''' tests that given a name of a group in EC2-VPC that the correct @@ -197,7 +197,7 @@ class BotoSecgroupTestCase(TestCase, LoaderModuleMockMixin): **conn_parameters) self.assertEqual(group_vpc.id, retrieved_group_id) - @mock_ec2 + @mock_ec2_deprecated def test_get_config_single_rule_group_name(self): ''' tests return of 'config' when given group name. get_config returns an OrderedDict. @@ -213,7 +213,7 @@ class BotoSecgroupTestCase(TestCase, LoaderModuleMockMixin): group = conn.create_security_group(name=group_name, description=group_name) group.authorize(ip_protocol=ip_protocol, from_port=from_port, to_port=to_port, cidr_ip=cidr_ip) # setup the expected get_config result - expected_get_config_result = OrderedDict([('name', group.name), ('group_id', group.id), ('owner_id', u'111122223333'), + expected_get_config_result = OrderedDict([('name', group.name), ('group_id', group.id), ('owner_id', u'123456789012'), ('description', group.description), ('tags', {}), ('rules', [{'to_port': to_port, 'from_port': from_port, 'ip_protocol': ip_protocol, 'cidr_ip': cidr_ip}]), @@ -221,7 +221,7 @@ class BotoSecgroupTestCase(TestCase, LoaderModuleMockMixin): secgroup_get_config_result = boto_secgroup.get_config(group_id=group.id, **conn_parameters) self.assertEqual(expected_get_config_result, secgroup_get_config_result) - @mock_ec2 + @mock_ec2_deprecated def test_exists_true_name_classic(self): ''' tests 'true' existence of a group in EC2-Classic when given name @@ -234,11 +234,11 @@ class BotoSecgroupTestCase(TestCase, LoaderModuleMockMixin): salt_exists_result = boto_secgroup.exists(name=group_name, **conn_parameters) self.assertTrue(salt_exists_result) - @mock_ec2 + @mock_ec2_deprecated def test_exists_false_name_classic(self): pass - @mock_ec2 + @mock_ec2_deprecated def test_exists_true_name_vpc(self): ''' tests 'true' existence of a group in EC2-VPC when given name and vpc_id @@ -250,7 +250,7 @@ class BotoSecgroupTestCase(TestCase, LoaderModuleMockMixin): salt_exists_result = boto_secgroup.exists(name=group_name, vpc_id=vpc_id, **conn_parameters) self.assertTrue(salt_exists_result) - @mock_ec2 + @mock_ec2_deprecated def test_exists_false_name_vpc(self): ''' tests 'false' existence of a group in vpc when given name and vpc_id @@ -259,7 +259,7 @@ class BotoSecgroupTestCase(TestCase, LoaderModuleMockMixin): salt_exists_result = boto_secgroup.exists(group_name, vpc_id=vpc_id, **conn_parameters) self.assertFalse(salt_exists_result) - @mock_ec2 + @mock_ec2_deprecated def test_exists_true_group_id(self): ''' tests 'true' existence of a group when given group_id @@ -271,7 +271,7 @@ class BotoSecgroupTestCase(TestCase, LoaderModuleMockMixin): salt_exists_result = boto_secgroup.exists(group_id=group.id, **conn_parameters) self.assertTrue(salt_exists_result) - @mock_ec2 + @mock_ec2_deprecated def test_exists_false_group_id(self): ''' tests 'false' existence of a group when given group_id @@ -280,7 +280,7 @@ class BotoSecgroupTestCase(TestCase, LoaderModuleMockMixin): salt_exists_result = boto_secgroup.exists(group_id=group_id, **conn_parameters) self.assertFalse(salt_exists_result) - @mock_ec2 + @mock_ec2_deprecated def test_delete_group_ec2_classic(self): ''' test deletion of a group in EC2-Classic. Test does the following: @@ -306,11 +306,11 @@ class BotoSecgroupTestCase(TestCase, LoaderModuleMockMixin): actual_groups = [group.id for group in conn.get_all_security_groups()] self.assertEqual(expected_groups, actual_groups) - @mock_ec2 + @mock_ec2_deprecated def test_delete_group_name_ec2_vpc(self): pass - @mock_ec2 + @mock_ec2_deprecated def test__get_conn_true(self): ''' tests ensures that _get_conn returns an boto.ec2.connection.EC2Connection object. diff --git a/tests/unit/modules/test_boto_vpc.py b/tests/unit/modules/test_boto_vpc.py index 2f53edf5e55..cda7fa29f39 100644 --- a/tests/unit/modules/test_boto_vpc.py +++ b/tests/unit/modules/test_boto_vpc.py @@ -26,7 +26,7 @@ from salt.exceptions import SaltInvocationError, CommandExecutionError from salt.modules.boto_vpc import _maybe_set_name_tag, _maybe_set_tags # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six # pylint: disable=import-error from salt.ext.six.moves import range # pylint: disable=redefined-builtin # pylint: disable=no-name-in-module,unused-import @@ -40,17 +40,17 @@ except ImportError: try: import moto - from moto import mock_ec2 + from moto import mock_ec2_deprecated HAS_MOTO = True except ImportError: HAS_MOTO = False - def mock_ec2(self): + def mock_ec2_deprecated(self): ''' - if the mock_ec2 function is not available due to import failure + if the mock_ec2_deprecated function is not available due to import failure this replaces the decorated function with stub_function. - Allows boto_vpc unit tests to use the @mock_ec2 decorator - without a "NameError: name 'mock_ec2' is not defined" error. + Allows boto_vpc unit tests to use the @mock_ec2_deprecated decorator + without a "NameError: name 'mock_ec2_deprecated' is not defined" error. ''' def stub_function(self): @@ -63,7 +63,7 @@ except ImportError: # which was added in boto 2.8.0 # https://github.com/boto/boto/commit/33ac26b416fbb48a60602542b4ce15dcc7029f12 required_boto_version = '2.8.0' -required_moto_version = '0.3.7' +required_moto_version = '1.0.0' region = 'us-east-1' access_key = 'GKTADJGHEIQSXMKKRBJ08H' @@ -272,7 +272,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): TestCase for salt.modules.boto_vpc module ''' - @mock_ec2 + @mock_ec2_deprecated def test_that_when_checking_if_a_vpc_exists_by_id_and_a_vpc_exists_the_vpc_exists_method_returns_true(self): ''' Tests checking vpc existence via id when the vpc already exists @@ -283,7 +283,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(vpc_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_checking_if_a_vpc_exists_by_id_and_a_vpc_does_not_exist_the_vpc_exists_method_returns_false( self): ''' @@ -295,7 +295,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(vpc_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_checking_if_a_vpc_exists_by_name_and_a_vpc_exists_the_vpc_exists_method_returns_true(self): ''' Tests checking vpc existence via name when vpc exists @@ -306,7 +306,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(vpc_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_checking_if_a_vpc_exists_by_name_and_a_vpc_does_not_exist_the_vpc_exists_method_returns_false( self): ''' @@ -318,7 +318,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(vpc_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_checking_if_a_vpc_exists_by_tags_and_a_vpc_exists_the_vpc_exists_method_returns_true(self): ''' Tests checking vpc existence via tag when vpc exists @@ -329,7 +329,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(vpc_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_checking_if_a_vpc_exists_by_tags_and_a_vpc_does_not_exist_the_vpc_exists_method_returns_false( self): ''' @@ -341,7 +341,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(vpc_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_checking_if_a_vpc_exists_by_cidr_and_a_vpc_exists_the_vpc_exists_method_returns_true(self): ''' Tests checking vpc existence via cidr when vpc exists @@ -352,7 +352,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(vpc_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_checking_if_a_vpc_exists_by_cidr_and_a_vpc_does_not_exist_the_vpc_exists_method_returns_false( self): ''' @@ -364,7 +364,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(vpc_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_that_when_checking_if_a_vpc_exists_but_providing_no_filters_the_vpc_exists_method_raises_a_salt_invocation_error(self): ''' @@ -375,7 +375,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): 'cidr or tags.'): boto_vpc.exists(**conn_parameters) - @mock_ec2 + @mock_ec2_deprecated def test_get_vpc_id_method_when_filtering_by_name(self): ''' Tests getting vpc id when filtering by name @@ -386,7 +386,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertEqual(vpc.id, get_id_result['id']) - @mock_ec2 + @mock_ec2_deprecated def test_get_vpc_id_method_when_filtering_by_invalid_name(self): ''' Tests getting vpc id when filtering by invalid name @@ -397,7 +397,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertEqual(get_id_result['id'], None) - @mock_ec2 + @mock_ec2_deprecated def test_get_vpc_id_method_when_filtering_by_cidr(self): ''' Tests getting vpc id when filtering by cidr @@ -408,7 +408,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertEqual(vpc.id, get_id_result['id']) - @mock_ec2 + @mock_ec2_deprecated def test_get_vpc_id_method_when_filtering_by_invalid_cidr(self): ''' Tests getting vpc id when filtering by invalid cidr @@ -419,7 +419,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertEqual(get_id_result['id'], None) - @mock_ec2 + @mock_ec2_deprecated def test_get_vpc_id_method_when_filtering_by_tags(self): ''' Tests getting vpc id when filtering by tags @@ -430,7 +430,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertEqual(vpc.id, get_id_result['id']) - @mock_ec2 + @mock_ec2_deprecated def test_get_vpc_id_method_when_filtering_by_invalid_tags(self): ''' Tests getting vpc id when filtering by invalid tags @@ -441,7 +441,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertEqual(get_id_result['id'], None) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_get_vpc_id_method_when_not_providing_filters_raises_a_salt_invocation_error(self): ''' @@ -450,7 +450,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): with self.assertRaisesRegex(SaltInvocationError, 'At least one of the following must be provided: vpc_id, vpc_name, cidr or tags.'): boto_vpc.get_id(**conn_parameters) - @mock_ec2 + @mock_ec2_deprecated def test_get_vpc_id_method_when_more_than_one_vpc_is_matched_raises_a_salt_command_execution_error(self): ''' Tests getting vpc id but providing no filters @@ -461,7 +461,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): with self.assertRaisesRegex(CommandExecutionError, 'Found more than one VPC matching the criteria.'): boto_vpc.get_id(cidr=u'10.0.0.0/24', **conn_parameters) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_a_vpc_succeeds_the_create_vpc_method_returns_true(self): ''' tests True VPC created. @@ -470,7 +470,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(vpc_creation_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_a_vpc_and_specifying_a_vpc_name_succeeds_the_create_vpc_method_returns_true(self): ''' tests True VPC created. @@ -479,7 +479,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(vpc_creation_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_a_vpc_and_specifying_tags_succeeds_the_create_vpc_method_returns_true(self): ''' tests True VPC created. @@ -488,7 +488,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(vpc_creation_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_that_when_creating_a_vpc_fails_the_create_vpc_method_returns_false(self): ''' @@ -499,7 +499,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(vpc_creation_result['created']) self.assertTrue('error' in vpc_creation_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_deleting_an_existing_vpc_the_delete_vpc_method_returns_true(self): ''' Tests deleting an existing vpc @@ -510,7 +510,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(vpc_deletion_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_deleting_a_non_existent_vpc_the_delete_vpc_method_returns_false(self): ''' Tests deleting a non-existent vpc @@ -519,7 +519,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(delete_vpc_result['deleted']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_describing_vpc_by_id_it_returns_the_dict_of_properties_returns_true(self): ''' Tests describing parameters via vpc id if vpc exist @@ -546,7 +546,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertEqual(describe_vpc, {'vpc': vpc_properties}) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_describing_vpc_by_id_it_returns_the_dict_of_properties_returns_false(self): ''' Tests describing parameters via vpc id if vpc does not exist @@ -557,7 +557,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(describe_vpc['vpc']) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_that_when_describing_vpc_by_id_on_connection_error_it_returns_error(self): ''' @@ -570,7 +570,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): describe_result = boto_vpc.describe(vpc_id=vpc.id, **conn_parameters) self.assertTrue('error' in describe_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_describing_vpc_but_providing_no_vpc_id_the_describe_method_raises_a_salt_invocation_error(self): ''' Tests describing vpc without vpc id @@ -588,7 +588,7 @@ class BotoVpcTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): .format(required_boto_version, _get_boto_version())) @skipIf(_has_required_moto() is False, 'The moto version must be >= to version {0}'.format(required_moto_version)) class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): - @mock_ec2 + @mock_ec2_deprecated def test_get_subnet_association_single_subnet(self): ''' tests that given multiple subnet ids in the same VPC that the VPC ID is @@ -601,7 +601,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): **conn_parameters) self.assertEqual(vpc.id, subnet_association['vpc_id']) - @mock_ec2 + @mock_ec2_deprecated def test_get_subnet_association_multiple_subnets_same_vpc(self): ''' tests that given multiple subnet ids in the same VPC that the VPC ID is @@ -614,7 +614,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): **conn_parameters) self.assertEqual(vpc.id, subnet_association['vpc_id']) - @mock_ec2 + @mock_ec2_deprecated def test_get_subnet_association_multiple_subnets_different_vpc(self): ''' tests that given multiple subnet ids in different VPCs that False is @@ -628,7 +628,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): **conn_parameters) self.assertEqual(set(subnet_association['vpc_ids']), set([vpc_a.id, vpc_b.id])) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_a_subnet_succeeds_the_create_subnet_method_returns_true(self): ''' Tests creating a subnet successfully @@ -640,7 +640,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(subnet_creation_result['created']) self.assertTrue('id' in subnet_creation_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_a_subnet_and_specifying_a_name_succeeds_the_create_subnet_method_returns_true(self): ''' Tests creating a subnet successfully when specifying a name @@ -651,7 +651,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(subnet_creation_result['created']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_a_subnet_and_specifying_tags_succeeds_the_create_subnet_method_returns_true(self): ''' Tests creating a subnet successfully when specifying a tag @@ -663,7 +663,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(subnet_creation_result['created']) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_that_when_creating_a_subnet_fails_the_create_subnet_method_returns_error(self): ''' @@ -675,7 +675,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): subnet_creation_result = boto_vpc.create_subnet(vpc.id, '10.0.0.0/24', **conn_parameters) self.assertTrue('error' in subnet_creation_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_deleting_an_existing_subnet_the_delete_subnet_method_returns_true(self): ''' Tests deleting an existing subnet @@ -687,7 +687,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(subnet_deletion_result['deleted']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_deleting_a_non_existent_subnet_the_delete_vpc_method_returns_false(self): ''' Tests deleting a subnet that doesn't exist @@ -695,7 +695,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): delete_subnet_result = boto_vpc.delete_subnet(subnet_id='1234', **conn_parameters) self.assertTrue('error' in delete_subnet_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_checking_if_a_subnet_exists_by_id_the_subnet_exists_method_returns_true(self): ''' Tests checking if a subnet exists when it does exist @@ -707,7 +707,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(subnet_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_a_subnet_does_not_exist_the_subnet_exists_method_returns_false(self): ''' Tests checking if a subnet exists which doesn't exist @@ -716,7 +716,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(subnet_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_checking_if_a_subnet_exists_by_name_the_subnet_exists_method_returns_true(self): ''' Tests checking subnet existence by name @@ -728,7 +728,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(subnet_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_checking_if_a_subnet_exists_by_name_the_subnet_does_not_exist_the_subnet_method_returns_false(self): ''' Tests checking subnet existence by name when it doesn't exist @@ -740,7 +740,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(subnet_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_checking_if_a_subnet_exists_by_tags_the_subnet_exists_method_returns_true(self): ''' Tests checking subnet existence by tag @@ -752,7 +752,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(subnet_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_checking_if_a_subnet_exists_by_tags_the_subnet_does_not_exist_the_subnet_method_returns_false(self): ''' Tests checking subnet existence by tag when subnet doesn't exist @@ -764,7 +764,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(subnet_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_that_when_checking_if_a_subnet_exists_but_providing_no_filters_the_subnet_exists_method_raises_a_salt_invocation_error(self): ''' @@ -776,7 +776,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): boto_vpc.subnet_exists(**conn_parameters) @skipIf(True, 'Skip these tests while investigating failures') - @mock_ec2 + @mock_ec2_deprecated def test_that_describe_subnet_by_id_for_existing_subnet_returns_correct_data(self): ''' Tests describing a subnet by id. @@ -791,7 +791,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertEqual(set(describe_subnet_results['subnet'].keys()), set(['id', 'cidr_block', 'availability_zone', 'tags'])) - @mock_ec2 + @mock_ec2_deprecated def test_that_describe_subnet_by_id_for_non_existent_subnet_returns_none(self): ''' Tests describing a non-existent subnet by id. @@ -805,7 +805,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertEqual(describe_subnet_results['subnet'], None) @skipIf(True, 'Skip these tests while investigating failures') - @mock_ec2 + @mock_ec2_deprecated def test_that_describe_subnet_by_name_for_existing_subnet_returns_correct_data(self): ''' Tests describing a subnet by name. @@ -820,7 +820,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertEqual(set(describe_subnet_results['subnet'].keys()), set(['id', 'cidr_block', 'availability_zone', 'tags'])) - @mock_ec2 + @mock_ec2_deprecated def test_that_describe_subnet_by_name_for_non_existent_subnet_returns_none(self): ''' Tests describing a non-existent subnet by id. @@ -834,7 +834,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertEqual(describe_subnet_results['subnet'], None) @skipIf(True, 'Skip these tests while investigating failures') - @mock_ec2 + @mock_ec2_deprecated def test_that_describe_subnets_by_id_for_existing_subnet_returns_correct_data(self): ''' Tests describing multiple subnets by id. @@ -852,7 +852,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): set(['id', 'cidr_block', 'availability_zone', 'tags'])) @skipIf(True, 'Skip these tests while investigating failures') - @mock_ec2 + @mock_ec2_deprecated def test_that_describe_subnets_by_name_for_existing_subnets_returns_correct_data(self): ''' Tests describing multiple subnets by id. @@ -869,7 +869,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertEqual(set(describe_subnet_results['subnets'][0].keys()), set(['id', 'cidr_block', 'availability_zone', 'tags'])) - @mock_ec2 + @mock_ec2_deprecated def test_create_subnet_passes_availability_zone(self): ''' Tests that the availability_zone kwarg is passed on to _create_resource @@ -890,7 +890,7 @@ class BotoVpcSubnetsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): ' or equal to version {}. Installed: {}' .format(required_boto_version, _get_boto_version())) class BotoVpcInternetGatewayTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_an_internet_gateway_the_create_internet_gateway_method_returns_true(self): ''' Tests creating an internet gateway successfully (with no vpc id or name) @@ -901,7 +901,7 @@ class BotoVpcInternetGatewayTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): keyid=access_key) self.assertTrue(igw_creation_result.get('created')) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_an_internet_gateway_with_non_existent_vpc_the_create_internet_gateway_method_returns_an_error(self): ''' Tests that creating an internet gateway for a non-existent VPC fails. @@ -913,7 +913,7 @@ class BotoVpcInternetGatewayTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): vpc_name='non-existent-vpc') self.assertTrue('error' in igw_creation_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_an_internet_gateway_with_vpc_name_specified_the_create_internet_gateway_method_returns_true(self): ''' Tests creating an internet gateway with vpc name specified. @@ -928,7 +928,7 @@ class BotoVpcInternetGatewayTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(igw_creation_result.get('created')) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_an_internet_gateway_with_vpc_id_specified_the_create_internet_gateway_method_returns_true(self): ''' Tests creating an internet gateway with vpc name specified. @@ -951,7 +951,7 @@ class BotoVpcInternetGatewayTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): ' or equal to version {}. Installed: {}' .format(required_boto_version, _get_boto_version())) class BotoVpcNatGatewayTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_an_nat_gateway_the_create_nat_gateway_method_returns_true(self): ''' Tests creating an nat gateway successfully (with subnet_id specified) @@ -965,7 +965,7 @@ class BotoVpcNatGatewayTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): keyid=access_key) self.assertTrue(ngw_creation_result.get('created')) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_an_nat_gateway_with_non_existent_subnet_the_create_nat_gateway_method_returns_an_error(self): ''' Tests that creating an nat gateway for a non-existent subnet fails. @@ -977,7 +977,7 @@ class BotoVpcNatGatewayTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): subnet_name='non-existent-subnet') self.assertTrue('error' in ngw_creation_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_an_nat_gateway_with_subnet_name_specified_the_create_nat_gateway_method_returns_true(self): ''' Tests creating an nat gateway with subnet name specified. @@ -1000,7 +1000,7 @@ class BotoVpcNatGatewayTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): ' or equal to version {}. Installed: {}' .format(required_boto_version, _get_boto_version())) class BotoVpcCustomerGatewayTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_creating_a_customer_gateway_the_create_customer_gateway_method_returns_true(self): ''' @@ -1010,7 +1010,7 @@ class BotoVpcCustomerGatewayTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): gw_creation_result = boto_vpc.create_customer_gateway('ipsec.1', '10.1.1.1', None) self.assertTrue(gw_creation_result.get('created')) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_checking_if_a_subnet_exists_by_id_the_subnet_exists_method_returns_true(self): ''' @@ -1021,7 +1021,7 @@ class BotoVpcCustomerGatewayTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): gw_exists_result = boto_vpc.customer_gateway_exists(customer_gateway_id=gw_creation_result['id']) self.assertTrue(gw_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_a_subnet_does_not_exist_the_subnet_exists_method_returns_false(self): ''' @@ -1039,7 +1039,7 @@ class BotoVpcCustomerGatewayTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): .format(required_boto_version, _get_boto_version())) @skipIf(_has_required_moto() is False, 'The moto version must be >= to version {0}'.format(required_moto_version)) class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_dhcp_options_succeeds_the_create_dhcp_options_method_returns_true(self): ''' Tests creating dhcp options successfully @@ -1048,7 +1048,7 @@ class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(dhcp_options_creation_result['created']) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_creating_dhcp_options_and_specifying_a_name_succeeds_the_create_dhcp_options_method_returns_true( self): @@ -1060,7 +1060,7 @@ class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(dhcp_options_creation_result['created']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_dhcp_options_and_specifying_tags_succeeds_the_create_dhcp_options_method_returns_true( self): ''' @@ -1071,7 +1071,7 @@ class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(dhcp_options_creation_result['created']) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_that_when_creating_dhcp_options_fails_the_create_dhcp_options_method_returns_error(self): ''' @@ -1082,7 +1082,7 @@ class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): r = dhcp_options_creation_result = boto_vpc.create_dhcp_options(**dhcp_options_parameters) self.assertTrue('error' in r) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_associating_an_existing_dhcp_options_set_to_an_existing_vpc_the_associate_dhcp_options_method_returns_true( self): ''' @@ -1096,7 +1096,7 @@ class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(dhcp_options_association_result['associated']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_associating_a_non_existent_dhcp_options_set_to_an_existing_vpc_the_associate_dhcp_options_method_returns_error( self): ''' @@ -1108,7 +1108,7 @@ class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue('error' in dhcp_options_association_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_associating_an_existing_dhcp_options_set_to_a_non_existent_vpc_the_associate_dhcp_options_method_returns_false( self): ''' @@ -1121,7 +1121,7 @@ class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue('error' in dhcp_options_association_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_dhcp_options_set_to_an_existing_vpc_succeeds_the_associate_new_dhcp_options_method_returns_true( self): ''' @@ -1133,7 +1133,7 @@ class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(dhcp_creation_result['created']) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_that_when_creating_and_associating_dhcp_options_set_to_an_existing_vpc_fails_creating_the_dhcp_options_the_associate_new_dhcp_options_method_raises_exception( self): @@ -1147,7 +1147,7 @@ class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): r = boto_vpc.associate_new_dhcp_options_to_vpc(vpc.id, **dhcp_options_parameters) self.assertTrue('error' in r) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_that_when_creating_and_associating_dhcp_options_set_to_an_existing_vpc_fails_associating_the_dhcp_options_the_associate_new_dhcp_options_method_raises_exception(self): ''' @@ -1160,7 +1160,7 @@ class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): r = boto_vpc.associate_new_dhcp_options_to_vpc(vpc.id, **dhcp_options_parameters) self.assertTrue('error' in r) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_dhcp_options_set_to_a_non_existent_vpc_the_dhcp_options_the_associate_new_dhcp_options_method_returns_false( self): ''' @@ -1170,7 +1170,7 @@ class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): r = boto_vpc.create_dhcp_options(vpc_name='fake', **dhcp_options_parameters) self.assertTrue('error' in r) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_dhcp_options_exists_the_dhcp_options_exists_method_returns_true(self): ''' Tests existence of dhcp options successfully @@ -1181,7 +1181,7 @@ class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(dhcp_options_exists_result['exists']) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_dhcp_options_do_not_exist_the_dhcp_options_exists_method_returns_false(self): ''' Tests existence of dhcp options failure @@ -1189,7 +1189,7 @@ class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): r = boto_vpc.dhcp_options_exists('fake', **conn_parameters) self.assertFalse(r['exists']) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_that_when_checking_if_dhcp_options_exists_but_providing_no_filters_the_dhcp_options_exists_method_raises_a_salt_invocation_error(self): ''' @@ -1206,7 +1206,7 @@ class BotoVpcDHCPOptionsTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): ' or equal to version {}. Installed: {}' .format(required_boto_version, _get_boto_version())) class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_network_acl_for_an_existing_vpc_the_create_network_acl_method_returns_true(self): ''' Tests creation of network acl with existing vpc @@ -1217,7 +1217,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(network_acl_creation_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_network_acl_for_an_existing_vpc_and_specifying_a_name_the_create_network_acl_method_returns_true( self): ''' @@ -1229,7 +1229,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(network_acl_creation_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_network_acl_for_an_existing_vpc_and_specifying_tags_the_create_network_acl_method_returns_true( self): ''' @@ -1241,7 +1241,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(network_acl_creation_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_network_acl_for_a_non_existent_vpc_the_create_network_acl_method_returns_an_error(self): ''' Tests creation of network acl with a non-existent vpc @@ -1250,7 +1250,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue('error' in network_acl_creation_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_creating_network_acl_fails_the_create_network_acl_method_returns_false(self): ''' @@ -1264,7 +1264,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(network_acl_creation_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_deleting_an_existing_network_acl_the_delete_network_acl_method_returns_true(self): ''' Tests deletion of existing network acl successfully @@ -1276,7 +1276,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(network_acl_deletion_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_deleting_a_non_existent_network_acl_the_delete_network_acl_method_returns_an_error(self): ''' Tests deleting a non-existent network acl @@ -1285,7 +1285,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue('error' in network_acl_deletion_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_a_network_acl_exists_the_network_acl_exists_method_returns_true(self): ''' Tests existence of network acl @@ -1297,7 +1297,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(network_acl_deletion_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_a_network_acl_does_not_exist_the_network_acl_exists_method_returns_false(self): ''' Tests checking network acl does not exist @@ -1306,7 +1306,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(network_acl_deletion_result['exists']) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_that_when_checking_if_network_acl_exists_but_providing_no_filters_the_network_acl_exists_method_raises_a_salt_invocation_error(self): ''' @@ -1318,7 +1318,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): ): boto_vpc.dhcp_options_exists(**conn_parameters) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_creating_a_network_acl_entry_successfully_the_create_network_acl_entry_method_returns_true(self): ''' @@ -1333,7 +1333,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(network_acl_entry_creation_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_creating_a_network_acl_entry_for_a_non_existent_network_acl_the_create_network_acl_entry_method_returns_false( self): @@ -1345,7 +1345,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(network_acl_entry_creation_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_replacing_a_network_acl_entry_successfully_the_replace_network_acl_entry_method_returns_true( self): @@ -1362,7 +1362,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(network_acl_entry_creation_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_replacing_a_network_acl_entry_for_a_non_existent_network_acl_the_replace_network_acl_entry_method_returns_false( self): @@ -1373,7 +1373,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): **conn_parameters) self.assertFalse(network_acl_entry_creation_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_deleting_an_existing_network_acl_entry_the_delete_network_acl_entry_method_returns_true(self): ''' @@ -1388,7 +1388,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(network_acl_entry_deletion_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_deleting_a_non_existent_network_acl_entry_the_delete_network_acl_entry_method_returns_false( self): @@ -1400,7 +1400,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(network_acl_entry_deletion_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_associating_an_existing_network_acl_to_an_existing_subnet_the_associate_network_acl_method_returns_true( self): @@ -1416,7 +1416,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(network_acl_association_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_associating_a_non_existent_network_acl_to_an_existing_subnet_the_associate_network_acl_method_returns_an_error( self): ''' @@ -1430,7 +1430,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue('error' in network_acl_association_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_associating_an_existing_network_acl_to_a_non_existent_subnet_the_associate_network_acl_method_returns_false( self): @@ -1445,7 +1445,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(network_acl_association_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_creating_and_associating_a_network_acl_to_a_subnet_succeeds_the_associate_new_network_acl_to_subnet_method_returns_true( self): @@ -1460,7 +1460,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(network_acl_creation_and_association_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_creating_and_associating_a_network_acl_to_a_subnet_and_specifying_a_name_succeeds_the_associate_new_network_acl_to_subnet_method_returns_true( self): @@ -1476,7 +1476,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(network_acl_creation_and_association_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_creating_and_associating_a_network_acl_to_a_subnet_and_specifying_tags_succeeds_the_associate_new_network_acl_to_subnet_method_returns_true( self): @@ -1493,7 +1493,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(network_acl_creation_and_association_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_creating_and_associating_a_network_acl_to_a_non_existent_subnet_the_associate_new_network_acl_to_subnet_method_returns_false( self): @@ -1507,7 +1507,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(network_acl_creation_and_association_result) - @mock_ec2 + @mock_ec2_deprecated def test_that_when_creating_a_network_acl_to_a_non_existent_vpc_the_associate_new_network_acl_to_subnet_method_returns_an_error( self): ''' @@ -1520,7 +1520,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue('error' in network_acl_creation_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_disassociating_network_acl_succeeds_the_disassociate_network_acl_method_should_return_true(self): ''' @@ -1533,7 +1533,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(dhcp_disassociate_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_disassociating_network_acl_for_a_non_existent_vpc_the_disassociate_network_acl_method_should_return_false( self): @@ -1547,7 +1547,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(dhcp_disassociate_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_disassociating_network_acl_for_a_non_existent_subnet_the_disassociate_network_acl_method_should_return_false( self): @@ -1568,7 +1568,7 @@ class BotoVpcNetworkACLTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): ' or equal to version {}. Installed: {}' .format(required_boto_version, _get_boto_version())) class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_creating_a_route_table_succeeds_the_create_route_table_method_returns_true(self): ''' @@ -1580,7 +1580,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(route_table_creation_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_creating_a_route_table_on_a_non_existent_vpc_the_create_route_table_method_returns_false(self): ''' @@ -1590,7 +1590,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(route_table_creation_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_deleting_a_route_table_succeeds_the_delete_route_table_method_returns_true(self): ''' @@ -1603,7 +1603,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(route_table_deletion_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_deleting_a_non_existent_route_table_the_delete_route_table_method_returns_false(self): ''' @@ -1613,7 +1613,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(route_table_deletion_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_route_table_exists_the_route_table_exists_method_returns_true(self): ''' @@ -1626,7 +1626,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(route_table_existence_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_route_table_does_not_exist_the_route_table_exists_method_returns_false(self): ''' @@ -1636,7 +1636,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(route_table_existence_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_that_when_checking_if_a_route_table_exists_but_providing_no_filters_the_route_table_exists_method_raises_a_salt_invocation_error(self): ''' @@ -1648,7 +1648,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): ): boto_vpc.dhcp_options_exists(**conn_parameters) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_associating_a_route_table_succeeds_the_associate_route_table_method_should_return_the_association_id( self): @@ -1663,7 +1663,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(association_id) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_associating_a_route_table_with_a_non_existent_route_table_the_associate_route_table_method_should_return_false( self): @@ -1677,7 +1677,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(association_id) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_associating_a_route_table_with_a_non_existent_subnet_the_associate_route_table_method_should_return_false( self): @@ -1691,7 +1691,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(association_id) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_disassociating_a_route_table_succeeds_the_disassociate_route_table_method_should_return_true( self): @@ -1708,7 +1708,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(dhcp_disassociate_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_creating_a_route_succeeds_the_create_route_method_should_return_true(self): ''' @@ -1721,7 +1721,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(route_creation_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_creating_a_route_with_a_non_existent_route_table_the_create_route_method_should_return_false( self): @@ -1732,7 +1732,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(route_creation_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_deleting_a_route_succeeds_the_delete_route_method_should_return_true(self): ''' @@ -1745,7 +1745,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(route_deletion_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_deleting_a_route_with_a_non_existent_route_table_the_delete_route_method_should_return_false( self): @@ -1756,7 +1756,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(route_deletion_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_replacing_a_route_succeeds_the_replace_route_method_should_return_true(self): ''' @@ -1769,7 +1769,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(route_replacing_result) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Moto has not implemented this feature. Skipping for now.') def test_that_when_replacing_a_route_with_a_non_existent_route_table_the_replace_route_method_should_return_false( self): @@ -1790,7 +1790,7 @@ class BotoVpcRouteTablesTestCase(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): @skipIf(_has_required_moto() is False, 'The moto version must be >= to version {0}'.format(required_moto_version)) class BotoVpcPeeringConnectionsTest(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): - @mock_ec2 + @mock_ec2_deprecated def test_request_vpc_peering_connection(self): ''' Run with 2 vpc ids and returns a message @@ -1803,7 +1803,7 @@ class BotoVpcPeeringConnectionsTest(BotoVpcTestCaseBase, BotoVpcTestCaseMixin): peer_vpc_id=other_vpc.id, **conn_parameters)) - @mock_ec2 + @mock_ec2_deprecated def test_raises_error_if_both_vpc_name_and_vpc_id_are_specified(self): ''' Must specify only one diff --git a/tests/unit/modules/test_btrfs.py b/tests/unit/modules/test_btrfs.py index 3b4de68a52c..4b727fd0fcb 100644 --- a/tests/unit/modules/test_btrfs.py +++ b/tests/unit/modules/test_btrfs.py @@ -17,7 +17,7 @@ from tests.support.mock import ( ) # Import Salt Libs -import salt +import salt.utils.files import salt.utils.fsutils import salt.modules.btrfs as btrfs from salt.exceptions import CommandExecutionError @@ -88,7 +88,7 @@ class BtrfsTestCase(TestCase, LoaderModuleMockMixin): 'stdout': 'Salt'}) with patch.dict(btrfs.__salt__, {'cmd.run_all': mock_run}): mock_file = mock_open(read_data='/dev/sda1 / ext4 rw,data=ordered 0 0') - with patch.object(salt.utils, 'fopen', mock_file): + with patch.object(salt.utils.files, 'fopen', mock_file): self.assertListEqual(btrfs.defragment('/dev/sda1'), ret) def test_defragment_error(self): @@ -101,7 +101,7 @@ class BtrfsTestCase(TestCase, LoaderModuleMockMixin): 'stdout': 'Salt'}) with patch.dict(btrfs.__salt__, {'cmd.run_all': mock_run}): mock_file = mock_open(read_data='/dev/sda1 / ext4 rw,data=ordered 0 0') - with patch.object(salt.utils, 'fopen', mock_file): + with patch.object(salt.utils.files, 'fopen', mock_file): self.assertRaises(CommandExecutionError, btrfs.defragment, '/dev/sda1') @@ -165,7 +165,7 @@ class BtrfsTestCase(TestCase, LoaderModuleMockMixin): with patch.dict(btrfs.__salt__, {'cmd.run_all': mock_cmd, 'btrfs.info': mock_info}): mock_file = mock_open(read_data='/dev/sda1 / ext4 rw,data=ordered 0 0') - with patch.object(salt.utils, 'fopen', mock_file): + with patch.object(salt.utils.files, 'fopen', mock_file): self.assertDictEqual(btrfs.mkfs('/dev/sda1'), {'log': 'Salt'}) def test_mkfs_error(self): @@ -313,7 +313,7 @@ class BtrfsTestCase(TestCase, LoaderModuleMockMixin): mock_blk = MagicMock(return_value={'/dev/sda1': {'type': 'ext4'}}) with patch.object(salt.utils.fsutils, '_blkid_output', mock_blk): mock_file = mock_open(read_data='/dev/sda1 / ext4 rw,data=ordered 0 0') - with patch.object(salt.utils, 'fopen', mock_file): + with patch.object(salt.utils.files, 'fopen', mock_file): self.assertRaises(CommandExecutionError, btrfs.convert, '/dev/sda1') diff --git a/tests/unit/modules/test_cassandra.py b/tests/unit/modules/test_cassandra.py index 90a15593fbc..6b34a0768f8 100644 --- a/tests/unit/modules/test_cassandra.py +++ b/tests/unit/modules/test_cassandra.py @@ -16,7 +16,7 @@ from tests.support.mock import ( ) # Import Salt Libs -import salt.ext.six as six +from salt.ext import six import salt.modules.cassandra as cassandra diff --git a/tests/unit/modules/test_chef.py b/tests/unit/modules/test_chef.py index 0899ce3a47c..46d0aae354c 100644 --- a/tests/unit/modules/test_chef.py +++ b/tests/unit/modules/test_chef.py @@ -25,7 +25,7 @@ class ChefTestCase(TestCase, LoaderModuleMockMixin): Test cases for salt.modules.chef ''' def setup_loader_modules(self): - patcher = patch('salt.utils.which', MagicMock(return_value=True)) + patcher = patch('salt.utils.path.which', MagicMock(return_value=True)) patcher.start() self.addCleanup(patcher.stop) return {chef: {'_exec_cmd': MagicMock(return_value={})}} @@ -36,7 +36,8 @@ class ChefTestCase(TestCase, LoaderModuleMockMixin): ''' Test if it execute a chef client run and return a dict ''' - self.assertDictEqual(chef.client(), {}) + with patch.dict(chef.__opts__, {'cachedir': r'c:\salt\var\cache\salt\minion'}): + self.assertDictEqual(chef.client(), {}) # 'solo' function tests: 1 @@ -44,4 +45,5 @@ class ChefTestCase(TestCase, LoaderModuleMockMixin): ''' Test if it execute a chef solo run and return a dict ''' - self.assertDictEqual(chef.solo('/dev/sda1'), {}) + with patch.dict(chef.__opts__, {'cachedir': r'c:\salt\var\cache\salt\minion'}): + self.assertDictEqual(chef.solo('/dev/sda1'), {}) diff --git a/tests/unit/modules/test_cmdmod.py b/tests/unit/modules/test_cmdmod.py index ac57e059d7e..ca2f879d5a7 100644 --- a/tests/unit/modules/test_cmdmod.py +++ b/tests/unit/modules/test_cmdmod.py @@ -10,7 +10,7 @@ import sys import tempfile # Import Salt Libs -import salt.utils +import salt.utils.platform import salt.modules.cmdmod as cmdmod from salt.exceptions import CommandExecutionError from salt.log import LOG_LEVELS @@ -121,7 +121,7 @@ class CMDMODTestCase(TestCase, LoaderModuleMockMixin): and os.path.isfile returns False ''' with patch('salt.modules.cmdmod._is_valid_shell', MagicMock(return_value=True)): - with patch('salt.utils.is_windows', MagicMock(return_value=False)): + with patch('salt.utils.platform.is_windows', MagicMock(return_value=False)): with patch('os.path.isfile', MagicMock(return_value=False)): self.assertRaises(CommandExecutionError, cmdmod._run, 'foo', 'bar') @@ -131,7 +131,7 @@ class CMDMODTestCase(TestCase, LoaderModuleMockMixin): os.path.isfile returns True, but os.access returns False ''' with patch('salt.modules.cmdmod._is_valid_shell', MagicMock(return_value=True)): - with patch('salt.utils.is_windows', MagicMock(return_value=False)): + with patch('salt.utils.platform.is_windows', MagicMock(return_value=False)): with patch('os.path.isfile', MagicMock(return_value=True)): with patch('os.access', MagicMock(return_value=False)): self.assertRaises(CommandExecutionError, cmdmod._run, 'foo', 'bar') @@ -141,7 +141,7 @@ class CMDMODTestCase(TestCase, LoaderModuleMockMixin): Tests error raised when runas is passed on windows ''' with patch('salt.modules.cmdmod._is_valid_shell', MagicMock(return_value=True)): - with patch('salt.utils.is_windows', MagicMock(return_value=True)): + with patch('salt.utils.platform.is_windows', MagicMock(return_value=True)): with patch.dict(cmdmod.__grains__, {'os': 'fake_os'}): self.assertRaises(CommandExecutionError, cmdmod._run, @@ -161,7 +161,7 @@ class CMDMODTestCase(TestCase, LoaderModuleMockMixin): Tests error raised when umask is set to zero ''' with patch('salt.modules.cmdmod._is_valid_shell', MagicMock(return_value=True)): - with patch('salt.utils.is_windows', MagicMock(return_value=False)): + with patch('salt.utils.platform.is_windows', MagicMock(return_value=False)): with patch('os.path.isfile', MagicMock(return_value=True)): with patch('os.access', MagicMock(return_value=True)): self.assertRaises(CommandExecutionError, cmdmod._run, 'foo', 'bar', umask=0) @@ -171,7 +171,7 @@ class CMDMODTestCase(TestCase, LoaderModuleMockMixin): Tests error raised when an invalid umask is given ''' with patch('salt.modules.cmdmod._is_valid_shell', MagicMock(return_value=True)): - with patch('salt.utils.is_windows', MagicMock(return_value=False)): + with patch('salt.utils.platform.is_windows', MagicMock(return_value=False)): with patch('os.path.isfile', MagicMock(return_value=True)): with patch('os.access', MagicMock(return_value=True)): self.assertRaises(CommandExecutionError, cmdmod._run, 'foo', 'bar', umask='baz') @@ -181,7 +181,7 @@ class CMDMODTestCase(TestCase, LoaderModuleMockMixin): Tests error raised when cwd is not an absolute path ''' with patch('salt.modules.cmdmod._is_valid_shell', MagicMock(return_value=True)): - with patch('salt.utils.is_windows', MagicMock(return_value=False)): + with patch('salt.utils.platform.is_windows', MagicMock(return_value=False)): with patch('os.path.isfile', MagicMock(return_value=True)): with patch('os.access', MagicMock(return_value=True)): self.assertRaises(CommandExecutionError, cmdmod._run, 'foo', 'bar') @@ -191,7 +191,7 @@ class CMDMODTestCase(TestCase, LoaderModuleMockMixin): Tests error raised when cwd is not a dir ''' with patch('salt.modules.cmdmod._is_valid_shell', MagicMock(return_value=True)): - with patch('salt.utils.is_windows', MagicMock(return_value=False)): + with patch('salt.utils.platform.is_windows', MagicMock(return_value=False)): with patch('os.path.isfile', MagicMock(return_value=True)): with patch('os.access', MagicMock(return_value=True)): with patch('os.path.isabs', MagicMock(return_value=True)): @@ -202,7 +202,7 @@ class CMDMODTestCase(TestCase, LoaderModuleMockMixin): Tests error raised when not useing vt and OSError is provided ''' with patch('salt.modules.cmdmod._is_valid_shell', MagicMock(return_value=True)): - with patch('salt.utils.is_windows', MagicMock(return_value=False)): + with patch('salt.utils.platform.is_windows', MagicMock(return_value=False)): with patch('os.path.isfile', MagicMock(return_value=True)): with patch('os.access', MagicMock(return_value=True)): with patch('salt.utils.timed_subprocess.TimedProc', MagicMock(side_effect=OSError)): @@ -213,19 +213,19 @@ class CMDMODTestCase(TestCase, LoaderModuleMockMixin): Tests error raised when not useing vt and IOError is provided ''' with patch('salt.modules.cmdmod._is_valid_shell', MagicMock(return_value=True)): - with patch('salt.utils.is_windows', MagicMock(return_value=False)): + with patch('salt.utils.platform.is_windows', MagicMock(return_value=False)): with patch('os.path.isfile', MagicMock(return_value=True)): with patch('os.access', MagicMock(return_value=True)): with patch('salt.utils.timed_subprocess.TimedProc', MagicMock(side_effect=IOError)): self.assertRaises(CommandExecutionError, cmdmod._run, 'foo') - @skipIf(salt.utils.is_windows(), 'Do not run on Windows') + @skipIf(salt.utils.platform.is_windows(), 'Do not run on Windows') def test_run(self): ''' Tests end result when a command is not found ''' with patch('salt.modules.cmdmod._is_valid_shell', MagicMock(return_value=True)): - with patch('salt.utils.is_windows', MagicMock(return_value=False)): + with patch('salt.utils.platform.is_windows', MagicMock(return_value=False)): with patch('os.path.isfile', MagicMock(return_value=True)): with patch('os.access', MagicMock(return_value=True)): ret = cmdmod._run('foo', use_vt=True).get('stderr') @@ -235,10 +235,10 @@ class CMDMODTestCase(TestCase, LoaderModuleMockMixin): ''' Tests return if running on windows ''' - with patch('salt.utils.is_windows', MagicMock(return_value=True)): + with patch('salt.utils.platform.is_windows', MagicMock(return_value=True)): self.assertTrue(cmdmod._is_valid_shell('foo')) - @skipIf(salt.utils.is_windows(), 'Do not run on Windows') + @skipIf(salt.utils.platform.is_windows(), 'Do not run on Windows') def test_is_valid_shell_none(self): ''' Tests return of when os.path.exists(/etc/shells) isn't available @@ -251,18 +251,19 @@ class CMDMODTestCase(TestCase, LoaderModuleMockMixin): Tests return when provided shell is available ''' with patch('os.path.exists', MagicMock(return_value=True)): - with patch('salt.utils.fopen', mock_open(read_data=MOCK_SHELL_FILE)): + with patch('salt.utils.files.fopen', mock_open(read_data=MOCK_SHELL_FILE)): self.assertTrue(cmdmod._is_valid_shell('/bin/bash')) - @skipIf(salt.utils.is_windows(), 'Do not run on Windows') + @skipIf(salt.utils.platform.is_windows(), 'Do not run on Windows') def test_is_valid_shell_unavailable(self): ''' Tests return when provided shell is not available ''' with patch('os.path.exists', MagicMock(return_value=True)): - with patch('salt.utils.fopen', mock_open(read_data=MOCK_SHELL_FILE)): + with patch('salt.utils.files.fopen', mock_open(read_data=MOCK_SHELL_FILE)): self.assertFalse(cmdmod._is_valid_shell('foo')) + @skipIf(salt.utils.platform.is_windows(), 'Do not run on Windows') def test_os_environment_remains_intact(self): ''' Make sure the OS environment is not tainted after running a command diff --git a/tests/unit/modules/test_cp.py b/tests/unit/modules/test_cp.py index 9030fe4c3f7..abff94183b3 100644 --- a/tests/unit/modules/test_cp.py +++ b/tests/unit/modules/test_cp.py @@ -19,10 +19,10 @@ from tests.support.mock import ( ) # Import Salt Libs -import salt.utils +import salt.utils.files +import salt.utils.templates as templates import salt.transport import salt.modules.cp as cp -from salt.utils import templates from salt.exceptions import CommandExecutionError @@ -62,7 +62,7 @@ class CpTestCase(TestCase, LoaderModuleMockMixin): 'data': file_data} with patch.dict(templates.TEMPLATE_REGISTRY, {'jinja': mock_jinja}): - with patch('salt.utils.fopen', mock_open(read_data=file_data)): + with patch('salt.utils.files.fopen', mock_open(read_data=file_data)): self.assertRaises(CommandExecutionError, cp._render_filenames, path, dest, saltenv, template) @@ -78,10 +78,10 @@ class CpTestCase(TestCase, LoaderModuleMockMixin): file_data = '/srv/salt/biscuits' mock_jinja = lambda *args, **kwargs: {'result': True, 'data': file_data} - ret = (file_data, file_data) # salt.utils.fopen can only be mocked once + ret = (file_data, file_data) # salt.utils.files.fopen can only be mocked once with patch.dict(templates.TEMPLATE_REGISTRY, {'jinja': mock_jinja}): - with patch('salt.utils.fopen', mock_open(read_data=file_data)): + with patch('salt.utils.files.fopen', mock_open(read_data=file_data)): self.assertEqual(cp._render_filenames( path, dest, saltenv, template), ret) @@ -104,7 +104,7 @@ class CpTestCase(TestCase, LoaderModuleMockMixin): file_data = 'Remember to keep your files well salted.' saltenv = 'base' ret = file_data - with patch('salt.utils.fopen', mock_open(read_data=file_data)): + with patch('salt.utils.files.fopen', mock_open(read_data=file_data)): with patch('salt.modules.cp.cache_file', MagicMock(return_value=dest)): self.assertEqual(cp.get_file_str(path, dest), ret) @@ -136,14 +136,14 @@ class CpTestCase(TestCase, LoaderModuleMockMixin): patch.multiple('salt.modules.cp', _auth=MagicMock(**{'return_value.gen_token.return_value': 'token'}), __opts__={'id': 'abc', 'file_buffer_size': 10}), \ - patch('salt.utils.fopen', mock_open(read_data='content')), \ + patch('salt.utils.files.fopen', mock_open(read_data='content')), \ patch('salt.transport.Channel.factory', MagicMock()): response = cp.push('/saltines/test.file') self.assertEqual(response, True) - self.assertEqual(salt.utils.fopen().read.call_count, 2) # pylint: disable=resource-leakage + self.assertEqual(salt.utils.files.fopen().read.call_count, 2) # pylint: disable=resource-leakage salt.transport.Channel.factory({}).send.assert_called_once_with( dict( - loc=salt.utils.fopen().tell(), # pylint: disable=resource-leakage + loc=salt.utils.files.fopen().tell(), # pylint: disable=resource-leakage cmd='_file_recv', tok='token', path=['saltines', 'test.file'], diff --git a/tests/unit/modules/test_data.py b/tests/unit/modules/test_data.py index e6d14ca2a36..a5fd028550c 100644 --- a/tests/unit/modules/test_data.py +++ b/tests/unit/modules/test_data.py @@ -48,7 +48,7 @@ class DataTestCase(TestCase, LoaderModuleMockMixin): mocked_fopen = MagicMock(return_value=True) mocked_fopen.__enter__ = MagicMock(return_value=mocked_fopen) mocked_fopen.__exit__ = MagicMock() - with patch('salt.utils.fopen', MagicMock(return_value=mocked_fopen)): + with patch('salt.utils.files.fopen', MagicMock(return_value=mocked_fopen)): with patch('salt.payload.Serial.loads', MagicMock(return_value=True)): with patch.dict(data.__opts__, {'cachedir': '/'}): self.assertTrue(data.load()) @@ -60,7 +60,7 @@ class DataTestCase(TestCase, LoaderModuleMockMixin): Test if it replace the entire datastore with a passed data structure ''' with patch.dict(data.__opts__, {'cachedir': '/'}): - with patch('salt.utils.fopen', mock_open()): + with patch('salt.utils.files.fopen', mock_open()): self.assertTrue(data.dump('{"eggs": "spam"}')) def test_dump_isinstance(self): @@ -76,7 +76,7 @@ class DataTestCase(TestCase, LoaderModuleMockMixin): ''' with patch.dict(data.__opts__, {'cachedir': '/'}): mock = MagicMock(side_effect=IOError('')) - with patch('salt.utils.fopen', mock): + with patch('salt.utils.files.fopen', mock): self.assertFalse(data.dump('{"eggs": "spam"}')) # 'update' function tests: 1 diff --git a/tests/unit/modules/test_ddns.py b/tests/unit/modules/test_ddns.py index 55d3dbca95e..16d7c9ef4b9 100644 --- a/tests/unit/modules/test_ddns.py +++ b/tests/unit/modules/test_ddns.py @@ -122,7 +122,7 @@ class DDNSTestCase(TestCase, LoaderModuleMockMixin): return MockAnswer with patch.object(dns.query, 'udp', mock_udp_query()): - with patch('salt.utils.fopen', mock_open(read_data=file_data), create=True): + with patch('salt.utils.files.fopen', mock_open(read_data=file_data), create=True): with patch.object(dns.tsigkeyring, 'from_text', return_value=True): with patch.object(ddns, '_get_keyring', return_value=None): with patch.object(ddns, '_config', return_value=None): diff --git a/tests/unit/modules/test_deb_apache.py b/tests/unit/modules/test_deb_apache.py index e646b39a6c7..f57341acda7 100644 --- a/tests/unit/modules/test_deb_apache.py +++ b/tests/unit/modules/test_deb_apache.py @@ -265,7 +265,7 @@ class DebApacheTestCase(TestCase, LoaderModuleMockMixin): ''' Test if it runs a2enconf for the given conf. ''' - with patch('salt.utils.which', MagicMock(return_value='a2enconf')): + with patch('salt.utils.path.which', MagicMock(return_value='a2enconf')): mock = MagicMock(return_value=1) with patch.dict(deb_apache.__salt__, {'cmd.retcode': mock}): self.assertEqual(deb_apache.a2enconf('security'), @@ -277,7 +277,7 @@ class DebApacheTestCase(TestCase, LoaderModuleMockMixin): ''' Test if it runs a2enconf for the given conf. ''' - with patch('salt.utils.which', MagicMock(return_value='a2enconf')): + with patch('salt.utils.path.which', MagicMock(return_value='a2enconf')): mock = MagicMock(return_value=0) with patch.dict(deb_apache.__salt__, {'cmd.retcode': mock}): self.assertEqual(deb_apache.a2enconf('security'), @@ -289,7 +289,7 @@ class DebApacheTestCase(TestCase, LoaderModuleMockMixin): ''' Test if it runs a2enconf for the given conf. ''' - with patch('salt.utils.which', MagicMock(return_value='a2enconf')): + with patch('salt.utils.path.which', MagicMock(return_value='a2enconf')): mock = MagicMock(return_value=2) with patch.dict(deb_apache.__salt__, {'cmd.retcode': mock}): self.assertEqual(deb_apache.a2enconf('security'), @@ -301,7 +301,7 @@ class DebApacheTestCase(TestCase, LoaderModuleMockMixin): ''' Test if it runs a2enconf for the given conf. ''' - with patch('salt.utils.which', MagicMock(return_value='a2enconf')): + with patch('salt.utils.path.which', MagicMock(return_value='a2enconf')): mock = MagicMock(side_effect=Exception('error')) with patch.dict(deb_apache.__salt__, {'cmd.retcode': mock}): self.assertEqual(str(deb_apache.a2enconf('security')), @@ -313,7 +313,7 @@ class DebApacheTestCase(TestCase, LoaderModuleMockMixin): ''' Test if it runs a2disconf for the given conf. ''' - with patch('salt.utils.which', MagicMock(return_value='a2disconf')): + with patch('salt.utils.path.which', MagicMock(return_value='a2disconf')): mock = MagicMock(return_value=256) with patch.dict(deb_apache.__salt__, {'cmd.retcode': mock}): self.assertEqual(deb_apache.a2disconf('security'), @@ -325,7 +325,7 @@ class DebApacheTestCase(TestCase, LoaderModuleMockMixin): ''' Test if it runs a2disconf for the given conf. ''' - with patch('salt.utils.which', MagicMock(return_value='a2disconf')): + with patch('salt.utils.path.which', MagicMock(return_value='a2disconf')): mock = MagicMock(return_value=0) with patch.dict(deb_apache.__salt__, {'cmd.retcode': mock}): self.assertEqual(deb_apache.a2disconf('security'), @@ -337,7 +337,7 @@ class DebApacheTestCase(TestCase, LoaderModuleMockMixin): ''' Test if it runs a2disconf for the given conf. ''' - with patch('salt.utils.which', MagicMock(return_value='a2disconf')): + with patch('salt.utils.path.which', MagicMock(return_value='a2disconf')): mock = MagicMock(return_value=2) with patch.dict(deb_apache.__salt__, {'cmd.retcode': mock}): self.assertEqual(deb_apache.a2disconf('security'), @@ -349,7 +349,7 @@ class DebApacheTestCase(TestCase, LoaderModuleMockMixin): ''' Test if it runs a2disconf for the given conf. ''' - with patch('salt.utils.which', MagicMock(return_value='a2disconf')): + with patch('salt.utils.path.which', MagicMock(return_value='a2disconf')): mock = MagicMock(side_effect=Exception('error')) with patch.dict(deb_apache.__salt__, {'cmd.retcode': mock}): self.assertEqual(str(deb_apache.a2disconf('security')), diff --git a/tests/unit/modules/test_deb_postgres.py b/tests/unit/modules/test_deb_postgres.py index f4235972f24..de16a83e005 100644 --- a/tests/unit/modules/test_deb_postgres.py +++ b/tests/unit/modules/test_deb_postgres.py @@ -9,7 +9,7 @@ from tests.support.unit import skipIf, TestCase from tests.support.mock import NO_MOCK, NO_MOCK_REASON, Mock, patch # Import salt libs -import salt.ext.six as six +from salt.ext import six import salt.modules.deb_postgres as deb_postgres LSCLUSTER = '''\ @@ -26,7 +26,7 @@ class PostgresClusterTestCase(TestCase, LoaderModuleMockMixin): def setup_loader_modules(self): self.cmd_run_all_mock = Mock(return_value={'stdout': LSCLUSTER}) self.addCleanup(delattr, self, 'cmd_run_all_mock') - patcher = patch('salt.utils.which', Mock(return_value='/usr/bin/pg_createcluster')) + patcher = patch('salt.utils.path.which', Mock(return_value='/usr/bin/pg_createcluster')) patcher.start() self.addCleanup(patcher.stop) return { @@ -71,7 +71,7 @@ class PostgresLsClusterTestCase(TestCase, LoaderModuleMockMixin): def setup_loader_modules(self): self.cmd_run_all_mock = Mock(return_value={'stdout': LSCLUSTER}) self.addCleanup(delattr, self, 'cmd_run_all_mock') - patcher = patch('salt.utils.which', Mock(return_value='/usr/bin/pg_lsclusters')) + patcher = patch('salt.utils.path.which', Mock(return_value='/usr/bin/pg_lsclusters')) patcher.start() self.addCleanup(patcher.stop) return { @@ -127,7 +127,7 @@ class PostgresDeleteClusterTestCase(TestCase, LoaderModuleMockMixin): def setup_loader_modules(self): self.cmd_run_all_mock = Mock(return_value={'stdout': LSCLUSTER}) self.addCleanup(delattr, self, 'cmd_run_all_mock') - patcher = patch('salt.utils.which', Mock(return_value='/usr/bin/pg_dropcluster')) + patcher = patch('salt.utils.path.which', Mock(return_value='/usr/bin/pg_dropcluster')) patcher.start() self.addCleanup(patcher.stop) return { diff --git a/tests/unit/modules/test_disk.py b/tests/unit/modules/test_disk.py index 3b23220a840..ef08b47fb73 100644 --- a/tests/unit/modules/test_disk.py +++ b/tests/unit/modules/test_disk.py @@ -13,7 +13,7 @@ from tests.support.mock import MagicMock, patch # Import Salt libs import salt.modules.disk as disk -import salt.utils +import salt.utils.path STUB_DISK_USAGE = { '/': {'filesystem': None, '1K-blocks': 10000, 'used': 10000, 'available': 10000, 'capacity': 10000}, @@ -124,15 +124,25 @@ class DiskTestCase(TestCase, LoaderModuleMockMixin): with patch.dict(disk.__salt__, {'cmd.run': mock}): mock_dump = MagicMock(return_value={'retcode': 0, 'stdout': ''}) with patch('salt.modules.disk.dump', mock_dump): - kwargs = {'read-ahead': 512, 'filesystem-read-ahead': 512} + kwargs = {'read-ahead': 512, 'filesystem-read-ahead': 1024} disk.tune('/dev/sda', **kwargs) - mock.assert_called_once_with( - 'blockdev --setra 512 --setfra 512 /dev/sda', - python_shell=False - ) - @skipIf(not salt.utils.which('sync'), 'sync not found') - @skipIf(not salt.utils.which('mkfs'), 'mkfs not found') + self.assert_called_once(mock) + + args, kwargs = mock.call_args + + # Assert called once with either 'blockdev --setra 512 --setfra 512 /dev/sda' or + # 'blockdev --setfra 512 --setra 512 /dev/sda' and python_shell=False kwarg. + self.assertEqual(len(args), 1) + self.assertTrue(args[0].startswith('blockdev ')) + self.assertTrue(args[0].endswith(' /dev/sda')) + self.assertIn(' --setra 512 ', args[0]) + self.assertIn(' --setfra 1024 ', args[0]) + self.assertEqual(len(args[0].split()), 6) + self.assertEqual(kwargs, {'python_shell': False}) + + @skipIf(not salt.utils.path.which('sync'), 'sync not found') + @skipIf(not salt.utils.path.which('mkfs'), 'mkfs not found') def test_format(self): ''' unit tests for disk.format @@ -153,7 +163,7 @@ class DiskTestCase(TestCase, LoaderModuleMockMixin): with patch.dict(disk.__salt__, {'cmd.run': mock}): self.assertEqual(disk.fstype(device), fs_type) - @skipIf(not salt.utils.which('resize2fs'), 'resize2fs not found') + @skipIf(not salt.utils.path.which('resize2fs'), 'resize2fs not found') def test_resize2fs(self): ''' unit tests for disk.resize2fs diff --git a/tests/unit/modules/test_djangomod.py b/tests/unit/modules/test_djangomod.py index 5f6a49c5426..6142b66e164 100644 --- a/tests/unit/modules/test_djangomod.py +++ b/tests/unit/modules/test_djangomod.py @@ -25,7 +25,7 @@ class DjangomodTestCase(TestCase, LoaderModuleMockMixin): Test cases for salt.modules.djangomod ''' def setup_loader_modules(self): - patcher = patch('salt.utils.which', lambda exe: exe) + patcher = patch('salt.utils.path.which', lambda exe: exe) patcher.start() self.addCleanup(patcher.stop) return {djangomod: {'_get_django_admin': MagicMock(return_value=True)}} @@ -92,7 +92,7 @@ class DjangomodCliCommandTestCase(TestCase, LoaderModuleMockMixin): Test cases for salt.modules.djangomod ''' def setup_loader_modules(self): - patcher = patch('salt.utils.which', lambda exe: exe) + patcher = patch('salt.utils.path.which', lambda exe: exe) patcher.start() self.addCleanup(patcher.stop) return {djangomod: {}} @@ -189,7 +189,7 @@ class DjangomodCliCommandTestCase(TestCase, LoaderModuleMockMixin): djangomod.createsuperuser( 'settings.py', 'testuser', 'user@example.com' ) - mock.assert_called_once() + self.assertEqual(mock.call_count, 1) args, kwargs = mock.call_args # cmdline arguments are extracted from a kwargs dict so order isn't guaranteed. self.assertEqual(len(args), 1) diff --git a/tests/unit/modules/test_dnsmasq.py b/tests/unit/modules/test_dnsmasq.py index 995c4671ee6..460932f668f 100644 --- a/tests/unit/modules/test_dnsmasq.py +++ b/tests/unit/modules/test_dnsmasq.py @@ -98,7 +98,7 @@ class DnsmasqTestCase(TestCase, LoaderModuleMockMixin): ''' with patch('os.path.isfile', MagicMock(return_value=True)): text_file_data = '\n'.join(["line here", "second line", "A=B", "#"]) - with patch('salt.utils.fopen', + with patch('salt.utils.files.fopen', mock_open(read_data=text_file_data), create=True) as m: m.return_value.__iter__.return_value = text_file_data.splitlines() diff --git a/tests/unit/modules/test_dnsutil.py b/tests/unit/modules/test_dnsutil.py index 7a061297d88..b2b2c7550dd 100644 --- a/tests/unit/modules/test_dnsutil.py +++ b/tests/unit/modules/test_dnsutil.py @@ -67,14 +67,14 @@ if NO_MOCK is False: @skipIf(NO_MOCK, NO_MOCK_REASON) class DNSUtilTestCase(TestCase): def test_parse_hosts(self): - with patch('salt.utils.fopen', mock_open(read_data=mock_hosts_file)): + with patch('salt.utils.files.fopen', mock_open(read_data=mock_hosts_file)): self.assertEqual(dnsutil.parse_hosts(), {'::1': ['localhost'], '255.255.255.255': ['broadcasthost'], '127.0.0.1': ['localhost'], 'fe80::1%lo0': ['localhost']}) def test_hosts_append(self): - with patch('salt.utils.fopen', mock_open(read_data=mock_hosts_file)) as m_open, \ + with patch('salt.utils.files.fopen', mock_open(read_data=mock_hosts_file)) as m_open, \ patch('salt.modules.dnsutil.parse_hosts', MagicMock(return_value=mock_hosts_file_rtn)): dnsutil.hosts_append('/etc/hosts', '127.0.0.1', 'ad1.yuk.co,ad2.yuk.co') helper_open = m_open() @@ -83,7 +83,7 @@ class DNSUtilTestCase(TestCase): def test_hosts_remove(self): to_remove = 'ad1.yuk.co' new_mock_file = mock_hosts_file + '\n127.0.0.1 ' + to_remove + '\n' - with patch('salt.utils.fopen', mock_open(read_data=new_mock_file)) as m_open: + with patch('salt.utils.files.fopen', mock_open(read_data=new_mock_file)) as m_open: dnsutil.hosts_remove('/etc/hosts', to_remove) helper_open = m_open() calls_list = helper_open.method_calls @@ -91,7 +91,7 @@ class DNSUtilTestCase(TestCase): @skipIf(True, 'Waiting on bug report fixes') def test_parse_zone(self): - with patch('salt.utils.fopen', mock_open(read_data=mock_soa_zone)): + with patch('salt.utils.files.fopen', mock_open(read_data=mock_soa_zone)): log.debug(mock_soa_zone) log.debug(dnsutil.parse_zone('/var/lib/named/example.com.zone')) diff --git a/tests/unit/modules/test_dockermod.py b/tests/unit/modules/test_dockermod.py index 2746c98ddb7..e43d7011e4c 100644 --- a/tests/unit/modules/test_dockermod.py +++ b/tests/unit/modules/test_dockermod.py @@ -18,6 +18,8 @@ from tests.support.mock import ( ) # Import Salt Libs +import salt.config +import salt.loader from salt.ext.six.moves import range from salt.exceptions import CommandExecutionError import salt.modules.dockermod as docker_mod @@ -39,7 +41,12 @@ class DockerTestCase(TestCase, LoaderModuleMockMixin): Validate docker module ''' def setup_loader_modules(self): - return {docker_mod: {'__context__': {'docker.docker_version': ''}}} + utils = salt.loader.utils( + salt.config.DEFAULT_MINION_OPTS, + whitelist=['state'] + ) + return {docker_mod: {'__context__': {'docker.docker_version': ''}, + '__utils__': utils}} try: docker_version = docker_mod.docker.version_info @@ -136,10 +143,11 @@ class DockerTestCase(TestCase, LoaderModuleMockMixin): with patch.dict(docker_mod.__salt__, {'mine.send': mine_send, 'container_resource.run': MagicMock(), - 'cp.cache_file': MagicMock(return_value=False), - 'docker.get_client_args': client_args_mock}): - with patch.object(docker_mod, '_get_client', client): - command('container', *args) + 'cp.cache_file': MagicMock(return_value=False)}): + with patch.dict(docker_mod.__utils__, + {'docker.get_client_args': client_args_mock}): + with patch.object(docker_mod, '_get_client', client): + command('container', *args) mine_send.assert_called_with('docker.ps', verbose=True, all=True, host=True) @@ -184,10 +192,23 @@ class DockerTestCase(TestCase, LoaderModuleMockMixin): with patch.object(docker_mod, '_get_client', get_client_mock): docker_mod.create_network('foo', driver='bridge', - driver_opts={}) + driver_opts={}, + gateway='192.168.0.1', + ip_range='192.168.0.128/25', + subnet='192.168.0.0/24' + ) client.create_network.assert_called_once_with('foo', driver='bridge', options={}, + ipam={ + 'Config': [{ + 'Gateway': '192.168.0.1', + 'IPRange': '192.168.0.128/25', + 'Subnet': '192.168.0.0/24' + }], + 'Driver': 'default', + 'Options': {} + }, check_duplicate=True) @skipIf(docker_version < (1, 5, 0), @@ -690,9 +711,9 @@ class DockerTestCase(TestCase, LoaderModuleMockMixin): self.assertEqual({"retcode": 0, "comment": "container cmd"}, ret) def test_images_with_empty_tags(self): - """ + ''' docker 1.12 reports also images without tags with `null`. - """ + ''' client = Mock() client.api_version = '1.24' client.images = Mock( @@ -708,3 +729,51 @@ class DockerTestCase(TestCase, LoaderModuleMockMixin): result = docker_mod.images() self.assertEqual(result, {'sha256:abcdefg': {'RepoTags': ['image:latest']}}) + + def test_compare_container_image_id_resolution(self): + ''' + Test comparing two containers when one's inspect output is an ID and + not formatted in image:tag notation. + ''' + def _inspect_container_effect(id_): + return { + 'container1': {'Config': {'Image': 'realimage:latest'}, + 'HostConfig': {}}, + 'container2': {'Config': {'Image': 'image_id'}, + 'HostConfig': {}}, + }[id_] + + def _inspect_image_effect(id_): + return { + 'realimage:latest': {'Id': 'image_id'}, + 'image_id': {'Id': 'image_id'}, + }[id_] + + inspect_container_mock = MagicMock(side_effect=_inspect_container_effect) + inspect_image_mock = MagicMock(side_effect=_inspect_image_effect) + + with patch.object(docker_mod, 'inspect_container', inspect_container_mock): + with patch.object(docker_mod, 'inspect_image', inspect_image_mock): + ret = docker_mod.compare_container('container1', 'container2') + self.assertEqual(ret, {}) + + def test_resolve_tag(self): + ''' + Test the resolve_tag function + ''' + with_prefix = 'docker.io/foo:latest' + no_prefix = 'bar:latest' + with patch.object(docker_mod, + 'list_tags', + MagicMock(return_value=[with_prefix])): + self.assertEqual(docker_mod.resolve_tag('foo'), with_prefix) + self.assertEqual(docker_mod.resolve_tag('foo:latest'), with_prefix) + self.assertEqual(docker_mod.resolve_tag(with_prefix), with_prefix) + self.assertEqual(docker_mod.resolve_tag('foo:bar'), False) + + with patch.object(docker_mod, + 'list_tags', + MagicMock(return_value=[no_prefix])): + self.assertEqual(docker_mod.resolve_tag('bar'), no_prefix) + self.assertEqual(docker_mod.resolve_tag(no_prefix), no_prefix) + self.assertEqual(docker_mod.resolve_tag('bar:baz'), False) diff --git a/tests/unit/modules/test_environ.py b/tests/unit/modules/test_environ.py index 94429420413..e1724e35270 100644 --- a/tests/unit/modules/test_environ.py +++ b/tests/unit/modules/test_environ.py @@ -52,7 +52,7 @@ class EnvironTestCase(TestCase, LoaderModuleMockMixin): with patch.dict(os.environ, {}), \ patch.dict(environ.__salt__, {'reg.set_value': MagicMock(), 'reg.delete_value': MagicMock()}), \ - patch('salt.utils.is_windows', MagicMock(return_value=True)): + patch('salt.utils.platform.is_windows', MagicMock(return_value=True)): environ.setval('key', 'Test', permanent=True) environ.__salt__['reg.set_value'].assert_called_with('HKCU', 'Environment', 'key', 'Test') diff --git a/tests/unit/modules/test_esxcluster.py b/tests/unit/modules/test_esxcluster.py new file mode 100644 index 00000000000..5a32980d165 --- /dev/null +++ b/tests/unit/modules/test_esxcluster.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- +''' + :codeauthor: :email:`Alexandru Bleotu ` + + Tests for functions in salt.modules.esxcluster +''' + +# Import Python Libs +from __future__ import absolute_import + +# Import Salt Libs +import salt.modules.esxcluster as esxcluster + +# Import Salt Testing Libs +from tests.support.mixins import LoaderModuleMockMixin +from tests.support.unit import TestCase, skipIf +from tests.support.mock import ( + MagicMock, + patch, + NO_MOCK, + NO_MOCK_REASON +) + + +@skipIf(NO_MOCK, NO_MOCK_REASON) +class GetDetailsTestCase(TestCase, LoaderModuleMockMixin): + '''Tests for salt.modules.esxcluster.get_details''' + def setup_loader_modules(self): + return {esxcluster: {'__virtual__': + MagicMock(return_value='esxcluster'), + '__proxy__': {}}} + + def test_get_details(self): + mock_get_details = MagicMock() + with patch.dict(esxcluster.__proxy__, + {'esxcluster.get_details': mock_get_details}): + esxcluster.get_details() + mock_get_details.assert_called_once_with() diff --git a/tests/unit/modules/test_esxdatacenter.py b/tests/unit/modules/test_esxdatacenter.py new file mode 100644 index 00000000000..a6e0c12a494 --- /dev/null +++ b/tests/unit/modules/test_esxdatacenter.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- +''' + :codeauthor: :email:`Alexandru Bleotu ` + + Tests for functions in salt.modules.esxdatacenter +''' + +# Import Python Libs +from __future__ import absolute_import + +# Import Salt Libs +import salt.modules.esxdatacenter as esxdatacenter + +# Import Salt Testing Libs +from tests.support.mixins import LoaderModuleMockMixin +from tests.support.unit import TestCase, skipIf +from tests.support.mock import ( + MagicMock, + patch, + NO_MOCK, + NO_MOCK_REASON +) + + +@skipIf(NO_MOCK, NO_MOCK_REASON) +class GetDetailsTestCase(TestCase, LoaderModuleMockMixin): + '''Tests for salt.modules.esxdatacenter.get_details''' + def setup_loader_modules(self): + return {esxdatacenter: {'__virtual__': + MagicMock(return_value='esxdatacenter'), + '__proxy__': {}}} + + def test_get_details(self): + mock_get_details = MagicMock() + with patch.dict(esxdatacenter.__proxy__, + {'esxdatacenter.get_details': mock_get_details}): + esxdatacenter.get_details() + mock_get_details.assert_called_once_with() diff --git a/tests/unit/modules/test_file.py b/tests/unit/modules/test_file.py index 1c7dbe13eb4..e4d74f12662 100644 --- a/tests/unit/modules/test_file.py +++ b/tests/unit/modules/test_file.py @@ -14,7 +14,9 @@ from tests.support.unit import TestCase from tests.support.mock import MagicMock, patch # Import Salt libs -import salt.utils +import salt.config +import salt.loader +import salt.utils.files import salt.modules.file as filemod import salt.modules.config as configmod import salt.modules.cmdmod as cmdmod @@ -45,7 +47,8 @@ class FileReplaceTestCase(TestCase, LoaderModuleMockMixin): 'cachedir': 'tmp', 'grains': {}, }, - '__grains__': {'kernel': 'Linux'} + '__grains__': {'kernel': 'Linux'}, + '__utils__': {'files.is_text_file': MagicMock(return_value=True)}, } } @@ -77,7 +80,7 @@ class FileReplaceTestCase(TestCase, LoaderModuleMockMixin): def test_replace(self): filemod.replace(self.tfile.name, r'Etiam', 'Salticus', backup=False) - with salt.utils.fopen(self.tfile.name, 'r') as fp: + with salt.utils.files.fopen(self.tfile.name, 'r') as fp: self.assertIn('Salticus', fp.read()) def test_replace_append_if_not_found(self): @@ -96,19 +99,19 @@ class FileReplaceTestCase(TestCase, LoaderModuleMockMixin): tfile.write(base + '\n') tfile.flush() filemod.replace(tfile.name, **args) - with salt.utils.fopen(tfile.name) as tfile2: + with salt.utils.files.fopen(tfile.name) as tfile2: self.assertEqual(tfile2.read(), expected) # File not ending with a newline, no match with tempfile.NamedTemporaryFile('w+') as tfile: tfile.write(base) tfile.flush() filemod.replace(tfile.name, **args) - with salt.utils.fopen(tfile.name) as tfile2: + with salt.utils.files.fopen(tfile.name) as tfile2: self.assertEqual(tfile2.read(), expected) # A newline should not be added in empty files with tempfile.NamedTemporaryFile('w+') as tfile: filemod.replace(tfile.name, **args) - with salt.utils.fopen(tfile.name) as tfile2: + with salt.utils.files.fopen(tfile.name) as tfile2: self.assertEqual(tfile2.read(), args['repl'] + '\n') # Using not_found_content, rather than repl with tempfile.NamedTemporaryFile('w+') as tfile: @@ -117,7 +120,7 @@ class FileReplaceTestCase(TestCase, LoaderModuleMockMixin): tfile.write(base) tfile.flush() filemod.replace(tfile.name, **args) - with salt.utils.fopen(tfile.name) as tfile2: + with salt.utils.files.fopen(tfile.name) as tfile2: self.assertEqual(tfile2.read(), expected) # not appending if matches with tempfile.NamedTemporaryFile('w+') as tfile: @@ -126,7 +129,7 @@ class FileReplaceTestCase(TestCase, LoaderModuleMockMixin): tfile.write(base) tfile.flush() filemod.replace(tfile.name, **args) - with salt.utils.fopen(tfile.name) as tfile2: + with salt.utils.files.fopen(tfile.name) as tfile2: self.assertEqual(tfile2.read(), expected) def test_backup(self): @@ -203,7 +206,8 @@ class FileBlockReplaceTestCase(TestCase, LoaderModuleMockMixin): 'cachedir': 'tmp', 'grains': {}, }, - '__grains__': {'kernel': 'Linux'} + '__grains__': {'kernel': 'Linux'}, + '__utils__': {'files.is_text_file': MagicMock(return_value=True)}, } } @@ -257,7 +261,7 @@ class FileBlockReplaceTestCase(TestCase, LoaderModuleMockMixin): new_multiline_content, backup=False) - with salt.utils.fopen(self.tfile.name, 'r') as fp: + with salt.utils.files.fopen(self.tfile.name, 'r') as fp: filecontent = fp.read() self.assertIn('#-- START BLOCK 1' + "\n" + new_multiline_content @@ -279,7 +283,7 @@ class FileBlockReplaceTestCase(TestCase, LoaderModuleMockMixin): append_if_not_found=False, backup=False ) - with salt.utils.fopen(self.tfile.name, 'r') as fp: + with salt.utils.files.fopen(self.tfile.name, 'r') as fp: self.assertNotIn('#-- START BLOCK 2' + "\n" + new_content + '#-- END BLOCK 2', fp.read()) @@ -291,7 +295,7 @@ class FileBlockReplaceTestCase(TestCase, LoaderModuleMockMixin): backup=False, append_if_not_found=True) - with salt.utils.fopen(self.tfile.name, 'r') as fp: + with salt.utils.files.fopen(self.tfile.name, 'r') as fp: self.assertIn('#-- START BLOCK 2' + "\n" + new_content + '#-- END BLOCK 2', fp.read()) @@ -315,19 +319,19 @@ class FileBlockReplaceTestCase(TestCase, LoaderModuleMockMixin): tfile.write(base + '\n') tfile.flush() filemod.blockreplace(tfile.name, **args) - with salt.utils.fopen(tfile.name) as tfile2: + with salt.utils.files.fopen(tfile.name) as tfile2: self.assertEqual(tfile2.read(), expected) # File not ending with a newline with tempfile.NamedTemporaryFile(mode='w+') as tfile: tfile.write(base) tfile.flush() filemod.blockreplace(tfile.name, **args) - with salt.utils.fopen(tfile.name) as tfile2: + with salt.utils.files.fopen(tfile.name) as tfile2: self.assertEqual(tfile2.read(), expected) # A newline should not be added in empty files with tempfile.NamedTemporaryFile(mode='w+') as tfile: filemod.blockreplace(tfile.name, **args) - with salt.utils.fopen(tfile.name) as tfile2: + with salt.utils.files.fopen(tfile.name) as tfile2: self.assertEqual(tfile2.read(), block) def test_replace_prepend(self): @@ -343,7 +347,7 @@ class FileBlockReplaceTestCase(TestCase, LoaderModuleMockMixin): prepend_if_not_found=False, backup=False ) - with salt.utils.fopen(self.tfile.name, 'r') as fp: + with salt.utils.files.fopen(self.tfile.name, 'r') as fp: self.assertNotIn( '#-- START BLOCK 2' + "\n" + new_content + '#-- END BLOCK 2', @@ -355,7 +359,7 @@ class FileBlockReplaceTestCase(TestCase, LoaderModuleMockMixin): backup=False, prepend_if_not_found=True) - with salt.utils.fopen(self.tfile.name, 'r') as fp: + with salt.utils.files.fopen(self.tfile.name, 'r') as fp: self.assertTrue( fp.read().startswith( '#-- START BLOCK 2' @@ -369,7 +373,7 @@ class FileBlockReplaceTestCase(TestCase, LoaderModuleMockMixin): 'new content 1', backup=False) - with salt.utils.fopen(self.tfile.name, 'r') as fp: + with salt.utils.files.fopen(self.tfile.name, 'r') as fp: filecontent = fp.read() self.assertIn('new content 1', filecontent) self.assertNotIn('to be removed', filecontent) @@ -489,7 +493,7 @@ class FileModuleTestCase(TestCase, LoaderModuleMockMixin): filemod.sed(path, before, after, limit=limit) - with salt.utils.fopen(path, 'r') as newfile: + with salt.utils.files.fopen(path, 'r') as newfile: self.assertEqual( SED_CONTENT.replace(before, ''), newfile.read() @@ -505,19 +509,19 @@ class FileModuleTestCase(TestCase, LoaderModuleMockMixin): tfile.write('foo\n') tfile.flush() filemod.append(tfile.name, 'bar') - with salt.utils.fopen(tfile.name) as tfile2: + with salt.utils.files.fopen(tfile.name) as tfile2: self.assertEqual(tfile2.read(), 'foo\nbar\n') # File not ending with a newline with tempfile.NamedTemporaryFile(mode='w+') as tfile: tfile.write('foo') tfile.flush() filemod.append(tfile.name, 'bar') - with salt.utils.fopen(tfile.name) as tfile2: + with salt.utils.files.fopen(tfile.name) as tfile2: self.assertEqual(tfile2.read(), 'foo\nbar\n') # A newline should not be added in empty files with tempfile.NamedTemporaryFile(mode='w+') as tfile: filemod.append(tfile.name, 'bar') - with salt.utils.fopen(tfile.name) as tfile2: + with salt.utils.files.fopen(tfile.name) as tfile2: self.assertEqual(tfile2.read(), 'bar\n') def test_extract_hash(self): @@ -641,7 +645,7 @@ class FileModuleTestCase(TestCase, LoaderModuleMockMixin): def test_patch(self): with patch('os.path.isdir', return_value=False) as mock_isdir, \ - patch('salt.utils.which', return_value='/bin/patch') as mock_which: + patch('salt.utils.path.which', return_value='/bin/patch') as mock_which: cmd_mock = MagicMock(return_value='test_retval') with patch.dict(filemod.__salt__, {'cmd.run_all': cmd_mock}): ret = filemod.patch('/path/to/file', '/path/to/patch') @@ -652,7 +656,7 @@ class FileModuleTestCase(TestCase, LoaderModuleMockMixin): def test_patch_dry_run(self): with patch('os.path.isdir', return_value=False) as mock_isdir, \ - patch('salt.utils.which', return_value='/bin/patch') as mock_which: + patch('salt.utils.path.which', return_value='/bin/patch') as mock_which: cmd_mock = MagicMock(return_value='test_retval') with patch.dict(filemod.__salt__, {'cmd.run_all': cmd_mock}): ret = filemod.patch('/path/to/file', '/path/to/patch', dry_run=True) @@ -663,7 +667,7 @@ class FileModuleTestCase(TestCase, LoaderModuleMockMixin): def test_patch_dir(self): with patch('os.path.isdir', return_value=True) as mock_isdir, \ - patch('salt.utils.which', return_value='/bin/patch') as mock_which: + patch('salt.utils.path.which', return_value='/bin/patch') as mock_which: cmd_mock = MagicMock(return_value='test_retval') with patch.dict(filemod.__salt__, {'cmd.run_all': cmd_mock}): ret = filemod.patch('/path/to/dir', '/path/to/patch') @@ -769,7 +773,7 @@ class FileBasicsTestCase(TestCase, LoaderModuleMockMixin): self.addCleanup(os.remove, self.tfile.name) self.addCleanup(delattr, self, 'tfile') self.myfile = os.path.join(TMP, 'myfile') - with salt.utils.fopen(self.myfile, 'w+') as fp: + with salt.utils.files.fopen(self.myfile, 'w+') as fp: fp.write('Hello\n') self.addCleanup(os.remove, self.myfile) self.addCleanup(delattr, self, 'myfile') diff --git a/tests/unit/modules/test_gem.py b/tests/unit/modules/test_gem.py index 14e38da893c..23221ba6461 100644 --- a/tests/unit/modules/test_gem.py +++ b/tests/unit/modules/test_gem.py @@ -65,7 +65,8 @@ class TestGemModule(TestCase, LoaderModuleMockMixin): with patch.dict(gem.__salt__, {'rvm.is_installed': MagicMock(return_value=False), 'rbenv.is_installed': MagicMock(return_value=True), - 'rbenv.do': mock}): + 'rbenv.do': mock}),\ + patch('salt.utils.platform.is_windows', return_value=False): gem._gem(['install', 'rails']) mock.assert_called_once_with( ['gem', 'install', 'rails'], diff --git a/tests/unit/modules/test_genesis.py b/tests/unit/modules/test_genesis.py index af135a04095..ae9f3e7b16b 100644 --- a/tests/unit/modules/test_genesis.py +++ b/tests/unit/modules/test_genesis.py @@ -46,12 +46,59 @@ class GenesisTestCase(TestCase, LoaderModuleMockMixin): with patch.dict(genesis.__salt__, {'disk.blkid': MagicMock(return_value={})}): self.assertEqual(genesis.bootstrap('rpm', 'root', 'dir'), None) - with patch.object(genesis, '_bootstrap_deb', return_value='A'): + common_parms = {'platform': 'deb', + 'root': 'root', + 'img_format': 'dir', + 'arch': 'amd64', + 'flavor': 'stable', + 'static_qemu': 'qemu'} + + param_sets = [ + + {'params': {}, + 'cmd': ['debootstrap', '--foreign', '--arch', 'amd64', + 'stable', 'root', 'http://ftp.debian.org/debian/'] + }, + + {'params': {'pkgs': 'vim'}, + 'cmd': ['debootstrap', '--foreign', '--arch', 'amd64', + '--include', 'vim', + 'stable', 'root', 'http://ftp.debian.org/debian/'] + }, + + {'params': {'pkgs': 'vim,emacs'}, + 'cmd': ['debootstrap', '--foreign', '--arch', 'amd64', + '--include', 'vim,emacs', + 'stable', 'root', 'http://ftp.debian.org/debian/'] + }, + + {'params': {'pkgs': ['vim', 'emacs']}, + 'cmd': ['debootstrap', '--foreign', '--arch', 'amd64', + '--include', 'vim,emacs', + 'stable', 'root', 'http://ftp.debian.org/debian/'] + }, + + {'params': {'pkgs': ['vim', 'emacs'], 'exclude_pkgs': ['vim', 'foo']}, + 'cmd': ['debootstrap', '--foreign', '--arch', 'amd64', + '--include', 'vim,emacs', '--exclude', 'vim,foo', + 'stable', 'root', 'http://ftp.debian.org/debian/'] + }, + + ] + + for param_set in param_sets: + with patch.dict(genesis.__salt__, {'mount.umount': MagicMock(), 'file.rmdir': MagicMock(), - 'file.directory_exists': MagicMock()}): - with patch.dict(genesis.__salt__, {'disk.blkid': MagicMock(return_value={})}): - self.assertEqual(genesis.bootstrap('deb', 'root', 'dir'), None) + 'file.directory_exists': MagicMock(), + 'cmd.run': MagicMock(), + 'disk.blkid': MagicMock(return_value={})}): + with patch('salt.modules.genesis.salt.utils.path.which', return_value=True): + with patch('salt.modules.genesis.salt.utils.validate.path.is_executable', + return_value=True): + param_set['params'].update(common_parms) + self.assertEqual(genesis.bootstrap(**param_set['params']), None) + genesis.__salt__['cmd.run'].assert_any_call(param_set['cmd'], python_shell=False) with patch.object(genesis, '_bootstrap_pacman', return_value='A') as pacman_patch: with patch.dict(genesis.__salt__, {'mount.umount': MagicMock(), @@ -65,7 +112,7 @@ class GenesisTestCase(TestCase, LoaderModuleMockMixin): ''' Test for Return which platforms are available ''' - with patch('salt.utils.which', MagicMock(return_value=False)): + with patch('salt.utils.path.which', MagicMock(return_value=False)): self.assertFalse(genesis.avail_platforms()['deb']) def test_pack(self): diff --git a/tests/unit/modules/test_grains.py b/tests/unit/modules/test_grains.py index 7eb4be077e5..e9ae6c5116a 100644 --- a/tests/unit/modules/test_grains.py +++ b/tests/unit/modules/test_grains.py @@ -19,7 +19,7 @@ from tests.support.mock import ( # Import Salt libs from salt.exceptions import SaltException import salt.modules.grains as grainsmod -from salt.utils import dictupdate +import salt.utils.dictupdate as dictupdate # Import 3rd-party libs from salt.utils.odict import OrderedDict diff --git a/tests/unit/modules/test_groupadd.py b/tests/unit/modules/test_groupadd.py index b836bd88050..a0646556ea6 100644 --- a/tests/unit/modules/test_groupadd.py +++ b/tests/unit/modules/test_groupadd.py @@ -5,7 +5,10 @@ # Import Python libs from __future__ import absolute_import -import grp +try: + import grp +except ImportError: + pass # Import Salt Testing Libs from tests.support.mixins import LoaderModuleMockMixin @@ -14,9 +17,11 @@ from tests.support.mock import MagicMock, patch, NO_MOCK, NO_MOCK_REASON # Import Salt Libs import salt.modules.groupadd as groupadd +import salt.utils.platform @skipIf(NO_MOCK, NO_MOCK_REASON) +@skipIf(salt.utils.platform.is_windows(), "Module not available on Windows") class GroupAddTestCase(TestCase, LoaderModuleMockMixin): ''' TestCase for salt.modules.groupadd @@ -113,16 +118,16 @@ class GroupAddTestCase(TestCase, LoaderModuleMockMixin): ''' os_version_list = [ {'grains': {'kernel': 'Linux', 'os_family': 'RedHat', 'osmajorrelease': '5'}, - 'cmd': ('gpasswd', '-a', 'root', 'test')}, + 'cmd': ['gpasswd', '-a', 'root', 'test']}, {'grains': {'kernel': 'Linux', 'os_family': 'Suse', 'osmajorrelease': '11'}, - 'cmd': ('usermod', '-A', 'test', 'root')}, + 'cmd': ['usermod', '-A', 'test', 'root']}, {'grains': {'kernel': 'Linux'}, - 'cmd': ('gpasswd', '--add', 'root', 'test')}, + 'cmd': ['gpasswd', '--add', 'root', 'test']}, {'grains': {'kernel': 'OTHERKERNEL'}, - 'cmd': ('usermod', '-G', 'test', 'root')}, + 'cmd': ['usermod', '-G', 'test', 'root']}, ] for os_version in os_version_list: @@ -140,16 +145,16 @@ class GroupAddTestCase(TestCase, LoaderModuleMockMixin): ''' os_version_list = [ {'grains': {'kernel': 'Linux', 'os_family': 'RedHat', 'osmajorrelease': '5'}, - 'cmd': ('gpasswd', '-d', 'root', 'test')}, + 'cmd': ['gpasswd', '-d', 'root', 'test']}, {'grains': {'kernel': 'Linux', 'os_family': 'Suse', 'osmajorrelease': '11'}, - 'cmd': ('usermod', '-R', 'test', 'root')}, + 'cmd': ['usermod', '-R', 'test', 'root']}, {'grains': {'kernel': 'Linux'}, - 'cmd': ('gpasswd', '--del', 'root', 'test')}, + 'cmd': ['gpasswd', '--del', 'root', 'test']}, {'grains': {'kernel': 'OpenBSD'}, - 'cmd': 'usermod -S foo root'}, + 'cmd': ['usermod', '-S', 'foo', 'root']}, ] for os_version in os_version_list: @@ -175,16 +180,16 @@ class GroupAddTestCase(TestCase, LoaderModuleMockMixin): ''' os_version_list = [ {'grains': {'kernel': 'Linux', 'os_family': 'RedHat', 'osmajorrelease': '5'}, - 'cmd': ('gpasswd', '-M', 'foo', 'test')}, + 'cmd': ['gpasswd', '-M', 'foo', 'test']}, {'grains': {'kernel': 'Linux', 'os_family': 'Suse', 'osmajorrelease': '11'}, - 'cmd': ('groupmod', '-A', 'foo', 'test')}, + 'cmd': ['groupmod', '-A', 'foo', 'test']}, {'grains': {'kernel': 'Linux'}, - 'cmd': ('gpasswd', '--members', 'foo', 'test')}, + 'cmd': ['gpasswd', '--members', 'foo', 'test']}, {'grains': {'kernel': 'OpenBSD'}, - 'cmd': 'usermod -G test foo'}, + 'cmd': ['usermod', '-G', 'test', 'foo']}, ] for os_version in os_version_list: diff --git a/tests/unit/modules/test_grub_legacy.py b/tests/unit/modules/test_grub_legacy.py index 0ff5c2c7e79..92dd3dc1d79 100644 --- a/tests/unit/modules/test_grub_legacy.py +++ b/tests/unit/modules/test_grub_legacy.py @@ -43,12 +43,12 @@ class GrublegacyTestCase(TestCase, LoaderModuleMockMixin): Test for Parse GRUB conf file ''' mock = MagicMock(side_effect=IOError('foo')) - with patch('salt.utils.fopen', mock): + with patch('salt.utils.files.fopen', mock): with patch.object(grub_legacy, '_detect_conf', return_value='A'): self.assertRaises(CommandExecutionError, grub_legacy.conf) file_data = '\n'.join(['#', 'A B C D,E,F G H']) - with patch('salt.utils.fopen', + with patch('salt.utils.files.fopen', mock_open(read_data=file_data), create=True) as f_mock: f_mock.return_value.__iter__.return_value = file_data.splitlines() with patch.object(grub_legacy, '_detect_conf', return_value='A'): diff --git a/tests/unit/modules/test_haproxyconn.py b/tests/unit/modules/test_haproxyconn.py index 183b5b713ea..fa7705b5b1d 100644 --- a/tests/unit/modules/test_haproxyconn.py +++ b/tests/unit/modules/test_haproxyconn.py @@ -161,7 +161,10 @@ class HaproxyConnTestCase(TestCase, LoaderModuleMockMixin): ''' Test listing all frontends ''' - self.assertItemsEqual(haproxyconn.list_frontends(), ['frontend-alpha', 'frontend-beta', 'frontend-gamma']) + self.assertEqual( + sorted(haproxyconn.list_frontends()), + sorted(['frontend-alpha', 'frontend-beta', 'frontend-gamma']) + ) # 'show_backends' function tests: 1 @@ -175,7 +178,10 @@ class HaproxyConnTestCase(TestCase, LoaderModuleMockMixin): ''' Test listing of all backends ''' - self.assertItemsEqual(haproxyconn.list_backends(), ['backend-alpha', 'backend-beta', 'backend-gamma']) + self.assertEqual( + sorted(haproxyconn.list_backends()), + sorted(['backend-alpha', 'backend-beta', 'backend-gamma']) + ) def test_get_backend(self): ''' diff --git a/tests/unit/modules/test_hosts.py b/tests/unit/modules/test_hosts.py index 4727e5e585d..56f01f56ab2 100644 --- a/tests/unit/modules/test_hosts.py +++ b/tests/unit/modules/test_hosts.py @@ -16,6 +16,7 @@ from tests.support.mock import ( ) # Import Salt Libs import salt.modules.hosts as hosts +import salt.utils from salt.ext.six.moves import StringIO @@ -92,8 +93,12 @@ class HostsTestCase(TestCase, LoaderModuleMockMixin): ''' Tests true if the alias is set ''' + hosts_file = '/etc/hosts' + if salt.utils.is_windows(): + hosts_file = r'C:\Windows\System32\Drivers\etc\hosts' + with patch('salt.modules.hosts.__get_hosts_filename', - MagicMock(return_value='/etc/hosts')), \ + MagicMock(return_value=hosts_file)), \ patch('os.path.isfile', MagicMock(return_value=False)), \ patch.dict(hosts.__salt__, {'config.option': MagicMock(return_value=None)}): @@ -106,7 +111,7 @@ class HostsTestCase(TestCase, LoaderModuleMockMixin): with patch('salt.modules.hosts.__get_hosts_filename', MagicMock(return_value='/etc/hosts')), \ patch('os.path.isfile', MagicMock(return_value=True)), \ - patch('salt.utils.fopen', mock_open()): + patch('salt.utils.files.fopen', mock_open()): mock_opt = MagicMock(return_value=None) with patch.dict(hosts.__salt__, {'config.option': mock_opt}): self.assertTrue(hosts.set_host('10.10.10.10', 'Salt1')) @@ -139,7 +144,16 @@ class HostsTestCase(TestCase, LoaderModuleMockMixin): self.close() def close(self): - data[0] = self.getvalue() + # Don't save unless there's something there. In Windows + # the class gets initialized the first time with mode = w + # which sets the initial value to ''. When the class closes + # it clears out data and causes the test to fail. + # I don't know why it get's initialized with a mode of 'w' + # For the purposes of this test data shouldn't be empty + # This is a problem with this class and not with the hosts + # module + if self.getvalue(): + data[0] = self.getvalue() StringIO.close(self) expected = '\n'.join(( @@ -147,10 +161,11 @@ class HostsTestCase(TestCase, LoaderModuleMockMixin): '3.3.3.3 asdf.asdfadsf asdf', )) + '\n' - with patch('salt.utils.fopen', TmpStringIO): + with patch('salt.utils.files.fopen', TmpStringIO): mock_opt = MagicMock(return_value=None) with patch.dict(hosts.__salt__, {'config.option': mock_opt}): self.assertTrue(hosts.set_host('1.1.1.1', ' ')) + self.assertEqual(data[0], expected) # 'rm_host' function tests: 2 @@ -159,7 +174,7 @@ class HostsTestCase(TestCase, LoaderModuleMockMixin): ''' Tests if specified host entry gets removed from the hosts file ''' - with patch('salt.utils.fopen', mock_open()), \ + with patch('salt.utils.files.fopen', mock_open()), \ patch('salt.modules.hosts.__get_hosts_filename', MagicMock(return_value='/etc/hosts')), \ patch('salt.modules.hosts.has_pair', @@ -182,9 +197,13 @@ class HostsTestCase(TestCase, LoaderModuleMockMixin): ''' Tests if specified host entry gets added from the hosts file ''' - with patch('salt.utils.fopen', mock_open()), \ + hosts_file = '/etc/hosts' + if salt.utils.is_windows(): + hosts_file = r'C:\Windows\System32\Drivers\etc\hosts' + + with patch('salt.utils.files.fopen', mock_open()), \ patch('salt.modules.hosts.__get_hosts_filename', - MagicMock(return_value='/etc/hosts')): + MagicMock(return_value=hosts_file)): mock_opt = MagicMock(return_value=None) with patch.dict(hosts.__salt__, {'config.option': mock_opt}): self.assertTrue(hosts.add_host('10.10.10.10', 'Salt1')) @@ -193,7 +212,7 @@ class HostsTestCase(TestCase, LoaderModuleMockMixin): ''' Tests if specified host entry gets added from the hosts file ''' - with patch('salt.utils.fopen', mock_open()), \ + with patch('salt.utils.files.fopen', mock_open()), \ patch('os.path.isfile', MagicMock(return_value=False)): mock_opt = MagicMock(return_value=None) with patch.dict(hosts.__salt__, {'config.option': mock_opt}): @@ -203,7 +222,7 @@ class HostsTestCase(TestCase, LoaderModuleMockMixin): ''' Tests if specified host entry gets added from the hosts file ''' - with patch('salt.utils.fopen', mock_open()), \ + with patch('salt.utils.files.fopen', mock_open()), \ patch('os.path.isfile', MagicMock(return_value=True)): mock_opt = MagicMock(return_value=None) with patch.dict(hosts.__salt__, {'config.option': mock_opt}): diff --git a/tests/unit/modules/test_ini_manage.py b/tests/unit/modules/test_ini_manage.py index 6edce9b6024..f3550262a97 100644 --- a/tests/unit/modules/test_ini_manage.py +++ b/tests/unit/modules/test_ini_manage.py @@ -9,44 +9,44 @@ import tempfile from tests.support.unit import TestCase # Import Salt libs -import salt.utils +import salt.utils.files import salt.modules.ini_manage as ini class IniManageTestCase(TestCase): - TEST_FILE_CONTENT = '''\ -# Comment on the first line - -# First main option -option1=main1 - -# Second main option -option2=main2 - - -[main] -# Another comment -test1=value 1 - -test2=value 2 - -[SectionB] -test1=value 1B - -# Blank line should be above -test3 = value 3B - -[SectionC] -# The following option is empty -empty_option= -''' + TEST_FILE_CONTENT = os.linesep.join([ + '# Comment on the first line', + '', + '# First main option', + 'option1=main1', + '', + '# Second main option', + 'option2=main2', + '', + '', + '[main]', + '# Another comment', + 'test1=value 1', + '', + 'test2=value 2', + '', + '[SectionB]', + 'test1=value 1B', + '', + '# Blank line should be above', + 'test3 = value 3B', + '', + '[SectionC]', + '# The following option is empty', + 'empty_option=' + ]) maxDiff = None def setUp(self): - self.tfile = tempfile.NamedTemporaryFile(delete=False, mode='w+') - self.tfile.write(self.TEST_FILE_CONTENT) + self.tfile = tempfile.NamedTemporaryFile(delete=False, mode='w+b') + self.tfile.write(salt.utils.to_bytes(self.TEST_FILE_CONTENT)) self.tfile.close() def tearDown(self): @@ -119,42 +119,44 @@ empty_option= ini.set_option(self.tfile.name, { 'SectionB': {'test3': 'new value 3B'}, }) - with salt.utils.fopen(self.tfile.name, 'r') as fp: + with salt.utils.files.fopen(self.tfile.name, 'r') as fp: file_content = fp.read() - self.assertIn('\nempty_option = \n', file_content, - 'empty_option was not preserved') + expected = '{0}{1}{0}'.format(os.linesep, 'empty_option = ') + self.assertIn(expected, file_content, 'empty_option was not preserved') def test_empty_lines_preserved_after_edit(self): ini.set_option(self.tfile.name, { 'SectionB': {'test3': 'new value 3B'}, }) - with salt.utils.fopen(self.tfile.name, 'r') as fp: + expected = os.linesep.join([ + '# Comment on the first line', + '', + '# First main option', + 'option1 = main1', + '', + '# Second main option', + 'option2 = main2', + '', + '[main]', + '# Another comment', + 'test1 = value 1', + '', + 'test2 = value 2', + '', + '[SectionB]', + 'test1 = value 1B', + '', + '# Blank line should be above', + 'test3 = new value 3B', + '', + '[SectionC]', + '# The following option is empty', + 'empty_option = ', + '' + ]) + with salt.utils.files.fopen(self.tfile.name, 'r') as fp: file_content = fp.read() - self.assertEqual('''\ -# Comment on the first line - -# First main option -option1 = main1 - -# Second main option -option2 = main2 - -[main] -# Another comment -test1 = value 1 - -test2 = value 2 - -[SectionB] -test1 = value 1B - -# Blank line should be above -test3 = new value 3B - -[SectionC] -# The following option is empty -empty_option = -''', file_content) + self.assertEqual(expected, file_content) def test_empty_lines_preserved_after_multiple_edits(self): ini.set_option(self.tfile.name, { diff --git a/tests/unit/modules/test_inspect_collector.py b/tests/unit/modules/test_inspect_collector.py index 769276afefd..0d37519a9e2 100644 --- a/tests/unit/modules/test_inspect_collector.py +++ b/tests/unit/modules/test_inspect_collector.py @@ -49,9 +49,15 @@ class InspectorCollectorTestCase(TestCase): :return: ''' - inspector = Inspector(cachedir='/foo/cache', piddir='/foo/pid', pidfilename='bar.pid') - self.assertEqual(inspector.dbfile, '/foo/cache/_minion_collector.db') - self.assertEqual(inspector.pidfile, '/foo/pid/bar.pid') + cachedir = os.sep + os.sep.join(['foo', 'cache']) + piddir = os.sep + os.sep.join(['foo', 'pid']) + inspector = Inspector(cachedir=cachedir, piddir=piddir, pidfilename='bar.pid') + self.assertEqual( + inspector.dbfile, + os.sep + os.sep.join(['foo', 'cache', '_minion_collector.db'])) + self.assertEqual( + inspector.pidfile, + os.sep + os.sep.join(['foo', 'pid', 'bar.pid'])) def test_file_tree(self): ''' @@ -60,12 +66,29 @@ class InspectorCollectorTestCase(TestCase): :return: ''' - inspector = Inspector(cachedir='/test', piddir='/test', pidfilename='bar.pid') + inspector = Inspector(cachedir=os.sep + 'test', + piddir=os.sep + 'test', + pidfilename='bar.pid') tree_root = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'inspectlib', 'tree_test') - expected_tree = (['/a/a/dummy.a', '/a/b/dummy.b', '/b/b.1', '/b/b.2', '/b/b.3'], - ['/a', '/a/a', '/a/b', '/a/c', '/b', '/c'], - ['/a/a/dummy.ln.a', '/a/b/dummy.ln.b', '/a/c/b.1', '/b/b.4', - '/b/b.5', '/c/b.1', '/c/b.2', '/c/b.3']) + expected_tree = ([os.sep + os.sep.join(['a', 'a', 'dummy.a']), + os.sep + os.sep.join(['a', 'b', 'dummy.b']), + os.sep + os.sep.join(['b', 'b.1']), + os.sep + os.sep.join(['b', 'b.2']), + os.sep + os.sep.join(['b', 'b.3'])], + [os.sep + 'a', + os.sep + os.sep.join(['a', 'a']), + os.sep + os.sep.join(['a', 'b']), + os.sep + os.sep.join(['a', 'c']), + os.sep + 'b', + os.sep + 'c'], + [os.sep + os.sep.join(['a', 'a', 'dummy.ln.a']), + os.sep + os.sep.join(['a', 'b', 'dummy.ln.b']), + os.sep + os.sep.join(['a', 'c', 'b.1']), + os.sep + os.sep.join(['b', 'b.4']), + os.sep + os.sep.join(['b', 'b.5']), + os.sep + os.sep.join(['c', 'b.1']), + os.sep + os.sep.join(['c', 'b.2']), + os.sep + os.sep.join(['c', 'b.3'])]) tree_result = [] for chunk in inspector._get_all_files(tree_root): buff = [] @@ -126,7 +149,7 @@ gcc-6-base:i386 inspector.grains_core.os_data = MagicMock() inspector.grains_core.os_data().get = MagicMock(return_value='Debian') self.assertEqual(inspector._get_cfg_pkgs(), 'dpkg') - inspector.grains_core.os_data().get = MagicMock(return_value='SUSE') + inspector.grains_core.os_data().get = MagicMock(return_value='Suse') self.assertEqual(inspector._get_cfg_pkgs(), 'rpm') inspector.grains_core.os_data().get = MagicMock(return_value='redhat') self.assertEqual(inspector._get_cfg_pkgs(), 'rpm') diff --git a/tests/unit/modules/test_inspect_fsdb.py b/tests/unit/modules/test_inspect_fsdb.py index 65bfc64d130..a9d29994763 100644 --- a/tests/unit/modules/test_inspect_fsdb.py +++ b/tests/unit/modules/test_inspect_fsdb.py @@ -30,7 +30,7 @@ from salt.modules.inspectlib.fsdb import CsvDB from salt.modules.inspectlib.entities import CsvDBEntity from salt.utils.odict import OrderedDict -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import StringIO diff --git a/tests/unit/modules/test_iptables.py b/tests/unit/modules/test_iptables.py index 1c4f34118fb..6fe9e912856 100644 --- a/tests/unit/modules/test_iptables.py +++ b/tests/unit/modules/test_iptables.py @@ -60,6 +60,9 @@ class IptablesTestCase(TestCase, LoaderModuleMockMixin): self.assertEqual(iptables.build_rule(**{'if': 'not eth0'}), '! -i eth0') + self.assertEqual(iptables.build_rule(**{'proto': 'tcp', 'syn': '!'}), + '-p tcp ! --syn') + self.assertEqual(iptables.build_rule(dports=[80, 443], proto='tcp'), '-p tcp -m multiport --dports 80,443') diff --git a/tests/unit/modules/test_junos.py b/tests/unit/modules/test_junos.py index 8874412414b..e8b1c6d99a8 100644 --- a/tests/unit/modules/test_junos.py +++ b/tests/unit/modules/test_junos.py @@ -28,7 +28,7 @@ except ImportError: import salt.modules.junos as junos -@skipIf(not HAS_JUNOS, 'Missing dependencies') +@skipIf(not HAS_JUNOS, 'Install junos-eznc to be able to run this test.') class Test_Junos_Module(TestCase, LoaderModuleMockMixin, XMLEqualityMixin): def setup_loader_modules(self): @@ -52,6 +52,7 @@ class Test_Junos_Module(TestCase, LoaderModuleMockMixin, XMLEqualityMixin): host='1.1.1.1', user='test', password='test123', + fact_style='old', gather_facts=False) self.dev.open() self.dev.timeout = 30 @@ -570,7 +571,7 @@ class Test_Junos_Module(TestCase, LoaderModuleMockMixin, XMLEqualityMixin): with patch('jnpr.junos.utils.config.Config.commit_check') as mock_commit_check, \ patch('jnpr.junos.utils.config.Config.commit') as mock_commit, \ patch('jnpr.junos.utils.config.Config.rollback') as mock_rollback, \ - patch('salt.modules.junos.fopen') as mock_fopen, \ + patch('salt.utils.files.fopen') as mock_fopen, \ patch('jnpr.junos.utils.config.Config.diff') as mock_diff: mock_commit_check.return_value = True mock_diff.return_value = 'diff' @@ -591,7 +592,7 @@ class Test_Junos_Module(TestCase, LoaderModuleMockMixin, XMLEqualityMixin): with patch('jnpr.junos.utils.config.Config.commit_check') as mock_commit_check, \ patch('jnpr.junos.utils.config.Config.commit') as mock_commit, \ patch('jnpr.junos.utils.config.Config.rollback') as mock_rollback, \ - patch('salt.modules.junos.fopen') as mock_fopen, \ + patch('salt.utils.files.fopen') as mock_fopen, \ patch('jnpr.junos.utils.config.Config.diff') as mock_diff: mock_commit_check.return_value = True mock_diff.return_value = None @@ -744,7 +745,7 @@ class Test_Junos_Module(TestCase, LoaderModuleMockMixin, XMLEqualityMixin): self.assertEqual(junos.cli('show version'), ret) def test_cli_write_output(self): - with patch('salt.modules.junos.fopen') as mock_fopen, \ + with patch('salt.utils.files.fopen') as mock_fopen, \ patch('jnpr.junos.device.Device.cli') as mock_cli: mock_cli.return_vale = 'cli text output' args = {'__pub_user': 'root', @@ -874,8 +875,8 @@ class Test_Junos_Module(TestCase, LoaderModuleMockMixin, XMLEqualityMixin): patch('jnpr.junos.utils.config.Config.commit_check') as mock_commit_check, \ patch('jnpr.junos.utils.config.Config.diff') as mock_diff, \ patch('jnpr.junos.utils.config.Config.load') as mock_load, \ - patch('salt.modules.junos.safe_rm') as mock_safe_rm, \ - patch('salt.modules.junos.files.mkstemp') as mock_mkstemp, \ + patch('salt.utils.files.safe_rm') as mock_safe_rm, \ + patch('salt.utils.files.mkstemp') as mock_mkstemp, \ patch('os.path.isfile') as mock_isfile, \ patch('os.path.getsize') as mock_getsize: mock_isfile.return_value = True @@ -895,8 +896,8 @@ class Test_Junos_Module(TestCase, LoaderModuleMockMixin, XMLEqualityMixin): patch('jnpr.junos.utils.config.Config.commit_check') as mock_commit_check, \ patch('jnpr.junos.utils.config.Config.diff') as mock_diff, \ patch('jnpr.junos.utils.config.Config.load') as mock_load, \ - patch('salt.modules.junos.safe_rm') as mock_safe_rm, \ - patch('salt.modules.junos.files.mkstemp') as mock_mkstemp, \ + patch('salt.utils.files.safe_rm') as mock_safe_rm, \ + patch('salt.utils.files.mkstemp') as mock_mkstemp, \ patch('os.path.isfile') as mock_isfile, \ patch('os.path.getsize') as mock_getsize: mock_isfile.return_value = True @@ -916,8 +917,8 @@ class Test_Junos_Module(TestCase, LoaderModuleMockMixin, XMLEqualityMixin): patch('jnpr.junos.utils.config.Config.commit_check') as mock_commit_check, \ patch('jnpr.junos.utils.config.Config.diff') as mock_diff, \ patch('jnpr.junos.utils.config.Config.load') as mock_load, \ - patch('salt.modules.junos.safe_rm') as mock_safe_rm, \ - patch('salt.modules.junos.files.mkstemp') as mock_mkstemp, \ + patch('salt.utils.files.safe_rm') as mock_safe_rm, \ + patch('salt.utils.files.mkstemp') as mock_mkstemp, \ patch('os.path.isfile') as mock_isfile, \ patch('os.path.getsize') as mock_getsize: mock_isfile.return_value = True @@ -937,8 +938,8 @@ class Test_Junos_Module(TestCase, LoaderModuleMockMixin, XMLEqualityMixin): patch('jnpr.junos.utils.config.Config.commit_check') as mock_commit_check, \ patch('jnpr.junos.utils.config.Config.diff') as mock_diff, \ patch('jnpr.junos.utils.config.Config.load') as mock_load, \ - patch('salt.modules.junos.safe_rm') as mock_safe_rm, \ - patch('salt.modules.junos.files.mkstemp') as mock_mkstemp, \ + patch('salt.utils.files.safe_rm') as mock_safe_rm, \ + patch('salt.utils.files.mkstemp') as mock_mkstemp, \ patch('os.path.isfile') as mock_isfile, \ patch('os.path.getsize') as mock_getsize: mock_isfile.return_value = True @@ -970,8 +971,8 @@ class Test_Junos_Module(TestCase, LoaderModuleMockMixin, XMLEqualityMixin): patch('jnpr.junos.utils.config.Config.commit_check') as mock_commit_check, \ patch('jnpr.junos.utils.config.Config.diff') as mock_diff, \ patch('jnpr.junos.utils.config.Config.load') as mock_load, \ - patch('salt.modules.junos.safe_rm') as mock_safe_rm, \ - patch('salt.modules.junos.files.mkstemp') as mock_mkstemp, \ + patch('salt.utils.files.safe_rm') as mock_safe_rm, \ + patch('salt.utils.files.mkstemp') as mock_mkstemp, \ patch('os.path.isfile') as mock_isfile, \ patch('os.path.getsize') as mock_getsize: mock_isfile.return_value = True @@ -1003,8 +1004,8 @@ class Test_Junos_Module(TestCase, LoaderModuleMockMixin, XMLEqualityMixin): patch('jnpr.junos.utils.config.Config.commit_check') as mock_commit_check, \ patch('jnpr.junos.utils.config.Config.diff') as mock_diff, \ patch('jnpr.junos.utils.config.Config.load') as mock_load, \ - patch('salt.modules.junos.safe_rm') as mock_safe_rm, \ - patch('salt.modules.junos.files.mkstemp') as mock_mkstemp, \ + patch('salt.utils.files.safe_rm') as mock_safe_rm, \ + patch('salt.utils.files.mkstemp') as mock_mkstemp, \ patch('os.path.isfile') as mock_isfile, \ patch('os.path.getsize') as mock_getsize: mock_isfile.return_value = True @@ -1032,8 +1033,8 @@ class Test_Junos_Module(TestCase, LoaderModuleMockMixin, XMLEqualityMixin): def test_install_config_load_causes_exception(self): with patch('jnpr.junos.utils.config.Config.diff') as mock_diff, \ patch('jnpr.junos.utils.config.Config.load') as mock_load, \ - patch('salt.modules.junos.safe_rm') as mock_safe_rm, \ - patch('salt.modules.junos.files.mkstemp') as mock_mkstemp, \ + patch('salt.utils.files.safe_rm') as mock_safe_rm, \ + patch('salt.utils.files.mkstemp') as mock_mkstemp, \ patch('os.path.isfile') as mock_isfile, \ patch('os.path.getsize') as mock_getsize: mock_isfile.return_value = True @@ -1051,8 +1052,8 @@ class Test_Junos_Module(TestCase, LoaderModuleMockMixin, XMLEqualityMixin): def test_install_config_no_diff(self): with patch('jnpr.junos.utils.config.Config.diff') as mock_diff, \ patch('jnpr.junos.utils.config.Config.load') as mock_load, \ - patch('salt.modules.junos.safe_rm') as mock_safe_rm, \ - patch('salt.modules.junos.files.mkstemp') as mock_mkstemp, \ + patch('salt.utils.files.safe_rm') as mock_safe_rm, \ + patch('salt.utils.files.mkstemp') as mock_mkstemp, \ patch('os.path.isfile') as mock_isfile, \ patch('os.path.getsize') as mock_getsize: mock_isfile.return_value = True @@ -1069,10 +1070,10 @@ class Test_Junos_Module(TestCase, LoaderModuleMockMixin, XMLEqualityMixin): patch('jnpr.junos.utils.config.Config.commit_check') as mock_commit_check, \ patch('jnpr.junos.utils.config.Config.diff') as mock_diff, \ patch('jnpr.junos.utils.config.Config.load') as mock_load, \ - patch('salt.modules.junos.safe_rm') as mock_safe_rm, \ - patch('salt.modules.junos.files.mkstemp') as mock_mkstemp, \ + patch('salt.utils.files.safe_rm') as mock_safe_rm, \ + patch('salt.utils.files.mkstemp') as mock_mkstemp, \ patch('os.path.isfile') as mock_isfile, \ - patch('salt.modules.junos.fopen') as mock_fopen, \ + patch('salt.utils.files.fopen') as mock_fopen, \ patch('os.path.getsize') as mock_getsize: mock_isfile.return_value = True mock_getsize.return_value = 10 @@ -1104,10 +1105,10 @@ class Test_Junos_Module(TestCase, LoaderModuleMockMixin, XMLEqualityMixin): patch('jnpr.junos.utils.config.Config.commit_check') as mock_commit_check, \ patch('jnpr.junos.utils.config.Config.diff') as mock_diff, \ patch('jnpr.junos.utils.config.Config.load') as mock_load, \ - patch('salt.modules.junos.safe_rm') as mock_safe_rm, \ - patch('salt.modules.junos.files.mkstemp') as mock_mkstemp, \ + patch('salt.utils.files.safe_rm') as mock_safe_rm, \ + patch('salt.utils.files.mkstemp') as mock_mkstemp, \ patch('os.path.isfile') as mock_isfile, \ - patch('salt.modules.junos.fopen') as mock_fopen, \ + patch('salt.utils.files.fopen') as mock_fopen, \ patch('os.path.getsize') as mock_getsize: mock_isfile.return_value = True mock_getsize.return_value = 10 @@ -1140,8 +1141,8 @@ class Test_Junos_Module(TestCase, LoaderModuleMockMixin, XMLEqualityMixin): patch('jnpr.junos.utils.config.Config.commit_check') as mock_commit_check, \ patch('jnpr.junos.utils.config.Config.diff') as mock_diff, \ patch('jnpr.junos.utils.config.Config.load') as mock_load, \ - patch('salt.modules.junos.safe_rm') as mock_safe_rm, \ - patch('salt.modules.junos.files.mkstemp') as mock_mkstemp, \ + patch('salt.utils.files.safe_rm') as mock_safe_rm, \ + patch('salt.utils.files.mkstemp') as mock_mkstemp, \ patch('os.path.isfile') as mock_isfile, \ patch('os.path.getsize') as mock_getsize: mock_isfile.return_value = True @@ -1173,8 +1174,8 @@ class Test_Junos_Module(TestCase, LoaderModuleMockMixin, XMLEqualityMixin): with patch('jnpr.junos.utils.config.Config.commit_check') as mock_commit_check, \ patch('jnpr.junos.utils.config.Config.diff') as mock_diff, \ patch('jnpr.junos.utils.config.Config.load') as mock_load, \ - patch('salt.modules.junos.safe_rm') as mock_safe_rm, \ - patch('salt.modules.junos.files.mkstemp') as mock_mkstemp, \ + patch('salt.utils.files.safe_rm') as mock_safe_rm, \ + patch('salt.utils.files.mkstemp') as mock_mkstemp, \ patch('os.path.isfile') as mock_isfile, \ patch('os.path.getsize') as mock_getsize: mock_isfile.return_value = True @@ -1192,8 +1193,8 @@ class Test_Junos_Module(TestCase, LoaderModuleMockMixin, XMLEqualityMixin): with patch('jnpr.junos.utils.config.Config.commit_check') as mock_commit_check, \ patch('jnpr.junos.utils.config.Config.diff') as mock_diff, \ patch('jnpr.junos.utils.config.Config.load') as mock_load, \ - patch('salt.modules.junos.safe_rm') as mock_safe_rm, \ - patch('salt.modules.junos.files.mkstemp') as mock_mkstemp, \ + patch('salt.utils.files.safe_rm') as mock_safe_rm, \ + patch('salt.utils.files.mkstemp') as mock_mkstemp, \ patch('os.path.isfile') as mock_isfile, \ patch('os.path.getsize') as mock_getsize: mock_isfile.return_value = True @@ -1212,8 +1213,8 @@ class Test_Junos_Module(TestCase, LoaderModuleMockMixin, XMLEqualityMixin): patch('jnpr.junos.utils.config.Config.commit_check') as mock_commit_check, \ patch('jnpr.junos.utils.config.Config.diff') as mock_diff, \ patch('jnpr.junos.utils.config.Config.load') as mock_load, \ - patch('salt.modules.junos.safe_rm') as mock_safe_rm, \ - patch('salt.modules.junos.files.mkstemp') as mock_mkstemp, \ + patch('salt.utils.files.safe_rm') as mock_safe_rm, \ + patch('salt.utils.files.mkstemp') as mock_mkstemp, \ patch('os.path.isfile') as mock_isfile, \ patch('os.path.getsize') as mock_getsize: mock_isfile.return_value = True @@ -1274,8 +1275,8 @@ class Test_Junos_Module(TestCase, LoaderModuleMockMixin, XMLEqualityMixin): def test_install_os(self): with patch('jnpr.junos.utils.sw.SW.install') as mock_install, \ - patch('salt.modules.junos.safe_rm') as mock_safe_rm, \ - patch('salt.modules.junos.files.mkstemp') as mock_mkstemp, \ + patch('salt.utils.files.safe_rm') as mock_safe_rm, \ + patch('salt.utils.files.mkstemp') as mock_mkstemp, \ patch('os.path.isfile') as mock_isfile, \ patch('os.path.getsize') as mock_getsize: mock_getsize.return_value = 10 @@ -1288,8 +1289,8 @@ class Test_Junos_Module(TestCase, LoaderModuleMockMixin, XMLEqualityMixin): def test_install_os_with_reboot_arg(self): with patch('jnpr.junos.utils.sw.SW.install') as mock_install, \ patch('jnpr.junos.utils.sw.SW.reboot') as mock_reboot, \ - patch('salt.modules.junos.safe_rm') as mock_safe_rm, \ - patch('salt.modules.junos.files.mkstemp') as mock_mkstemp, \ + patch('salt.utils.files.safe_rm') as mock_safe_rm, \ + patch('salt.utils.files.mkstemp') as mock_mkstemp, \ patch('os.path.isfile') as mock_isfile, \ patch('os.path.getsize') as mock_getsize: mock_getsize.return_value = 10 @@ -1305,8 +1306,8 @@ class Test_Junos_Module(TestCase, LoaderModuleMockMixin, XMLEqualityMixin): def test_install_os_pyez_install_throws_exception(self): with patch('jnpr.junos.utils.sw.SW.install') as mock_install, \ - patch('salt.modules.junos.safe_rm') as mock_safe_rm, \ - patch('salt.modules.junos.files.mkstemp') as mock_mkstemp, \ + patch('salt.utils.files.safe_rm') as mock_safe_rm, \ + patch('salt.utils.files.mkstemp') as mock_mkstemp, \ patch('os.path.isfile') as mock_isfile, \ patch('os.path.getsize') as mock_getsize: mock_getsize.return_value = 10 @@ -1320,8 +1321,8 @@ class Test_Junos_Module(TestCase, LoaderModuleMockMixin, XMLEqualityMixin): def test_install_os_with_reboot_raises_exception(self): with patch('jnpr.junos.utils.sw.SW.install') as mock_install, \ patch('jnpr.junos.utils.sw.SW.reboot') as mock_reboot, \ - patch('salt.modules.junos.safe_rm') as mock_safe_rm, \ - patch('salt.modules.junos.files.mkstemp') as mock_mkstemp, \ + patch('salt.utils.files.safe_rm') as mock_safe_rm, \ + patch('salt.utils.files.mkstemp') as mock_mkstemp, \ patch('os.path.isfile') as mock_isfile, \ patch('os.path.getsize') as mock_getsize: mock_getsize.return_value = 10 @@ -1489,7 +1490,7 @@ class Test_Junos_Module(TestCase, LoaderModuleMockMixin, XMLEqualityMixin): mock_execute.return_value = etree.XML( 'text rpc reply') m = mock_open() - with patch('salt.modules.junos.fopen', m, create=True): + with patch('salt.utils.files.fopen', m, create=True): junos.rpc('get-chassis-inventory', '/path/to/file', 'text') handle = m() handle.write.assert_called_with('text rpc reply') @@ -1499,7 +1500,7 @@ class Test_Junos_Module(TestCase, LoaderModuleMockMixin, XMLEqualityMixin): patch('salt.modules.junos.json.dumps') as mock_dumps: mock_dumps.return_value = 'json rpc reply' m = mock_open() - with patch('salt.modules.junos.fopen', m, create=True): + with patch('salt.utils.files.fopen', m, create=True): junos.rpc('get-chassis-inventory', '/path/to/file', format='json') handle = m() handle.write.assert_called_with('json rpc reply') @@ -1510,7 +1511,7 @@ class Test_Junos_Module(TestCase, LoaderModuleMockMixin, XMLEqualityMixin): patch('jnpr.junos.device.Device.execute') as mock_execute: mock_tostring.return_value = 'xml rpc reply' m = mock_open() - with patch('salt.modules.junos.fopen', m, create=True): + with patch('salt.utils.files.fopen', m, create=True): junos.rpc('get-chassis-inventory', '/path/to/file') handle = m() handle.write.assert_called_with('xml rpc reply') diff --git a/tests/unit/modules/test_k8s.py b/tests/unit/modules/test_k8s.py index fd6e0b67d24..dde48aa27ca 100644 --- a/tests/unit/modules/test_k8s.py +++ b/tests/unit/modules/test_k8s.py @@ -16,7 +16,7 @@ from tests.support.unit import TestCase from tests.support.helpers import skip_if_binaries_missing # Import Salt libs -import salt.utils +import salt.utils.files import salt.modules.k8s as k8s # Import 3rd-party libs @@ -86,7 +86,7 @@ class TestK8SSecrets(TestCase): def test_get_one_secret(self): name = self.name filename = "/tmp/{0}.json".format(name) - with salt.utils.fopen(filename, 'w') as f: + with salt.utils.files.fopen(filename, 'w') as f: json.dump(self.request, f) create = Popen(["kubectl", "--namespace=default", "create", "-f", filename], stdout=PIPE) @@ -102,7 +102,7 @@ class TestK8SSecrets(TestCase): def test_get_decoded_secret(self): name = self.name filename = "/tmp/{0}.json".format(name) - with salt.utils.fopen(filename, 'w') as f: + with salt.utils.files.fopen(filename, 'w') as f: json.dump(self.request, f) create = Popen(["kubectl", "--namespace=default", "create", "-f", filename], stdout=PIPE) @@ -118,7 +118,7 @@ class TestK8SSecrets(TestCase): expected_data = {} for i in range(2): names.append("/tmp/{0}-{1}".format(name, i)) - with salt.utils.fopen("/tmp/{0}-{1}".format(name, i), 'w') as f: + with salt.utils.files.fopen("/tmp/{0}-{1}".format(name, i), 'w') as f: expected_data["{0}-{1}".format(name, i)] = base64.b64encode("{0}{1}".format(name, i)) f.write("{0}{1}".format(name, i)) res = k8s.create_secret("default", name, names, apiserver_url="http://127.0.0.1:8080") @@ -132,7 +132,7 @@ class TestK8SSecrets(TestCase): def test_update_secret(self): name = self.name filename = "/tmp/{0}.json".format(name) - with salt.utils.fopen(filename, 'w') as f: + with salt.utils.files.fopen(filename, 'w') as f: json.dump(self.request, f) create = Popen(["kubectl", "--namespace=default", "create", "-f", filename], stdout=PIPE) @@ -142,7 +142,7 @@ class TestK8SSecrets(TestCase): names = [] for i in range(3): names.append("/tmp/{0}-{1}-updated".format(name, i)) - with salt.utils.fopen("/tmp/{0}-{1}-updated".format(name, i), 'w') as f: + with salt.utils.files.fopen("/tmp/{0}-{1}-updated".format(name, i), 'w') as f: expected_data["{0}-{1}-updated".format(name, i)] = base64.b64encode("{0}{1}-updated".format(name, i)) f.write("{0}{1}-updated".format(name, i)) @@ -158,7 +158,7 @@ class TestK8SSecrets(TestCase): def test_delete_secret(self): name = self.name filename = "/tmp/{0}.json".format(name) - with salt.utils.fopen(filename, 'w') as f: + with salt.utils.files.fopen(filename, 'w') as f: json.dump(self.request, f) create = Popen(["kubectl", "--namespace=default", "create", "-f", filename], stdout=PIPE) @@ -205,7 +205,7 @@ spec: services: "5" """.format(name) filename = "/tmp/{0}.yaml".format(name) - with salt.utils.fopen(filename, 'w') as f: + with salt.utils.files.fopen(filename, 'w') as f: f.write(request) create = Popen(["kubectl", "--namespace={0}".format(namespace), "create", "-f", filename], stdout=PIPE) @@ -239,7 +239,7 @@ spec: services: "5" """.format(name) filename = "/tmp/{0}.yaml".format(name) - with salt.utils.fopen(filename, 'w') as f: + with salt.utils.files.fopen(filename, 'w') as f: f.write(request) create = Popen(["kubectl", "--namespace={0}".format(namespace), "create", "-f", filename], stdout=PIPE) @@ -286,7 +286,7 @@ spec: services: "5" """.format(name) filename = "/tmp/{0}.yaml".format(name) - with salt.utils.fopen(filename, 'w') as f: + with salt.utils.files.fopen(filename, 'w') as f: f.write(request) create = Popen(["kubectl", "--namespace={0}".format(namespace), "create", "-f", filename], stdout=PIPE) @@ -352,7 +352,7 @@ spec: } } filename = "/tmp/{0}.yaml".format(name) - with salt.utils.fopen(filename, 'w') as f: + with salt.utils.files.fopen(filename, 'w') as f: f.write(request) create = Popen(["kubectl", "--namespace=default", "create", "-f", filename], stdout=PIPE) @@ -390,7 +390,7 @@ spec: type: Container """.format(name) filename = "/tmp/{0}.yaml".format(name) - with salt.utils.fopen(filename, 'w') as f: + with salt.utils.files.fopen(filename, 'w') as f: f.write(request) create = Popen(["kubectl", "--namespace=default", "create", "-f", filename], stdout=PIPE) diff --git a/tests/unit/modules/test_kernelpkg.py b/tests/unit/modules/test_kernelpkg.py index 97d21913418..52feae061d4 100644 --- a/tests/unit/modules/test_kernelpkg.py +++ b/tests/unit/modules/test_kernelpkg.py @@ -5,18 +5,21 @@ :maturity: develop versionadded:: oxygen ''' +# pylint: disable=invalid-name,no-member + from __future__ import absolute_import # Salt testing libs try: from tests.support.mock import MagicMock, patch + from salt.exceptions import CommandExecutionError except ImportError: pass class KernelPkgTestCase(object): ''' - Test cases for salt.modules.yumkernelpkg + Test cases shared by all kernelpkg virtual modules ''' def test_active(self): @@ -132,7 +135,7 @@ class KernelPkgTestCase(object): self.assertEqual(result['latest_installed'], self.KERNEL_LIST[-1]) self.assertEqual(result['reboot_requested'], True) self.assertEqual(result['reboot_required'], True) - self._kernelpkg.__salt__['system.reboot'].assert_called_once() + self.assert_called_once(self._kernelpkg.__salt__['system.reboot']) def test_upgrade_needed_without_reboot(self): ''' @@ -171,3 +174,25 @@ class KernelPkgTestCase(object): with patch.object(self._kernelpkg, 'latest_available', return_value=self.KERNEL_LIST[0]): with patch.object(self._kernelpkg, 'latest_installed', return_value=self.KERNEL_LIST[-1]): self.assertFalse(self._kernelpkg.upgrade_available()) + + def test_remove_active(self): + ''' + Test - remove kernel package + ''' + mock = MagicMock(return_value={'retcode': 0, 'stderr': []}) + with patch.dict(self._kernelpkg.__salt__, {'cmd.run_all': mock}): + with patch.object(self._kernelpkg, 'active', return_value=self.KERNEL_LIST[-1]): + with patch.object(self._kernelpkg, 'list_installed', return_value=self.KERNEL_LIST): + self.assertRaises(CommandExecutionError, self._kernelpkg.remove, release=self.KERNEL_LIST[-1]) + self._kernelpkg.__salt__['cmd.run_all'].assert_not_called() + + def test_remove_invalid(self): + ''' + Test - remove kernel package + ''' + mock = MagicMock(return_value={'retcode': 0, 'stderr': []}) + with patch.dict(self._kernelpkg.__salt__, {'cmd.run_all': mock}): + with patch.object(self._kernelpkg, 'active', return_value=self.KERNEL_LIST[-1]): + with patch.object(self._kernelpkg, 'list_installed', return_value=self.KERNEL_LIST): + self.assertRaises(CommandExecutionError, self._kernelpkg.remove, release='invalid') + self._kernelpkg.__salt__['cmd.run_all'].assert_not_called() diff --git a/tests/unit/modules/test_kernelpkg_linux_apt.py b/tests/unit/modules/test_kernelpkg_linux_apt.py index 38488f4e90d..0a20ada9860 100644 --- a/tests/unit/modules/test_kernelpkg_linux_apt.py +++ b/tests/unit/modules/test_kernelpkg_linux_apt.py @@ -5,6 +5,7 @@ :maturity: develop versionadded:: oxygen ''' +# pylint: disable=invalid-name,no-member # Import Python Libs from __future__ import absolute_import @@ -19,6 +20,7 @@ try: # Import Salt Libs from tests.unit.modules.test_kernelpkg import KernelPkgTestCase import salt.modules.kernelpkg_linux_apt as kernelpkg + from salt.exceptions import CommandExecutionError HAS_MODULES = True except ImportError: HAS_MODULES = False @@ -27,6 +29,9 @@ except ImportError: @skipIf(NO_MOCK, NO_MOCK_REASON) @skipIf(not HAS_MODULES, 'Salt modules could not be loaded') class AptKernelPkgTestCase(KernelPkgTestCase, TestCase, LoaderModuleMockMixin): + ''' + Test cases for salt.modules.kernelpkg_linux_apt + ''' _kernelpkg = kernelpkg KERNEL_LIST = ['4.4.0-70-generic', '4.4.0-71-generic', '4.5.1-14-generic'] @@ -38,7 +43,7 @@ class AptKernelPkgTestCase(KernelPkgTestCase, TestCase, LoaderModuleMockMixin): cls.LATEST = '{0}.{1}'.format(version.group(1), version.group(2)) for kernel in cls.KERNEL_LIST: - pkg = '{0}-{1}'.format(kernelpkg._package_prefix(), kernel) + pkg = '{0}-{1}'.format(kernelpkg._package_prefix(), kernel) # pylint: disable=protected-access cls.PACKAGE_DICT[pkg] = pkg def setup_loader_modules(self): @@ -51,6 +56,7 @@ class AptKernelPkgTestCase(KernelPkgTestCase, TestCase, LoaderModuleMockMixin): 'pkg.install': MagicMock(return_value={}), 'pkg.latest_version': MagicMock(return_value=self.LATEST), 'pkg.list_pkgs': MagicMock(return_value=self.PACKAGE_DICT), + 'pkg.purge': MagicMock(return_value=None), 'system.reboot': MagicMock(return_value=None) } } @@ -60,7 +66,7 @@ class AptKernelPkgTestCase(KernelPkgTestCase, TestCase, LoaderModuleMockMixin): ''' Test - Return return the latest installed kernel version ''' - PACKAGE_LIST = ['{0}-{1}'.format(kernelpkg._package_prefix(), kernel) for kernel in self.KERNEL_LIST] + PACKAGE_LIST = ['{0}-{1}'.format(kernelpkg._package_prefix(), kernel) for kernel in self.KERNEL_LIST] # pylint: disable=protected-access mock = MagicMock(return_value=PACKAGE_LIST) with patch.dict(self._kernelpkg.__salt__, {'pkg.list_pkgs': mock}): @@ -73,3 +79,24 @@ class AptKernelPkgTestCase(KernelPkgTestCase, TestCase, LoaderModuleMockMixin): mock = MagicMock(return_value=None) with patch.dict(self._kernelpkg.__salt__, {'pkg.list_pkgs': mock}): self.assertListEqual(self._kernelpkg.list_installed(), []) + + def test_remove_success(self): + ''' + Test - remove kernel package + ''' + with patch.object(self._kernelpkg, 'active', return_value=self.KERNEL_LIST[-1]): + with patch.object(self._kernelpkg, 'list_installed', return_value=self.KERNEL_LIST): + result = self._kernelpkg.remove(release=self.KERNEL_LIST[0]) + self.assertIn('removed', result) + target = '{0}-{1}'.format(self._kernelpkg._package_prefix(), self.KERNEL_LIST[0]) # pylint: disable=protected-access + self.assertListEqual(result['removed'], [target]) + + def test_remove_error(self): + ''' + Test - remove kernel package + ''' + mock = MagicMock(side_effect=CommandExecutionError()) + with patch.dict(self._kernelpkg.__salt__, {'pkg.purge': mock}): + with patch.object(self._kernelpkg, 'active', return_value=self.KERNEL_LIST[-1]): + with patch.object(self._kernelpkg, 'list_installed', return_value=self.KERNEL_LIST): + self.assertRaises(CommandExecutionError, self._kernelpkg.remove, release=self.KERNEL_LIST[0]) diff --git a/tests/unit/modules/test_kernelpkg_linux_yum.py b/tests/unit/modules/test_kernelpkg_linux_yum.py index 9674c049bf0..95753337e02 100644 --- a/tests/unit/modules/test_kernelpkg_linux_yum.py +++ b/tests/unit/modules/test_kernelpkg_linux_yum.py @@ -5,6 +5,7 @@ :maturity: develop versionadded:: oxygen ''' +# pylint: disable=invalid-name,no-member # Import Python Libs from __future__ import absolute_import @@ -19,6 +20,7 @@ try: from tests.unit.modules.test_kernelpkg import KernelPkgTestCase import salt.modules.kernelpkg_linux_yum as kernelpkg import salt.modules.yumpkg as pkg + from salt.exceptions import CommandExecutionError HAS_MODULES = True except ImportError: HAS_MODULES = False @@ -27,23 +29,30 @@ except ImportError: @skipIf(NO_MOCK, NO_MOCK_REASON) @skipIf(not HAS_MODULES, 'Salt modules could not be loaded') class YumKernelPkgTestCase(KernelPkgTestCase, TestCase, LoaderModuleMockMixin): + ''' + Test cases for salt.modules.kernelpkg_linux_yum + ''' _kernelpkg = kernelpkg KERNEL_LIST = ['3.10.0-327.el7', '3.11.0-327.el7', '4.9.1-100.el7'] LATEST = KERNEL_LIST[-1] OS_ARCH = 'x86_64' + OS_NAME = 'RedHat' def setup_loader_modules(self): return { kernelpkg: { '__grains__': { + 'os': self.OS_NAME, 'kernelrelease': '{0}.{1}'.format(self.KERNEL_LIST[0], self.OS_ARCH) }, '__salt__': { 'pkg.normalize_name': pkg.normalize_name, 'pkg.upgrade': MagicMock(return_value={}), + 'pkg.list_pkgs': MagicMock(return_value={}), 'pkg.version': MagicMock(return_value=self.KERNEL_LIST), - 'system.reboot': MagicMock(return_value=None) + 'system.reboot': MagicMock(return_value=None), + 'config.get': MagicMock(return_value=True) } }, pkg: { @@ -68,3 +77,26 @@ class YumKernelPkgTestCase(KernelPkgTestCase, TestCase, LoaderModuleMockMixin): mock = MagicMock(return_value=None) with patch.dict(self._kernelpkg.__salt__, {'pkg.version': mock}): self.assertListEqual(self._kernelpkg.list_installed(), []) + + def test_remove_success(self): + ''' + Test - remove kernel package + ''' + mock = MagicMock(return_value={'retcode': 0, 'stderr': []}) + with patch.dict(self._kernelpkg.__salt__, {'cmd.run_all': mock}): + with patch.object(self._kernelpkg, 'active', return_value=self.KERNEL_LIST[-1]): + with patch.object(self._kernelpkg, 'list_installed', return_value=self.KERNEL_LIST): + result = self._kernelpkg.remove(release=self.KERNEL_LIST[0]) + self.assertIn('removed', result) + target = '{0}-{1}'.format(self._kernelpkg._package_name(), self.KERNEL_LIST[0]) # pylint: disable=protected-access + self.assertListEqual(result['removed'], [target]) + + def test_remove_error(self): + ''' + Test - remove kernel package + ''' + mock = MagicMock(return_value={'retcode': -1, 'stderr': []}) + with patch.dict(self._kernelpkg.__salt__, {'cmd.run_all': mock}): + with patch.object(self._kernelpkg, 'active', return_value=self.KERNEL_LIST[-1]): + with patch.object(self._kernelpkg, 'list_installed', return_value=self.KERNEL_LIST): + self.assertRaises(CommandExecutionError, self._kernelpkg.remove, release=self.KERNEL_LIST[0]) diff --git a/tests/unit/modules/test_kubernetes.py b/tests/unit/modules/test_kubernetes.py new file mode 100644 index 00000000000..f53ce7845a9 --- /dev/null +++ b/tests/unit/modules/test_kubernetes.py @@ -0,0 +1,130 @@ +# -*- coding: utf-8 -*- +''' + :codeauthor: :email:`Jochen Breuer ` +''' + +# Import Python Libs +from __future__ import absolute_import + +# Import Salt Testing Libs +from tests.support.mixins import LoaderModuleMockMixin +from tests.support.unit import TestCase, skipIf +from tests.support.mock import ( + Mock, + patch, + NO_MOCK, + NO_MOCK_REASON +) + +try: + from salt.modules import kubernetes +except ImportError: + kubernetes = False +if not kubernetes.HAS_LIBS: + kubernetes = False + + +@skipIf(NO_MOCK, NO_MOCK_REASON) +@skipIf(kubernetes is False, "Probably Kubernetes client lib is not installed. \ + Skipping test_kubernetes.py") +class KubernetesTestCase(TestCase, LoaderModuleMockMixin): + ''' + Test cases for salt.modules.kubernetes + ''' + + def setup_loader_modules(self): + return { + kubernetes: { + '__salt__': {}, + } + } + + def test_nodes(self): + ''' + Test node listing. + :return: + ''' + with patch('salt.modules.kubernetes.kubernetes') as mock_kubernetes_lib: + with patch.dict(kubernetes.__salt__, {'config.option': Mock(return_value="")}): + mock_kubernetes_lib.client.CoreV1Api.return_value = Mock( + **{"list_node.return_value.to_dict.return_value": + {'items': [{'metadata': {'name': 'mock_node_name'}}]}} + ) + self.assertEqual(kubernetes.nodes(), ['mock_node_name']) + self.assertTrue(kubernetes.kubernetes.client.CoreV1Api().list_node().to_dict.called) + + def test_deployments(self): + ''' + Tests deployment listing. + :return: + ''' + with patch('salt.modules.kubernetes.kubernetes') as mock_kubernetes_lib: + with patch.dict(kubernetes.__salt__, {'config.option': Mock(return_value="")}): + mock_kubernetes_lib.client.ExtensionsV1beta1Api.return_value = Mock( + **{"list_namespaced_deployment.return_value.to_dict.return_value": + {'items': [{'metadata': {'name': 'mock_deployment_name'}}]}} + ) + self.assertEqual(kubernetes.deployments(), ['mock_deployment_name']) + self.assertTrue( + kubernetes.kubernetes.client.ExtensionsV1beta1Api().list_namespaced_deployment().to_dict.called) + + def test_services(self): + ''' + Tests services listing. + :return: + ''' + with patch('salt.modules.kubernetes.kubernetes') as mock_kubernetes_lib: + with patch.dict(kubernetes.__salt__, {'config.option': Mock(return_value="")}): + mock_kubernetes_lib.client.CoreV1Api.return_value = Mock( + **{"list_namespaced_service.return_value.to_dict.return_value": + {'items': [{'metadata': {'name': 'mock_service_name'}}]}} + ) + self.assertEqual(kubernetes.services(), ['mock_service_name']) + self.assertTrue(kubernetes.kubernetes.client.CoreV1Api().list_namespaced_service().to_dict.called) + + def test_pods(self): + ''' + Tests pods listing. + :return: + ''' + with patch('salt.modules.kubernetes.kubernetes') as mock_kubernetes_lib: + with patch.dict(kubernetes.__salt__, {'config.option': Mock(return_value="")}): + mock_kubernetes_lib.client.CoreV1Api.return_value = Mock( + **{"list_namespaced_pod.return_value.to_dict.return_value": + {'items': [{'metadata': {'name': 'mock_pod_name'}}]}} + ) + self.assertEqual(kubernetes.pods(), ['mock_pod_name']) + self.assertTrue(kubernetes.kubernetes.client.CoreV1Api(). + list_namespaced_pod().to_dict.called) + + def test_delete_deployments(self): + ''' + Tests deployment creation. + :return: + ''' + with patch('salt.modules.kubernetes.kubernetes') as mock_kubernetes_lib: + with patch.dict(kubernetes.__salt__, {'config.option': Mock(return_value="")}): + mock_kubernetes_lib.client.V1DeleteOptions = Mock(return_value="") + mock_kubernetes_lib.client.ExtensionsV1beta1Api.return_value = Mock( + **{"delete_namespaced_deployment.return_value.to_dict.return_value": {}} + ) + self.assertEqual(kubernetes.delete_deployment("test"), {}) + self.assertTrue( + kubernetes.kubernetes.client.ExtensionsV1beta1Api(). + delete_namespaced_deployment().to_dict.called) + + def test_create_deployments(self): + ''' + Tests deployment creation. + :return: + ''' + with patch('salt.modules.kubernetes.kubernetes') as mock_kubernetes_lib: + with patch.dict(kubernetes.__salt__, {'config.option': Mock(return_value="")}): + mock_kubernetes_lib.client.ExtensionsV1beta1Api.return_value = Mock( + **{"create_namespaced_deployment.return_value.to_dict.return_value": {}} + ) + self.assertEqual(kubernetes.create_deployment("test", "default", {}, {}, + None, None, None), {}) + self.assertTrue( + kubernetes.kubernetes.client.ExtensionsV1beta1Api(). + create_namespaced_deployment().to_dict.called) diff --git a/tests/unit/modules/test_launchctl.py b/tests/unit/modules/test_launchctl.py index 32d21fb6efe..108ba90c342 100644 --- a/tests/unit/modules/test_launchctl.py +++ b/tests/unit/modules/test_launchctl.py @@ -17,8 +17,9 @@ from tests.support.mock import ( ) # Import Salt Libs -import salt.ext.six as six +from salt.ext import six import salt.utils +import salt.utils.stringutils import salt.modules.launchctl as launchctl @@ -88,7 +89,7 @@ class LaunchctlTestCase(TestCase, LoaderModuleMockMixin): return_value={'plist': {'Label': 'A'}}): if six.PY3: - launchctl_data = salt.utils.to_bytes(launchctl_data) + launchctl_data = salt.utils.stringutils.to_bytes(launchctl_data) with patch.object(launchctl, '_get_launchctl_data', return_value=launchctl_data): self.assertTrue(launchctl.status('job_label')) diff --git a/tests/unit/modules/test_linux_acl.py b/tests/unit/modules/test_linux_acl.py index 9fc07569bed..429f45190d1 100644 --- a/tests/unit/modules/test_linux_acl.py +++ b/tests/unit/modules/test_linux_acl.py @@ -6,7 +6,7 @@ from __future__ import absolute_import # Import Salt Testing libs from tests.support.mixins import LoaderModuleMockMixin from tests.support.unit import skipIf, TestCase -from tests.support.mock import NO_MOCK, NO_MOCK_REASON, MagicMock +from tests.support.mock import NO_MOCK, NO_MOCK_REASON, MagicMock, patch # Import salt libs import salt.modules.linux_acl as linux_acl @@ -84,63 +84,70 @@ class LinuxAclTestCase(TestCase, LoaderModuleMockMixin): def test_modfacl__u_w_single_arg(self): linux_acl.modfacl(*(self.u_acl + [self.file])) - self.cmdrun.assert_called_once_with('setfacl -m ' + ' '.join([self.user_acl_cmd, self.quoted_file]), python_shell=False) + self.cmdrun.assert_called_once_with('setfacl -m ' + ' '.join([self.user_acl_cmd, self.quoted_file]), python_shell=False, raise_err=False) def test_modfacl__u_w_multiple_args(self): linux_acl.modfacl(*(self.u_acl + self.files)) - self.cmdrun.assert_called_once_with('setfacl -m ' + ' '.join([self.user_acl_cmd] + self.quoted_files), python_shell=False) + self.cmdrun.assert_called_once_with('setfacl -m ' + ' '.join([self.user_acl_cmd] + self.quoted_files), python_shell=False, raise_err=False) def test_modfacl__user_w_single_arg(self): linux_acl.modfacl(*(self.user_acl + [self.file])) - self.cmdrun.assert_called_once_with('setfacl -m ' + ' '.join([self.user_acl_cmd, self.quoted_file]), python_shell=False) + self.cmdrun.assert_called_once_with('setfacl -m ' + ' '.join([self.user_acl_cmd, self.quoted_file]), python_shell=False, raise_err=False) def test_modfacl__user_w_multiple_args(self): linux_acl.modfacl(*(self.user_acl + self.files)) - self.cmdrun.assert_called_once_with('setfacl -m ' + ' '.join([self.user_acl_cmd] + self.quoted_files), python_shell=False) + self.cmdrun.assert_called_once_with('setfacl -m ' + ' '.join([self.user_acl_cmd] + self.quoted_files), python_shell=False, raise_err=False) def test_modfacl__g_w_single_arg(self): linux_acl.modfacl(*(self.g_acl + [self.file])) - self.cmdrun.assert_called_once_with('setfacl -m ' + ' '.join([self.group_acl_cmd, self.quoted_file]), python_shell=False) + self.cmdrun.assert_called_once_with('setfacl -m ' + ' '.join([self.group_acl_cmd, self.quoted_file]), python_shell=False, raise_err=False) def test_modfacl__g_w_multiple_args(self): linux_acl.modfacl(*(self.g_acl + self.files)) - self.cmdrun.assert_called_once_with('setfacl -m ' + ' '.join([self.group_acl_cmd] + self.quoted_files), python_shell=False) + self.cmdrun.assert_called_once_with('setfacl -m ' + ' '.join([self.group_acl_cmd] + self.quoted_files), python_shell=False, raise_err=False) def test_modfacl__group_w_single_arg(self): linux_acl.modfacl(*(self.group_acl + [self.file])) - self.cmdrun.assert_called_once_with('setfacl -m ' + ' '.join([self.group_acl_cmd, self.quoted_file]), python_shell=False) + self.cmdrun.assert_called_once_with('setfacl -m ' + ' '.join([self.group_acl_cmd, self.quoted_file]), python_shell=False, raise_err=False) def test_modfacl__group_w_multiple_args(self): linux_acl.modfacl(*(self.group_acl + self.files)) - self.cmdrun.assert_called_once_with('setfacl -m ' + ' '.join([self.group_acl_cmd] + self.quoted_files), python_shell=False) + self.cmdrun.assert_called_once_with('setfacl -m ' + ' '.join([self.group_acl_cmd] + self.quoted_files), python_shell=False, raise_err=False) def test_modfacl__d_u_w_single_arg(self): linux_acl.modfacl(*(self.d_u_acl + [self.file])) - self.cmdrun.assert_called_once_with('setfacl -m ' + ' '.join([self.default_user_acl_cmd, self.quoted_file]), python_shell=False) + self.cmdrun.assert_called_once_with('setfacl -m ' + ' '.join([self.default_user_acl_cmd, self.quoted_file]), python_shell=False, raise_err=False) def test_modfacl__d_u_w_multiple_args(self): linux_acl.modfacl(*(self.d_u_acl + self.files)) - self.cmdrun.assert_called_once_with('setfacl -m ' + ' '.join([self.default_user_acl_cmd] + self.quoted_files), python_shell=False) + self.cmdrun.assert_called_once_with('setfacl -m ' + ' '.join([self.default_user_acl_cmd] + self.quoted_files), python_shell=False, raise_err=False) def test_modfacl__d_user_w_single_arg(self): linux_acl.modfacl(*(self.d_user_acl + [self.file])) - self.cmdrun.assert_called_once_with('setfacl -m ' + ' '.join([self.default_user_acl_cmd, self.quoted_file]), python_shell=False) + self.cmdrun.assert_called_once_with('setfacl -m ' + ' '.join([self.default_user_acl_cmd, self.quoted_file]), python_shell=False, raise_err=False) def test_modfacl__d_user_w_multiple_args(self): linux_acl.modfacl(*(self.d_user_acl + self.files)) - self.cmdrun.assert_called_once_with('setfacl -m ' + ' '.join([self.default_user_acl_cmd] + self.quoted_files), python_shell=False) + self.cmdrun.assert_called_once_with('setfacl -m ' + ' '.join([self.default_user_acl_cmd] + self.quoted_files), python_shell=False, raise_err=False) def test_modfacl__default_user_w_single_arg(self): linux_acl.modfacl(*(self.default_user_acl + [self.file])) - self.cmdrun.assert_called_once_with('setfacl -m ' + ' '.join([self.default_user_acl_cmd, self.quoted_file]), python_shell=False) + self.cmdrun.assert_called_once_with('setfacl -m ' + ' '.join([self.default_user_acl_cmd, self.quoted_file]), python_shell=False, raise_err=False) def test_modfacl__default_user_w_multiple_args(self): linux_acl.modfacl(*(self.default_user_acl + self.files)) - self.cmdrun.assert_called_once_with('setfacl -m ' + ' '.join([self.default_user_acl_cmd] + self.quoted_files), python_shell=False) + self.cmdrun.assert_called_once_with('setfacl -m ' + ' '.join([self.default_user_acl_cmd] + self.quoted_files), python_shell=False, raise_err=False) def test_modfacl__recursive_w_multiple_args(self): linux_acl.modfacl(*(self.user_acl + self.files), recursive=True) - self.cmdrun.assert_called_once_with('setfacl -R -m ' + ' '.join([self.user_acl_cmd] + self.quoted_files), python_shell=False) + self.cmdrun.assert_called_once_with('setfacl -R -m ' + ' '.join([self.user_acl_cmd] + self.quoted_files), python_shell=False, raise_err=False) + + def test_modfacl_raise_err(self): + mock = MagicMock(side_effect=CommandExecutionError('Custom err')) + with patch.dict(linux_acl.__salt__, {'cmd.run': mock}): + with self.assertRaises(CommandExecutionError) as excinfo: + linux_acl.modfacl(*(self.user_acl + self.files), raise_err=True) + self.assertEqual(excinfo.exception.strerror, 'Custom err') def test_delfacl_wo_args(self): for acl in [self.u_acl, self.user_acl, self.g_acl, self.group_acl]: diff --git a/tests/unit/modules/test_linux_sysctl.py b/tests/unit/modules/test_linux_sysctl.py index 1299bbebffe..0cbd23169f1 100644 --- a/tests/unit/modules/test_linux_sysctl.py +++ b/tests/unit/modules/test_linux_sysctl.py @@ -91,7 +91,7 @@ class LinuxSysctlTestCase(TestCase, LoaderModuleMockMixin): mock_cmd = MagicMock(return_value=cmd) with patch.dict(linux_sysctl.__salt__, {'cmd.run_stdout': mock_cmd, 'cmd.run_all': mock_asn_cmd}): - with patch('salt.utils.fopen', mock_open()) as m_open: + with patch('salt.utils.files.fopen', mock_open()) as m_open: self.assertRaises(CommandExecutionError, linux_sysctl.persist, 'net.ipv4.ip_forward', @@ -110,7 +110,7 @@ class LinuxSysctlTestCase(TestCase, LoaderModuleMockMixin): sys_cmd = 'systemd 208\n+PAM +LIBWRAP' mock_sys_cmd = MagicMock(return_value=sys_cmd) - with patch('salt.utils.fopen', mock_open()) as m_open: + with patch('salt.utils.files.fopen', mock_open()) as m_open: with patch.dict(linux_sysctl.__context__, {'salt.utils.systemd.version': 232}): with patch.dict(linux_sysctl.__salt__, {'cmd.run_stdout': mock_sys_cmd, @@ -136,7 +136,7 @@ class LinuxSysctlTestCase(TestCase, LoaderModuleMockMixin): sys_cmd = 'systemd 208\n+PAM +LIBWRAP' mock_sys_cmd = MagicMock(return_value=sys_cmd) - with patch('salt.utils.fopen', mock_open()): + with patch('salt.utils.files.fopen', mock_open()): with patch.dict(linux_sysctl.__context__, {'salt.utils.systemd.version': 232}): with patch.dict(linux_sysctl.__salt__, {'cmd.run_stdout': mock_sys_cmd, diff --git a/tests/unit/modules/test_localemod.py b/tests/unit/modules/test_localemod.py index 0c9c9bed480..23c6bf7ea08 100644 --- a/tests/unit/modules/test_localemod.py +++ b/tests/unit/modules/test_localemod.py @@ -113,7 +113,7 @@ class LocalemodTestCase(TestCase, LoaderModuleMockMixin): Tests the return of gen_locale when the provided locale is not found ''' with patch.dict(localemod.__grains__, {'os': 'Debian'}), \ - patch('salt.utils.which', MagicMock(return_value='/some/dir/path')), \ + patch('salt.utils.path.which', MagicMock(return_value='/some/dir/path')), \ patch.dict(localemod.__salt__, {'file.search': MagicMock(return_value=False)}): self.assertFalse(localemod.gen_locale('foo')) @@ -124,7 +124,7 @@ class LocalemodTestCase(TestCase, LoaderModuleMockMixin): ''' ret = {'stdout': 'saltines', 'stderr': 'biscuits', 'retcode': 0, 'pid': 1337} with patch.dict(localemod.__grains__, {'os': 'Debian'}), \ - patch('salt.utils.which', MagicMock(return_value='/some/dir/path')), \ + patch('salt.utils.path.which', MagicMock(return_value='/some/dir/path')), \ patch.dict(localemod.__salt__, {'file.search': MagicMock(return_value=True), 'file.replace': MagicMock(return_value=True), @@ -146,7 +146,7 @@ class LocalemodTestCase(TestCase, LoaderModuleMockMixin): ret = {'stdout': 'saltines', 'stderr': 'biscuits', 'retcode': 0, 'pid': 1337} with patch.dict(localemod.__grains__, {'os': 'Debian'}), \ - patch('salt.utils.which', MagicMock(return_value='/some/dir/path')), \ + patch('salt.utils.path.which', MagicMock(return_value='/some/dir/path')), \ patch.dict(localemod.__salt__, {'file.search': file_search, 'file.replace': MagicMock(return_value=True), @@ -163,7 +163,7 @@ class LocalemodTestCase(TestCase, LoaderModuleMockMixin): 'file.touch': MagicMock(return_value=None), 'file.append': MagicMock(return_value=None), 'cmd.run_all': MagicMock(return_value=ret)}), \ - patch('salt.utils.which', MagicMock(return_value='/some/dir/path')), \ + patch('salt.utils.path.which', MagicMock(return_value='/some/dir/path')), \ patch('os.listdir', MagicMock(return_value=['en_US'])), \ patch.dict(localemod.__grains__, {'os': 'Ubuntu'}): self.assertTrue(localemod.gen_locale('en_US.UTF-8')) @@ -174,7 +174,7 @@ class LocalemodTestCase(TestCase, LoaderModuleMockMixin): ''' ret = {'stdout': 'saltines', 'stderr': 'biscuits', 'retcode': 0, 'pid': 1337} with patch.dict(localemod.__grains__, {'os_family': 'Gentoo'}), \ - patch('salt.utils.which', MagicMock(return_value='/some/dir/path')), \ + patch('salt.utils.path.which', MagicMock(return_value='/some/dir/path')), \ patch('os.listdir', MagicMock(return_value=['en_US.UTF-8'])), \ patch.dict(localemod.__salt__, {'file.search': MagicMock(return_value=True), @@ -197,7 +197,7 @@ class LocalemodTestCase(TestCase, LoaderModuleMockMixin): ret = {'stdout': 'saltines', 'stderr': 'biscuits', 'retcode': 0, 'pid': 1337} with patch.dict(localemod.__grains__, {'os_family': 'Gentoo'}), \ - patch('salt.utils.which', MagicMock(return_value='/some/dir/path')), \ + patch('salt.utils.path.which', MagicMock(return_value='/some/dir/path')), \ patch('os.listdir', MagicMock(return_value=['en_US.UTF-8'])), \ patch.dict(localemod.__salt__, {'file.search': file_search, @@ -213,7 +213,7 @@ class LocalemodTestCase(TestCase, LoaderModuleMockMixin): with patch.dict(localemod.__salt__, {'cmd.run_all': MagicMock(return_value=ret), 'file.replace': MagicMock()}), \ - patch('salt.utils.which', MagicMock(return_value='/some/dir/path')), \ + patch('salt.utils.path.which', MagicMock(return_value='/some/dir/path')), \ patch('os.listdir', MagicMock(return_value=['en_US'])): self.assertTrue(localemod.gen_locale('en_US.UTF-8')) @@ -225,7 +225,7 @@ class LocalemodTestCase(TestCase, LoaderModuleMockMixin): with patch.dict(localemod.__salt__, {'cmd.run_all': MagicMock(return_value=ret), 'file.replace': MagicMock()}), \ - patch('salt.utils.which', MagicMock(return_value='/some/dir/path')), \ + patch('salt.utils.path.which', MagicMock(return_value='/some/dir/path')), \ patch('os.listdir', MagicMock(return_value=['en_US'])): self.assertEqual(localemod.gen_locale('en_US.UTF-8', verbose=True), ret) diff --git a/tests/unit/modules/test_mac_group.py b/tests/unit/modules/test_mac_group.py index 2c03deb3578..d69288ccb9b 100644 --- a/tests/unit/modules/test_mac_group.py +++ b/tests/unit/modules/test_mac_group.py @@ -5,11 +5,15 @@ # Import python libs from __future__ import absolute_import -import grp +HAS_GRP = True +try: + import grp +except ImportError: + HAS_GRP = False # Import Salt Testing Libs from tests.support.mixins import LoaderModuleMockMixin -from tests.support.unit import TestCase +from tests.support.unit import TestCase, skipIf from tests.support.mock import MagicMock, patch # Import Salt Libs @@ -17,6 +21,7 @@ import salt.modules.mac_group as mac_group from salt.exceptions import SaltInvocationError, CommandExecutionError +@skipIf(not HAS_GRP, "Missing required library 'grp'") class MacGroupTestCase(TestCase, LoaderModuleMockMixin): ''' TestCase for the salt.modules.mac_group module diff --git a/tests/unit/modules/test_mac_sysctl.py b/tests/unit/modules/test_mac_sysctl.py index d35bd7615ed..1779cba8427 100644 --- a/tests/unit/modules/test_mac_sysctl.py +++ b/tests/unit/modules/test_mac_sysctl.py @@ -67,7 +67,7 @@ class DarwinSysctlTestCase(TestCase, LoaderModuleMockMixin): ''' Tests adding of config file failure ''' - with patch('salt.utils.fopen', mock_open()) as m_open, \ + with patch('salt.utils.files.fopen', mock_open()) as m_open, \ patch('os.path.isfile', MagicMock(return_value=False)): m_open.side_effect = IOError(13, 'Permission denied', '/file') self.assertRaises(CommandExecutionError, @@ -79,7 +79,7 @@ class DarwinSysctlTestCase(TestCase, LoaderModuleMockMixin): ''' Tests successful add of config file when previously not one ''' - with patch('salt.utils.fopen', mock_open()) as m_open, \ + with patch('salt.utils.files.fopen', mock_open()) as m_open, \ patch('os.path.isfile', MagicMock(return_value=False)): mac_sysctl.persist('net.inet.icmp.icmplim', 50) helper_open = m_open() @@ -92,7 +92,7 @@ class DarwinSysctlTestCase(TestCase, LoaderModuleMockMixin): ''' to_write = '#\n# Kernel sysctl configuration\n#\n' m_calls_list = [call.writelines(['net.inet.icmp.icmplim=50', '\n'])] - with patch('salt.utils.fopen', mock_open(read_data=to_write)) as m_open, \ + with patch('salt.utils.files.fopen', mock_open(read_data=to_write)) as m_open, \ patch('os.path.isfile', MagicMock(return_value=True)): mac_sysctl.persist('net.inet.icmp.icmplim', 50, config=to_write) helper_open = m_open() diff --git a/tests/unit/modules/test_mac_user.py b/tests/unit/modules/test_mac_user.py index 51402e6cd00..c639f022dad 100644 --- a/tests/unit/modules/test_mac_user.py +++ b/tests/unit/modules/test_mac_user.py @@ -2,10 +2,13 @@ ''' :codeauthor: :email:`Nicole Thomas ` ''' - # Import python libs from __future__ import absolute_import -import pwd +HAS_PWD = True +try: + import pwd +except ImportError: + HAS_PWD = False # Import Salt Testing Libs from tests.support.mixins import LoaderModuleMockMixin @@ -17,6 +20,7 @@ import salt.modules.mac_user as mac_user from salt.exceptions import SaltInvocationError, CommandExecutionError +@skipIf(not HAS_PWD, "Missing required library 'pwd'") @skipIf(NO_MOCK, NO_MOCK_REASON) class MacUserTestCase(TestCase, LoaderModuleMockMixin): ''' @@ -26,14 +30,15 @@ class MacUserTestCase(TestCase, LoaderModuleMockMixin): def setup_loader_modules(self): return {mac_user: {}} - mock_pwall = [pwd.struct_passwd(('_amavisd', '*', 83, 83, 'AMaViS Daemon', - '/var/virusmails', '/usr/bin/false')), - pwd.struct_passwd(('_appleevents', '*', 55, 55, - 'AppleEvents Daemon', - '/var/empty', '/usr/bin/false')), - pwd.struct_passwd(('_appowner', '*', 87, 87, - 'Application Owner', - '/var/empty', '/usr/bin/false'))] + if HAS_PWD: + mock_pwall = [pwd.struct_passwd(('_amavisd', '*', 83, 83, 'AMaViS Daemon', + '/var/virusmails', '/usr/bin/false')), + pwd.struct_passwd(('_appleevents', '*', 55, 55, + 'AppleEvents Daemon', + '/var/empty', '/usr/bin/false')), + pwd.struct_passwd(('_appowner', '*', 87, 87, + 'Application Owner', + '/var/empty', '/usr/bin/false'))] mock_info_ret = {'shell': '/bin/bash', 'name': 'test', 'gid': 4376, 'groups': ['TEST_GROUP'], 'home': '/Users/foo', 'fullname': 'TEST USER', 'uid': 4376} diff --git a/tests/unit/modules/test_mdadm.py b/tests/unit/modules/test_mdadm.py index 4c6f5acafe4..ac62b67cfed 100644 --- a/tests/unit/modules/test_mdadm.py +++ b/tests/unit/modules/test_mdadm.py @@ -18,9 +18,6 @@ from tests.support.mock import NO_MOCK, NO_MOCK_REASON, MagicMock, patch # Import salt libs import salt.modules.mdadm as mdadm -# Import 3rd-party libs -import salt.ext.six as six - @skipIf(NO_MOCK, NO_MOCK_REASON) class MdadmTestCase(TestCase, LoaderModuleMockMixin): @@ -31,7 +28,7 @@ class MdadmTestCase(TestCase, LoaderModuleMockMixin): def test_create(self): mock = MagicMock(return_value='salt') with patch.dict(mdadm.__salt__, {'cmd.run': mock}), \ - patch('salt.utils.which', lambda exe: exe): + patch('salt.utils.path.which', lambda exe: exe): ret = mdadm.create( '/dev/md0', 5, devices=['/dev/sdb1', '/dev/sdc1', '/dev/sdd1'], @@ -40,32 +37,28 @@ class MdadmTestCase(TestCase, LoaderModuleMockMixin): chunk=256 ) self.assertEqual('salt', ret) - if six.PY2: - expected_args = [ - 'mdadm', - '-C', '/dev/md0', - '-R', - '-v', - '--chunk', '256', - '--force', - '-l', '5', - '-e', 'default', - '-n', '3', - '/dev/sdb1', '/dev/sdc1', '/dev/sdd1'] - else: - expected_args = [ - 'mdadm', - '-C', '/dev/md0', - '-R', - '-v', - '--force', - '--chunk', '256', - '-l', '5', - '-e', 'default', - '-n', '3', - '/dev/sdb1', '/dev/sdc1', '/dev/sdd1' - ] - mock.assert_called_once_with(expected_args, python_shell=False) + + self.assert_called_once(mock) + + args, kwargs = mock.call_args + # expected cmd is + # mdadm -C /dev/md0 -R -v --chunk 256 --force -l 5 -e default -n 3 /dev/sdb1 /dev/sdc1 /dev/sdd1 + # where args between -v and -l could be in any order + self.assertEqual(len(args), 1) + self.assertEqual(len(args[0]), 17) + self.assertEqual(args[0][:5], [ + 'mdadm', + '-C', '/dev/md0', + '-R', + '-v']) + self.assertEqual(args[0][8:], [ + '-l', '5', + '-e', 'default', + '-n', '3', + '/dev/sdb1', '/dev/sdc1', '/dev/sdd1']) + self.assertEqual(sorted(args[0][5:8]), sorted(['--chunk', '256', '--force'])) + self.assertIn('--chunk 256', ' '.join(args[0][5:8])) + self.assertEqual(kwargs, {'python_shell': False}) def test_create_test_mode(self): mock = MagicMock() diff --git a/tests/unit/modules/test_mod_random.py b/tests/unit/modules/test_mod_random.py index 12663d05070..1a80bd926d5 100644 --- a/tests/unit/modules/test_mod_random.py +++ b/tests/unit/modules/test_mod_random.py @@ -21,7 +21,7 @@ import salt.utils.pycrypto from salt.exceptions import SaltInvocationError # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six def _test_hashlib(): diff --git a/tests/unit/modules/test_mount.py b/tests/unit/modules/test_mount.py index 4f72f28a1a5..8d8dcb60672 100644 --- a/tests/unit/modules/test_mount.py +++ b/tests/unit/modules/test_mount.py @@ -5,6 +5,7 @@ # Import Python libs from __future__ import absolute_import import os +import textwrap # Import Salt Testing Libs from tests.support.mixins import LoaderModuleMockMixin @@ -18,9 +19,10 @@ from tests.support.mock import ( ) # Import Salt Libs -import salt.utils -from salt.exceptions import CommandExecutionError +import salt.utils.files +import salt.utils.path import salt.modules.mount as mount +from salt.exceptions import CommandExecutionError MOCK_SHELL_FILE = 'A B C D F G\n' @@ -90,7 +92,7 @@ class MountTestCase(TestCase, LoaderModuleMockMixin): with patch.object(os.path, 'isfile', mock): file_data = '\n'.join(['#', 'A B C D,E,F G H']) - with patch('salt.utils.fopen', + with patch('salt.utils.files.fopen', mock_open(read_data=file_data), create=True) as m: m.return_value.__iter__.return_value = file_data.splitlines() @@ -113,7 +115,7 @@ class MountTestCase(TestCase, LoaderModuleMockMixin): with patch.object(os.path, 'isfile', mock): file_data = '\n'.join(['#', 'swap - /tmp tmpfs - yes size=2048m']) - with patch('salt.utils.fopen', + with patch('salt.utils.files.fopen', mock_open(read_data=file_data), create=True) as m: m.return_value.__iter__.return_value = file_data.splitlines() @@ -131,7 +133,7 @@ class MountTestCase(TestCase, LoaderModuleMockMixin): mock_fstab = MagicMock(return_value={}) with patch.dict(mount.__grains__, {'kernel': ''}): with patch.object(mount, 'fstab', mock_fstab): - with patch('salt.utils.fopen', mock_open()): + with patch('salt.utils.files.fopen', mock_open()): self.assertTrue(mount.rm_fstab('name', 'device')) def test_set_fstab(self): @@ -148,13 +150,13 @@ class MountTestCase(TestCase, LoaderModuleMockMixin): mock = MagicMock(return_value=True) mock_read = MagicMock(side_effect=OSError) with patch.object(os.path, 'isfile', mock): - with patch.object(salt.utils, 'fopen', mock_read): + with patch.object(salt.utils.files, 'fopen', mock_read): self.assertRaises(CommandExecutionError, mount.set_fstab, 'A', 'B', 'C') mock = MagicMock(return_value=True) with patch.object(os.path, 'isfile', mock): - with patch('salt.utils.fopen', + with patch('salt.utils.files.fopen', mock_open(read_data=MOCK_SHELL_FILE)): self.assertEqual(mount.set_fstab('A', 'B', 'C'), 'new') @@ -242,15 +244,30 @@ class MountTestCase(TestCase, LoaderModuleMockMixin): ''' Returns true if the command passed is a fuse mountable application ''' - with patch.object(salt.utils, 'which', return_value=None): + with patch.object(salt.utils.path, 'which', return_value=None): self.assertFalse(mount.is_fuse_exec('cmd')) - with patch.object(salt.utils, 'which', return_value=True): - self.assertFalse(mount.is_fuse_exec('cmd')) - - mock = MagicMock(side_effect=[1, 0]) - with patch.object(salt.utils, 'which', mock): - self.assertFalse(mount.is_fuse_exec('cmd')) + def _ldd_side_effect(cmd, *args, **kwargs): + ''' + Neither of these are full ldd output, but what is_fuse_exec is + looking for is 'libfuse' in the ldd output, so these examples + should be sufficient enough to test both the True and False cases. + ''' + return { + 'ldd cmd1': textwrap.dedent('''\ + linux-vdso.so.1 (0x00007ffeaf5fb000) + libfuse3.so.3 => /usr/lib/libfuse3.so.3 (0x00007f91e66ac000) + '''), + 'ldd cmd2': textwrap.dedent('''\ + linux-vdso.so.1 (0x00007ffeaf5fb000) + ''') + }[cmd] + which_mock = MagicMock(side_effect=lambda x: x) + ldd_mock = MagicMock(side_effect=_ldd_side_effect) + with patch.object(salt.utils.path, 'which', which_mock): + with patch.dict(mount.__salt__, {'cmd.run': _ldd_side_effect}): + self.assertTrue(mount.is_fuse_exec('cmd1')) + self.assertFalse(mount.is_fuse_exec('cmd2')) def test_swaps(self): ''' @@ -260,7 +277,7 @@ class MountTestCase(TestCase, LoaderModuleMockMixin): file_data = '\n'.join(['Filename Type Size Used Priority', '/dev/sda1 partition 31249404 4100 -1']) with patch.dict(mount.__grains__, {'os': '', 'kernel': ''}): - with patch('salt.utils.fopen', + with patch('salt.utils.files.fopen', mock_open(read_data=file_data), create=True) as m: m.return_value.__iter__.return_value = file_data.splitlines() diff --git a/tests/unit/modules/test_network.py b/tests/unit/modules/test_network.py index 98a03c4ff2e..9a190fab453 100644 --- a/tests/unit/modules/test_network.py +++ b/tests/unit/modules/test_network.py @@ -20,8 +20,9 @@ from tests.support.mock import ( ) # Import Salt Libs -import salt.ext.six as six -import salt.utils +from salt.ext import six +import salt.utils.network +import salt.utils.path import salt.modules.network as network from salt.exceptions import CommandExecutionError if six.PY2: @@ -95,7 +96,8 @@ class NetworkTestCase(TestCase, LoaderModuleMockMixin): ''' with patch.dict(network.__grains__, {'kernel': 'Linux'}): with patch.object(network, '_netstat_linux', return_value='A'): - self.assertEqual(network.netstat(), 'A') + with patch.object(network, '_ss_linux', return_value='A'): + self.assertEqual(network.netstat(), 'A') with patch.dict(network.__grains__, {'kernel': 'OpenBSD'}): with patch.object(network, '_netstat_bsd', return_value='A'): @@ -117,7 +119,7 @@ class NetworkTestCase(TestCase, LoaderModuleMockMixin): ''' Test for Performs a traceroute to a 3rd party host ''' - with patch.object(salt.utils, 'which', side_effect=[False, True]): + with patch.object(salt.utils.path, 'which', side_effect=[False, True]): self.assertListEqual(network.traceroute('host'), []) with patch.object(salt.utils.network, 'sanitize_host', @@ -130,7 +132,7 @@ class NetworkTestCase(TestCase, LoaderModuleMockMixin): ''' Test for Performs a DNS lookup with dig ''' - with patch('salt.utils.which', MagicMock(return_value='dig')), \ + with patch('salt.utils.path.which', MagicMock(return_value='dig')), \ patch.object(salt.utils.network, 'sanitize_host', return_value='A'), \ patch.dict(network.__salt__, {'cmd.run': @@ -144,7 +146,7 @@ class NetworkTestCase(TestCase, LoaderModuleMockMixin): with patch.dict(network.__salt__, {'cmd.run': MagicMock(return_value='A,B,C,D\nE,F,G,H\n')}), \ - patch('salt.utils.which', MagicMock(return_value='')): + patch('salt.utils.path.which', MagicMock(return_value='')): self.assertDictEqual(network.arp(), {}) def test_interfaces(self): @@ -229,11 +231,11 @@ class NetworkTestCase(TestCase, LoaderModuleMockMixin): ''' self.assertFalse(network.mod_hostname(None)) - with patch.object(salt.utils, 'which', return_value='hostname'): + with patch.object(salt.utils.path, 'which', return_value='hostname'): with patch.dict(network.__salt__, {'cmd.run': MagicMock(return_value=None)}): file_d = '\n'.join(['#', 'A B C D,E,F G H']) - with patch('salt.utils.fopen', mock_open(read_data=file_d), + with patch('salt.utils.files.fopen', mock_open(read_data=file_d), create=True) as mfi: mfi.return_value.__iter__.return_value = file_d.splitlines() with patch.dict(network.__grains__, {'os_family': 'A'}): @@ -340,10 +342,12 @@ class NetworkTestCase(TestCase, LoaderModuleMockMixin): with patch.dict(network.__grains__, {'kernel': 'Linux'}): with patch.object(network, '_netstat_route_linux', side_effect=['A', [{'addr_family': 'inet'}]]): - self.assertEqual(network.routes(None), 'A') + with patch.object(network, '_ip_route_linux', + side_effect=['A', [{'addr_family': 'inet'}]]): + self.assertEqual(network.routes(None), 'A') - self.assertListEqual(network.routes('inet'), - [{'addr_family': 'inet'}]) + self.assertListEqual(network.routes('inet'), + [{'addr_family': 'inet'}]) def test_default_route(self): ''' diff --git a/tests/unit/modules/test_nfs3.py b/tests/unit/modules/test_nfs3.py index 737d73c0bc8..36b82881411 100644 --- a/tests/unit/modules/test_nfs3.py +++ b/tests/unit/modules/test_nfs3.py @@ -33,11 +33,11 @@ class NfsTestCase(TestCase, LoaderModuleMockMixin): Test for List configured exports ''' file_d = '\n'.join(['A B1(23']) - with patch('salt.utils.fopen', + with patch('salt.utils.files.fopen', mock_open(read_data=file_d), create=True) as mfi: mfi.return_value.__iter__.return_value = file_d.splitlines() self.assertDictEqual(nfs3.list_exports(), - {'A': [{'hosts': ['B1'], 'options': ['23']}]}) + {'A': [{'hosts': 'B1', 'options': ['23']}]}) def test_del_export(self): ''' diff --git a/tests/unit/modules/test_nftables.py b/tests/unit/modules/test_nftables.py index cad979e8a43..ecbe72c091a 100644 --- a/tests/unit/modules/test_nftables.py +++ b/tests/unit/modules/test_nftables.py @@ -19,7 +19,7 @@ from tests.support.mock import ( # Import Salt Libs import salt.modules.nftables as nftables -import salt.utils +import salt.utils.files from salt.exceptions import CommandExecutionError @@ -79,7 +79,7 @@ class NftablesTestCase(TestCase, LoaderModuleMockMixin): Test if it return a data structure of the rules in the conf file ''' with patch.dict(nftables.__grains__, {'os_family': 'Debian'}): - with patch.object(salt.utils, 'fopen', MagicMock(mock_open())): + with patch.object(salt.utils.files, 'fopen', MagicMock(mock_open())): self.assertListEqual(nftables.get_saved_rules(), []) # 'get_rules' function tests: 1 @@ -105,10 +105,10 @@ class NftablesTestCase(TestCase, LoaderModuleMockMixin): with patch.dict(nftables.__grains__, {'os_family': 'Debian'}): mock = MagicMock(return_value=False) with patch.dict(nftables.__salt__, {'cmd.run': mock}): - with patch.object(salt.utils, 'fopen', MagicMock(mock_open())): + with patch.object(salt.utils.files, 'fopen', MagicMock(mock_open())): self.assertEqual(nftables.save(), '#! nft -f\n\n') - with patch.object(salt.utils, 'fopen', + with patch.object(salt.utils.files, 'fopen', MagicMock(side_effect=IOError)): self.assertRaises(CommandExecutionError, nftables.save) diff --git a/tests/unit/modules/test_nginx.py b/tests/unit/modules/test_nginx.py index daf3590a602..c28147cc250 100644 --- a/tests/unit/modules/test_nginx.py +++ b/tests/unit/modules/test_nginx.py @@ -30,7 +30,7 @@ class MockUrllibStatus(object): class NginxTestCase(TestCase, LoaderModuleMockMixin): def setup_loader_modules(self): - patcher = patch('salt.utils.which', Mock(return_value='/usr/bin/nginx')) + patcher = patch('salt.utils.path.which', Mock(return_value='/usr/bin/nginx')) patcher.start() self.addCleanup(patcher.stop) return {nginx: {'_urlopen': Mock(return_value=MockUrllibStatus())}} diff --git a/tests/unit/modules/test_openscap.py b/tests/unit/modules/test_openscap.py index 5f393bddd22..0bb330c8042 100644 --- a/tests/unit/modules/test_openscap.py +++ b/tests/unit/modules/test_openscap.py @@ -18,7 +18,7 @@ from tests.support.mock import ( ) # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six @skipIf(NO_MOCK, NO_MOCK_REASON) diff --git a/tests/unit/modules/test_openstack_config.py b/tests/unit/modules/test_openstack_config.py index 493f11e97ae..c14a5513c3c 100644 --- a/tests/unit/modules/test_openstack_config.py +++ b/tests/unit/modules/test_openstack_config.py @@ -27,7 +27,7 @@ class OpenstackConfigTestCase(TestCase, LoaderModuleMockMixin): Test cases for salt.modules.openstack_config ''' def setup_loader_modules(self): - patcher = patch('salt.utils.which', MagicMock(return_value=True)) + patcher = patch('salt.utils.path.which', MagicMock(return_value=True)) patcher.start() self.addCleanup(patcher.stop) return {openstack_config: {}} diff --git a/tests/unit/modules/test_pam.py b/tests/unit/modules/test_pam.py index 9cb2bc4577e..05dcfdb2cce 100644 --- a/tests/unit/modules/test_pam.py +++ b/tests/unit/modules/test_pam.py @@ -34,7 +34,8 @@ class PamTestCase(TestCase): ''' Test if the parsing function works ''' - with patch('salt.utils.fopen', mock_open(read_data=MOCK_FILE)): + with patch('os.path.exists', return_value=True), \ + patch('salt.utils.files.fopen', mock_open(read_data=MOCK_FILE)): self.assertListEqual(pam.read_file('/etc/pam.d/login'), [{'arguments': [], 'control_flag': 'ok', 'interface': 'ok', 'module': 'ignore'}]) diff --git a/tests/unit/modules/test_parallels.py b/tests/unit/modules/test_parallels.py index 369cdf9d61c..302a7e89301 100644 --- a/tests/unit/modules/test_parallels.py +++ b/tests/unit/modules/test_parallels.py @@ -14,7 +14,7 @@ from tests.support.unit import TestCase, skipIf from tests.support.mock import MagicMock, patch, NO_MOCK, NO_MOCK_REASON # Import third party libs -import salt.ext.six as six +from salt.ext import six @skipIf(NO_MOCK, NO_MOCK_REASON) @@ -33,7 +33,7 @@ class ParallelsTestCase(TestCase, LoaderModuleMockMixin): mock_false = MagicMock(return_value=False) # Validate false return - with patch('salt.utils.which', mock_false): + with patch('salt.utils.path.which', mock_false): ret = parallels.__virtual__() self.assertTrue(isinstance(ret, tuple)) self.assertEqual(len(ret), 2) @@ -41,7 +41,7 @@ class ParallelsTestCase(TestCase, LoaderModuleMockMixin): self.assertTrue(isinstance(ret[1], six.string_types)) # Validate true return - with patch('salt.utils.which', mock_true): + with patch('salt.utils.path.which', mock_true): ret = parallels.__virtual__() self.assertTrue(ret) self.assertEqual(ret, 'parallels') diff --git a/tests/unit/modules/test_parted.py b/tests/unit/modules/test_parted.py index 991c6787a2f..6accff745a1 100644 --- a/tests/unit/modules/test_parted.py +++ b/tests/unit/modules/test_parted.py @@ -16,7 +16,7 @@ from tests.support.mixins import LoaderModuleMockMixin from tests.support.unit import skipIf, TestCase from tests.support.mock import NO_MOCK, NO_MOCK_REASON, MagicMock, patch -# Import salt libs +# Import Salt libs from salt.exceptions import CommandExecutionError import salt.modules.parted as parted @@ -41,38 +41,50 @@ class PartedTestCase(TestCase, LoaderModuleMockMixin): # Test __virtual__ function for module registration def test_virtual_bails_on_windows(self): - '''If running windows, __virtual__ shouldn't register module''' - with patch('salt.utils.is_windows', lambda: True): + ''' + If running windows, __virtual__ shouldn't register module + ''' + with patch('salt.utils.platform.is_windows', lambda: True): ret = parted.__virtual__() err = (False, 'The parted execution module failed to load Windows systems are not supported.') self.assertEqual(err, ret) def test_virtual_bails_without_parted(self): - '''If parted not in PATH, __virtual__ shouldn't register module''' - with patch('salt.utils.which', lambda exe: not exe == "parted"): + ''' + If parted not in PATH, __virtual__ shouldn't register module + ''' + with patch('salt.utils.path.which', lambda exe: not exe == "parted"),\ + patch('salt.utils.platform.is_windows', return_value=False): ret = parted.__virtual__() err = (False, 'The parted execution module failed to load parted binary is not in the path.') self.assertEqual(err, ret) def test_virtual_bails_without_lsblk(self): - '''If lsblk not in PATH, __virtual__ shouldn't register module''' - with patch('salt.utils.which', lambda exe: not exe == "lsblk"): + ''' + If lsblk not in PATH, __virtual__ shouldn't register module + ''' + with patch('salt.utils.path.which', lambda exe: not exe == "lsblk"),\ + patch('salt.utils.platform.is_windows', return_value=False): ret = parted.__virtual__() err = (False, 'The parted execution module failed to load lsblk binary is not in the path.') self.assertEqual(err, ret) def test_virtual_bails_without_partprobe(self): - '''If partprobe not in PATH, __virtual__ shouldn't register module''' - with patch('salt.utils.which', lambda exe: not exe == "partprobe"): + ''' + If partprobe not in PATH, __virtual__ shouldn't register module + ''' + with patch('salt.utils.path.which', lambda exe: not exe == "partprobe"),\ + patch('salt.utils.platform.is_windows', return_value=False): ret = parted.__virtual__() err = (False, 'The parted execution module failed to load partprobe binary is not in the path.') self.assertEqual(err, ret) def test_virtual(self): - '''On expected platform with correct utils in PATH, register - "partition" module''' - with patch('salt.utils.is_windows', lambda: False), \ - patch('salt.utils.which', lambda exe: exe in ('parted', 'lsblk', 'partprobe')): + ''' + On expected platform with correct utils in PATH, register "partition" module + ''' + with patch('salt.utils.platform.is_windows', lambda: False), \ + patch('salt.utils.path.which', lambda exe: exe in ('parted', 'lsblk', 'partprobe')): ret = parted.__virtual__() expect = 'partition' self.assertEqual(ret, expect) diff --git a/tests/unit/modules/test_pillar.py b/tests/unit/modules/test_pillar.py index 85225f05b71..f952e375d0a 100644 --- a/tests/unit/modules/test_pillar.py +++ b/tests/unit/modules/test_pillar.py @@ -14,7 +14,7 @@ from tests.support.mock import ( ) # Import Salt libs -import salt.ext.six as six +from salt.ext import six from salt.utils.odict import OrderedDict import salt.modules.pillar as pillarmod diff --git a/tests/unit/modules/test_pip.py b/tests/unit/modules/test_pip.py index 88b8f1a23ed..7d7735ff708 100644 --- a/tests/unit/modules/test_pip.py +++ b/tests/unit/modules/test_pip.py @@ -10,6 +10,7 @@ from tests.support.unit import skipIf, TestCase from tests.support.mock import NO_MOCK, NO_MOCK_REASON, MagicMock, patch # Import salt libs +import salt.utils.platform import salt.modules.pip as pip from salt.exceptions import CommandExecutionError @@ -289,17 +290,23 @@ class PipTestCase(TestCase, LoaderModuleMockMixin): mock_path.isdir.return_value = True pkg = 'mock' - venv_path = '/test_env' def join(*args): - return '/'.join(args) + return os.sep.join(args) + mock_path.join = join mock = MagicMock(return_value={'retcode': 0, 'stdout': ''}) with patch.dict(pip.__salt__, {'cmd.run_all': mock}): + if salt.utils.platform.is_windows(): + venv_path = 'c:\\test_env' + bin_path = os.path.join(venv_path, 'Scripts', 'pip.exe').encode('string-escape') + else: + venv_path = '/test_env' + bin_path = os.path.join(venv_path, 'bin', 'pip') pip.install(pkg, bin_env=venv_path) mock.assert_called_once_with( - [os.path.join(venv_path, 'bin', 'pip'), 'install', pkg], - env={'VIRTUAL_ENV': '/test_env'}, + [bin_path, 'install', pkg], + env={'VIRTUAL_ENV': venv_path}, saltenv='base', runas=None, use_vt=False, diff --git a/tests/unit/modules/test_pkg_resource.py b/tests/unit/modules/test_pkg_resource.py index 2e51fab6f5f..e00d8cda771 100644 --- a/tests/unit/modules/test_pkg_resource.py +++ b/tests/unit/modules/test_pkg_resource.py @@ -20,7 +20,7 @@ from tests.support.mock import ( # Import Salt Libs import salt.utils import salt.modules.pkg_resource as pkg_resource -import salt.ext.six as six +from salt.ext import six @skipIf(NO_MOCK, NO_MOCK_REASON) diff --git a/tests/unit/modules/test_portage_config.py b/tests/unit/modules/test_portage_config.py index a0d01199541..02a76b63d2e 100644 --- a/tests/unit/modules/test_portage_config.py +++ b/tests/unit/modules/test_portage_config.py @@ -7,11 +7,14 @@ ''' # Import Python libs from __future__ import absolute_import +import re # Import Salt Testing libs from tests.support.mixins import LoaderModuleMockMixin from tests.support.unit import skipIf, TestCase from tests.support.mock import NO_MOCK, NO_MOCK_REASON, MagicMock, patch +from tests.support.paths import TMP +import salt.utils.files # Import salt libs import salt.modules.portage_config as portage_config @@ -19,27 +22,101 @@ import salt.modules.portage_config as portage_config @skipIf(NO_MOCK, NO_MOCK_REASON) class PortageConfigTestCase(TestCase, LoaderModuleMockMixin): - class DummyAtom(object): - def __init__(self, atom): - self.cp, self.repo = atom.split("::") if "::" in atom else (atom, None) + def __init__(self): + self.cp = None + self.repo = None + + def __call__(self, atom, *_, **__): + if atom == '#' or isinstance(atom, MagicMock): + self.repo = None + self.cp = None + return self + + # extract (and remove) repo + atom, self.repo = atom.split('::') if '::' in atom else (atom, None) + + # remove '>, >=, <=, =, ~' etc. + atom = re.sub(r'[<>~+=]', '', atom) + # remove slots + atom = re.sub(r':[0-9][^:]*', '', atom) + # remove version + atom = re.sub(r'-[0-9][\.0-9]*', '', atom) + + self.cp = atom + return self def setup_loader_modules(self): - self.portage = MagicMock() - self.addCleanup(delattr, self, 'portage') - return {portage_config: {'portage': self.portage}} + try: + import portage + return {} + except ImportError: + dummy_atom = self.DummyAtom() + self.portage = MagicMock() + self.portage.dep.Atom = MagicMock(side_effect=dummy_atom) + self.portage.dep_getkey = MagicMock(side_effect=lambda x: dummy_atom(x).cp) + self.portage.exception.InvalidAtom = Exception + self.addCleanup(delattr, self, 'portage') + return {portage_config: {'portage': self.portage}} def test_get_config_file_wildcards(self): pairs = [ ('*/*::repo', '/etc/portage/package.mask/repo'), ('*/pkg::repo', '/etc/portage/package.mask/pkg'), - ('cat/*', '/etc/portage/package.mask/cat'), + ('cat/*', '/etc/portage/package.mask/cat_'), ('cat/pkg', '/etc/portage/package.mask/cat/pkg'), ('cat/pkg::repo', '/etc/portage/package.mask/cat/pkg'), ] for (atom, expected) in pairs: - dummy_atom = self.DummyAtom(atom) - self.portage.dep.Atom = MagicMock(return_value=dummy_atom) - with patch.object(portage_config, '_p_to_cp', MagicMock(return_value=dummy_atom.cp)): - self.assertEqual(portage_config._get_config_file('mask', atom), expected) + self.assertEqual(portage_config._get_config_file('mask', atom), expected) + + def test_enforce_nice_config(self): + atoms = [ + ('*/*::repo', 'repo'), + ('*/pkg1::repo', 'pkg1'), + ('cat/*', 'cat_'), + ('cat/pkg2', 'cat/pkg2'), + ('cat/pkg3::repo', 'cat/pkg3'), + ('cat/pkg5-0.0.0.0:0', 'cat/pkg5'), + ('>cat/pkg6-0.0.0.0:0::repo', 'cat/pkg6'), + ('<=cat/pkg7-0.0.0.0', 'cat/pkg7'), + ('=cat/pkg8-0.0.0.0', 'cat/pkg8'), + ] + + supported = [ + ('accept_keywords', ['~amd64']), + ('env', ['glibc.conf']), + ('license', ['LICENCE1', 'LICENCE2']), + ('mask', ['']), + ('properties', ['* -interactive']), + ('unmask', ['']), + ('use', ['apple', '-banana', 'ananas', 'orange']), + ] + + base_path = TMP + '/package.{0}' + + def make_line(atom, addition): + return atom + (' ' + addition if addition != '' else '') + '\n' + + for typ, additions in supported: + path = base_path.format(typ) + with salt.utils.files.fopen(path, 'a') as fh: + for atom, _ in atoms: + for addition in additions: + line = make_line(atom, addition) + fh.write('# comment for: ' + line) + fh.write(line) + + with patch.object(portage_config, 'BASE_PATH', base_path): + with patch.object(portage_config, '_merge_flags', lambda l1, l2, _: list(set(l1 + l2))): + portage_config.enforce_nice_config() + + for typ, additions in supported: + for atom, file_name in atoms: + with salt.utils.files.fopen(base_path.format(typ) + "/" + file_name, 'r') as fh: + for line in fh: + self.assertTrue(atom in line, msg="'{}' not in '{}'".format(addition, line)) + for addition in additions: + self.assertTrue(addition in line, msg="'{}' not in '{}'".format(addition, line)) diff --git a/tests/unit/modules/test_postgres.py b/tests/unit/modules/test_postgres.py index 326c2f1a683..2ad28c18348 100644 --- a/tests/unit/modules/test_postgres.py +++ b/tests/unit/modules/test_postgres.py @@ -2,6 +2,7 @@ # Import python libs from __future__ import absolute_import, print_function +import datetime import re # Import Salt Testing libs @@ -53,7 +54,7 @@ test_privileges_list_group_csv = ( @skipIf(NO_MOCK, NO_MOCK_REASON) class PostgresTestCase(TestCase, LoaderModuleMockMixin): def setup_loader_modules(self): - patcher = patch('salt.utils.which', Mock(return_value='/usr/bin/pgsql')) + patcher = patch('salt.utils.path.which', Mock(return_value='/usr/bin/pgsql')) patcher.start() self.addCleanup(patcher.stop) return { @@ -335,6 +336,7 @@ class PostgresTestCase(TestCase, LoaderModuleMockMixin): superuser=False, replication=False, rolepassword='test_role_pass', + valid_until='2042-07-01', groups='test_groups', runas='foo' ) @@ -345,9 +347,10 @@ class PostgresTestCase(TestCase, LoaderModuleMockMixin): call = postgres._run_psql.call_args[0][0][14] self.assertTrue(re.match('CREATE ROLE "testuser"', call)) for i in ( - 'INHERIT NOCREATEDB NOCREATEROLE ' - 'NOSUPERUSER NOREPLICATION LOGIN UNENCRYPTED PASSWORD' - ).split(): + 'INHERIT', 'NOCREATEDB', 'NOCREATEROLE', 'NOSUPERUSER', + 'NOREPLICATION', 'LOGIN', 'UNENCRYPTED', 'PASSWORD', + 'VALID UNTIL', + ): self.assertTrue(i in call, '{0} not in {1}'.format(i, call)) def test_user_exists(self): @@ -368,6 +371,7 @@ class PostgresTestCase(TestCase, LoaderModuleMockMixin): 'replication': None, 'password': 'test_password', 'connections': '-1', + 'expiry time': '', 'defaults variables': None }])): ret = postgres.user_exists( @@ -398,6 +402,7 @@ class PostgresTestCase(TestCase, LoaderModuleMockMixin): 'can login': 't', 'replication': None, 'connections': '-1', + 'expiry time': '2017-08-16 08:57:46', 'defaults variables': None }])): ret = postgres.user_list( @@ -416,7 +421,8 @@ class PostgresTestCase(TestCase, LoaderModuleMockMixin): 'can create roles': True, 'connections': None, 'replication': None, - 'expiry time': None, + 'expiry time': datetime.datetime( + 2017, 8, 16, 8, 57, 46), 'can login': True, 'can update system catalogs': True, 'groups': [], @@ -464,6 +470,7 @@ class PostgresTestCase(TestCase, LoaderModuleMockMixin): login=True, replication=False, rolepassword='test_role_pass', + valid_until='2017-07-01', groups='test_groups', runas='foo' ) @@ -475,7 +482,8 @@ class PostgresTestCase(TestCase, LoaderModuleMockMixin): re.match( 'ALTER ROLE "test_username" WITH INHERIT NOCREATEDB ' 'NOCREATEROLE NOREPLICATION LOGIN ' - 'UNENCRYPTED PASSWORD [\'"]{0,5}test_role_pass[\'"]{0,5};' + 'UNENCRYPTED PASSWORD [\'"]{0,5}test_role_pass[\'"]{0,5} ' + 'VALID UNTIL \'2017-07-01\';' ' GRANT "test_groups" TO "test_username"', postgres._run_psql.call_args[0][0][14] ) diff --git a/tests/unit/modules/test_poudriere.py b/tests/unit/modules/test_poudriere.py index 52e8f322e37..8b839aad152 100644 --- a/tests/unit/modules/test_poudriere.py +++ b/tests/unit/modules/test_poudriere.py @@ -76,7 +76,7 @@ class PoudriereTestCase(TestCase, LoaderModuleMockMixin): ''' mock = MagicMock(return_value='/tmp/salt') with patch.dict(poudriere.__salt__, {'config.option': mock}), \ - patch('salt.utils.fopen', mock_open()), \ + patch('salt.utils.files.fopen', mock_open()), \ patch.object(poudriere, '_check_config_exists', MagicMock(side_effect=[True, False])): self.assertDictEqual(poudriere.parse_config(), {}) diff --git a/tests/unit/modules/test_puppet.py b/tests/unit/modules/test_puppet.py index bb8d770d062..5a23ba77447 100644 --- a/tests/unit/modules/test_puppet.py +++ b/tests/unit/modules/test_puppet.py @@ -19,7 +19,8 @@ from tests.support.mock import ( ) # Import Salt Libs -import salt.utils +import salt.utils.args +import salt.utils.files import salt.modules.puppet as puppet from salt.exceptions import CommandExecutionError @@ -37,7 +38,7 @@ class PuppetTestCase(TestCase, LoaderModuleMockMixin): Test to execute a puppet run ''' mock = MagicMock(return_value={"A": "B"}) - with patch.object(salt.utils, 'clean_kwargs', mock): + with patch.object(salt.utils.args, 'clean_kwargs', mock): mock = MagicMock(return_value={'retcode': 0}) mock_lst = MagicMock(return_value=[]) with patch.dict(puppet.__salt__, {'cmd.run_all': mock, @@ -80,11 +81,11 @@ class PuppetTestCase(TestCase, LoaderModuleMockMixin): with patch.object(os.path, 'isfile', mock): self.assertFalse(puppet.disable()) - with patch('salt.utils.fopen', mock_open()): + with patch('salt.utils.files.fopen', mock_open()): self.assertTrue(puppet.disable()) try: - with patch('salt.utils.fopen', mock_open()) as m_open: + with patch('salt.utils.files.fopen', mock_open()) as m_open: m_open.side_effect = IOError(13, 'Permission denied:', '/file') self.assertRaises(CommandExecutionError, puppet.disable) except StopIteration: @@ -103,7 +104,7 @@ class PuppetTestCase(TestCase, LoaderModuleMockMixin): mock = MagicMock(side_effect=[False, True]) with patch.object(os.path, 'isfile', mock): - with patch('salt.utils.fopen', mock_open(read_data="1")): + with patch('salt.utils.files.fopen', mock_open(read_data="1")): mock = MagicMock(return_value=True) with patch.object(os, 'kill', mock): self.assertEqual(puppet.status(), @@ -111,21 +112,21 @@ class PuppetTestCase(TestCase, LoaderModuleMockMixin): mock = MagicMock(side_effect=[False, True]) with patch.object(os.path, 'isfile', mock): - with patch('salt.utils.fopen', mock_open()): + with patch('salt.utils.files.fopen', mock_open()): mock = MagicMock(return_value=True) with patch.object(os, 'kill', mock): self.assertEqual(puppet.status(), "Stale lockfile") mock = MagicMock(side_effect=[False, False, True]) with patch.object(os.path, 'isfile', mock): - with patch('salt.utils.fopen', mock_open(read_data="1")): + with patch('salt.utils.files.fopen', mock_open(read_data="1")): mock = MagicMock(return_value=True) with patch.object(os, 'kill', mock): self.assertEqual(puppet.status(), "Idle daemon") mock = MagicMock(side_effect=[False, False, True]) with patch.object(os.path, 'isfile', mock): - with patch('salt.utils.fopen', mock_open()): + with patch('salt.utils.files.fopen', mock_open()): mock = MagicMock(return_value=True) with patch.object(os, 'kill', mock): self.assertEqual(puppet.status(), "Stale pidfile") @@ -140,11 +141,11 @@ class PuppetTestCase(TestCase, LoaderModuleMockMixin): ''' mock_lst = MagicMock(return_value=[]) with patch.dict(puppet.__salt__, {'cmd.run': mock_lst}): - with patch('salt.utils.fopen', + with patch('salt.utils.files.fopen', mock_open(read_data="resources: 1")): self.assertDictEqual(puppet.summary(), {'resources': 1}) - with patch('salt.utils.fopen', mock_open()) as m_open: + with patch('salt.utils.files.fopen', mock_open()) as m_open: m_open.side_effect = IOError(13, 'Permission denied:', '/file') self.assertRaises(CommandExecutionError, puppet.summary) diff --git a/tests/unit/modules/test_pw_group.py b/tests/unit/modules/test_pw_group.py index 3d21bbd43c3..2cfc5f32d2a 100644 --- a/tests/unit/modules/test_pw_group.py +++ b/tests/unit/modules/test_pw_group.py @@ -18,6 +18,7 @@ from tests.support.mock import ( # Import Salt Libs import salt.modules.pw_group as pw_group +import salt.utils.platform @skipIf(NO_MOCK, NO_MOCK_REASON) @@ -44,6 +45,7 @@ class PwGroupTestCase(TestCase, LoaderModuleMockMixin): with patch.dict(pw_group.__salt__, {'cmd.run_all': mock}): self.assertTrue(pw_group.delete('a')) + @skipIf(salt.utils.platform.is_windows(), 'grp not available on Windows') def test_info(self): ''' Tests to return information about a group @@ -57,6 +59,7 @@ class PwGroupTestCase(TestCase, LoaderModuleMockMixin): with patch.dict(pw_group.grinfo, mock): self.assertDictEqual(pw_group.info('name'), {}) + @skipIf(salt.utils.platform.is_windows(), 'grp not available on Windows') def test_getent(self): ''' Tests for return info on all groups diff --git a/tests/unit/modules/test_qemu_nbd.py b/tests/unit/modules/test_qemu_nbd.py index ec6ec845870..59361c00502 100644 --- a/tests/unit/modules/test_qemu_nbd.py +++ b/tests/unit/modules/test_qemu_nbd.py @@ -80,15 +80,14 @@ class QemuNbdTestCase(TestCase, LoaderModuleMockMixin): with patch.dict(qemu_nbd.__salt__, {'cmd.run': mock}): self.assertEqual(qemu_nbd.init('/srv/image.qcow2'), '') - with patch.object(os.path, 'isfile', mock): - with patch.object(glob, 'glob', - MagicMock(return_value=['/dev/nbd0'])): - with patch.dict(qemu_nbd.__salt__, - {'cmd.run': mock, - 'mount.mount': mock, - 'cmd.retcode': MagicMock(side_effect=[1, 0])}): - self.assertDictEqual(qemu_nbd.init('/srv/image.qcow2'), - {'{0}/nbd/nbd0/nbd0'.format(tempfile.gettempdir()): '/dev/nbd0'}) + with patch.object(os.path, 'isfile', mock),\ + patch.object(glob, 'glob', MagicMock(return_value=['/dev/nbd0'])),\ + patch.dict(qemu_nbd.__salt__, + {'cmd.run': mock, + 'mount.mount': mock, + 'cmd.retcode': MagicMock(side_effect=[1, 0])}): + expected = {os.sep.join([tempfile.gettempdir(), 'nbd', 'nbd0', 'nbd0']): '/dev/nbd0'} + self.assertDictEqual(qemu_nbd.init('/srv/image.qcow2'), expected) # 'clear' function tests: 1 diff --git a/tests/unit/modules/test_reg_win.py b/tests/unit/modules/test_reg_win.py index 960c00e5d06..e83d89b127e 100644 --- a/tests/unit/modules/test_reg_win.py +++ b/tests/unit/modules/test_reg_win.py @@ -27,7 +27,7 @@ except ImportError: PY2 = sys.version_info[0] == 2 # The following used to make sure we are not # testing already existing data -# Note strftime retunrns a str, so we need to make it unicode +# Note strftime returns a str, so we need to make it unicode TIMEINT = int(time.time()) if PY2: diff --git a/tests/unit/modules/test_rh_ip.py b/tests/unit/modules/test_rh_ip.py index 08b3367a73f..f115ccf5baa 100644 --- a/tests/unit/modules/test_rh_ip.py +++ b/tests/unit/modules/test_rh_ip.py @@ -58,7 +58,7 @@ class RhipTestCase(TestCase, LoaderModuleMockMixin): ''' Test to build an interface script for a network interface. ''' - with patch.dict(rh_ip.__grains__, {'os': 'Fedora'}): + with patch.dict(rh_ip.__grains__, {'os': 'Fedora', 'osmajorrelease': 26}): with patch.object(rh_ip, '_raise_error_iface', return_value=None): self.assertRaises(AttributeError, diff --git a/tests/unit/modules/test_saltcheck.py b/tests/unit/modules/test_saltcheck.py new file mode 100644 index 00000000000..5907af17efe --- /dev/null +++ b/tests/unit/modules/test_saltcheck.py @@ -0,0 +1,335 @@ +# -*- coding: utf-8 -*- +'''Unit test for saltcheck execution module''' + +# Import Python libs +from __future__ import absolute_import +import os.path + +try: + import salt.modules.saltcheck as saltcheck + import salt.config + import salt.syspaths as syspaths +except: + raise + +# Import Salt Testing Libs +try: + from tests.support.mixins import LoaderModuleMockMixin + from tests.support.unit import skipIf, TestCase + from tests.support.mock import ( + MagicMock, + patch, + NO_MOCK, + NO_MOCK_REASON + ) +except: + raise + + +@skipIf(NO_MOCK, NO_MOCK_REASON) +class LinuxSysctlTestCase(TestCase, LoaderModuleMockMixin): + ''' + TestCase for salt.modules.saltcheck module + ''' + + def setup_loader_modules(self): + # Setting the environment to be local + local_opts = salt.config.minion_config( + os.path.join(syspaths.CONFIG_DIR, u'minion')) + local_opts['file_client'] = 'local' + patcher = patch('salt.config.minion_config', + MagicMock(return_value=local_opts)) + patcher.start() + self.addCleanup(patcher.stop) + return {saltcheck: {}} + + def test_call_salt_command(self): + '''test simple test.echo module''' + with patch.dict(saltcheck.__salt__, {'config.get': MagicMock(return_value=True), + 'sys.list_modules': MagicMock(return_value=['module1']), + 'cp.cache_master': MagicMock(return_value=[True]) + }): + sc_instance = saltcheck.SaltCheck() + returned = sc_instance.call_salt_command(fun="test.echo", args=['hello'], kwargs=None) + self.assertEqual(returned, 'hello') + + def test_update_master_cache(self): + '''test master cache''' + self.assertTrue(saltcheck.update_master_cache) + + def test_call_salt_command2(self): + '''test simple test.echo module again''' + with patch.dict(saltcheck.__salt__, {'config.get': MagicMock(return_value=True), + 'sys.list_modules': MagicMock(return_value=['module1']), + 'cp.cache_master': MagicMock(return_value=[True]) + }): + sc_instance = saltcheck.SaltCheck() + returned = sc_instance.call_salt_command(fun="test.echo", args=['hello'], kwargs=None) + self.assertNotEqual(returned, 'not-hello') + + def test__assert_equal1(self): + '''test''' + with patch.dict(saltcheck.__salt__, {'config.get': MagicMock(return_value=True), + 'cp.cache_master': MagicMock(return_value=[True]) + }): + sc_instance = saltcheck.SaltCheck() + aaa = {'a': 1, 'b': 2} + bbb = {'a': 1, 'b': 2} + mybool = sc_instance._SaltCheck__assert_equal(aaa, bbb) + self.assertTrue(mybool) + + def test__assert_equal2(self): + '''test''' + with patch.dict(saltcheck.__salt__, {'config.get': MagicMock(return_value=True), + 'cp.cache_master': MagicMock(return_value=[True]) + }): + sc_instance = saltcheck.SaltCheck() + mybool = sc_instance._SaltCheck__assert_equal(False, True) + self.assertNotEqual(mybool, True) + + def test__assert_not_equal1(self): + '''test''' + with patch.dict(saltcheck.__salt__, {'config.get': MagicMock(return_value=True), + 'cp.cache_master': MagicMock(return_value=[True]) + }): + sc_instance = saltcheck.SaltCheck() + aaa = {'a': 1, 'b': 2} + bbb = {'a': 1, 'b': 2, 'c': 3} + mybool = sc_instance._SaltCheck__assert_not_equal(aaa, bbb) + self.assertTrue(mybool) + + def test__assert_not_equal2(self): + '''test''' + with patch.dict(saltcheck.__salt__, {'config.get': MagicMock(return_value=True), + 'cp.cache_master': MagicMock(return_value=[True]) + }): + sc_instance = saltcheck.SaltCheck() + aaa = {'a': 1, 'b': 2} + bbb = {'a': 1, 'b': 2} + mybool = sc_instance._SaltCheck__assert_not_equal(aaa, bbb) + self.assertNotEqual(mybool, True) + + def test__assert_true1(self): + '''test''' + with patch.dict(saltcheck.__salt__, {'config.get': MagicMock(return_value=True), + 'cp.cache_master': MagicMock(return_value=[True]) + }): + sc_instance = saltcheck.SaltCheck() + mybool = sc_instance._SaltCheck__assert_equal(True, True) + self.assertTrue(mybool) + + def test__assert_true2(self): + '''test''' + with patch.dict(saltcheck.__salt__, {'config.get': MagicMock(return_value=True), + 'cp.cache_master': MagicMock(return_value=[True]) + }): + sc_instance = saltcheck.SaltCheck() + mybool = sc_instance._SaltCheck__assert_equal(False, True) + self.assertNotEqual(mybool, True) + + def test__assert_false1(self): + '''test''' + with patch.dict(saltcheck.__salt__, {'config.get': MagicMock(return_value=True), + 'cp.cache_master': MagicMock(return_value=[True]) + }): + sc_instance = saltcheck.SaltCheck() + mybool = sc_instance._SaltCheck__assert_false(False) + self.assertTrue(mybool) + + def test__assert_false2(self): + '''test''' + with patch.dict(saltcheck.__salt__, {'config.get': MagicMock(return_value=True), + 'cp.cache_master': MagicMock(return_value=[True]) + }): + sc_instance = saltcheck.SaltCheck() + mybool = sc_instance._SaltCheck__assert_false(True) + self.assertNotEqual(mybool, True) + + def test__assert_in1(self): + '''test''' + with patch.dict(saltcheck.__salt__, {'config.get': MagicMock(return_value=True), + 'cp.cache_master': MagicMock(return_value=[True]) + }): + sc_instance = saltcheck.SaltCheck() + aaa = "bob" + mylist = ['alice', 'bob', 'charles', 'dana'] + mybool = sc_instance._SaltCheck__assert_in(aaa, mylist) + self.assertTrue(mybool, True) + + def test__assert_in2(self): + '''test''' + with patch.dict(saltcheck.__salt__, {'config.get': MagicMock(return_value=True), + 'cp.cache_master': MagicMock(return_value=[True]) + }): + sc_instance = saltcheck.SaltCheck() + aaa = "elaine" + mylist = ['alice', 'bob', 'charles', 'dana'] + mybool = sc_instance._SaltCheck__assert_in(aaa, mylist) + self.assertNotEqual(mybool, True) + + def test__assert_not_in1(self): + '''test''' + with patch.dict(saltcheck.__salt__, {'config.get': MagicMock(return_value=True), + 'cp.cache_master': MagicMock(return_value=[True]) + }): + sc_instance = saltcheck.SaltCheck() + aaa = "elaine" + mylist = ['alice', 'bob', 'charles', 'dana'] + mybool = sc_instance._SaltCheck__assert_not_in(aaa, mylist) + self.assertTrue(mybool, True) + + def test__assert_not_in2(self): + '''test''' + with patch.dict(saltcheck.__salt__, {'config.get': MagicMock(return_value=True), + 'cp.cache_master': MagicMock(return_value=[True]) + }): + sc_instance = saltcheck.SaltCheck() + aaa = "bob" + mylist = ['alice', 'bob', 'charles', 'dana'] + mybool = sc_instance._SaltCheck__assert_not_in(aaa, mylist) + self.assertNotEqual(mybool, True) + + def test__assert_greater1(self): + '''test''' + with patch.dict(saltcheck.__salt__, {'config.get': MagicMock(return_value=True), + 'cp.cache_master': MagicMock(return_value=[True]) + }): + sc_instance = saltcheck.SaltCheck() + aaa = 110 + bbb = 100 + mybool = sc_instance._SaltCheck__assert_greater(aaa, bbb) + self.assertTrue(mybool, True) + + def test__assert_greater2(self): + '''test''' + with patch.dict(saltcheck.__salt__, {'config.get': MagicMock(return_value=True), + 'cp.cache_master': MagicMock(return_value=[True]) + }): + sc_instance = saltcheck.SaltCheck() + aaa = 100 + bbb = 110 + mybool = sc_instance._SaltCheck__assert_greater(aaa, bbb) + self.assertNotEqual(mybool, True) + + def test__assert_greater3(self): + '''test''' + with patch.dict(saltcheck.__salt__, {'config.get': MagicMock(return_value=True), + 'cp.cache_master': MagicMock(return_value=[True]) + }): + sc_instance = saltcheck.SaltCheck() + aaa = 100 + bbb = 100 + mybool = sc_instance._SaltCheck__assert_greater(aaa, bbb) + self.assertNotEqual(mybool, True) + + def test__assert_greater_equal1(self): + '''test''' + with patch.dict(saltcheck.__salt__, {'config.get': MagicMock(return_value=True), + 'cp.cache_master': MagicMock(return_value=[True]) + }): + sc_instance = saltcheck.SaltCheck() + aaa = 110 + bbb = 100 + mybool = sc_instance._SaltCheck__assert_greater_equal(aaa, bbb) + self.assertTrue(mybool, True) + + def test__assert_greater_equal2(self): + '''test''' + with patch.dict(saltcheck.__salt__, {'config.get': MagicMock(return_value=True), + 'cp.cache_master': MagicMock(return_value=[True]) + }): + sc_instance = saltcheck.SaltCheck() + aaa = 100 + bbb = 110 + mybool = sc_instance._SaltCheck__assert_greater_equal(aaa, bbb) + self.assertNotEqual(mybool, True) + + def test__assert_greater_equal3(self): + '''test''' + with patch.dict(saltcheck.__salt__, {'config.get': MagicMock(return_value=True), + 'cp.cache_master': MagicMock(return_value=[True]) + }): + sc_instance = saltcheck.SaltCheck() + aaa = 100 + bbb = 100 + mybool = sc_instance._SaltCheck__assert_greater_equal(aaa, bbb) + self.assertEqual(mybool, 'Pass') + + def test__assert_less1(self): + '''test''' + with patch.dict(saltcheck.__salt__, {'config.get': MagicMock(return_value=True), + 'cp.cache_master': MagicMock(return_value=[True]) + }): + sc_instance = saltcheck.SaltCheck() + aaa = 99 + bbb = 100 + mybool = sc_instance._SaltCheck__assert_less(aaa, bbb) + self.assertTrue(mybool, True) + + def test__assert_less2(self): + '''test''' + with patch.dict(saltcheck.__salt__, {'config.get': MagicMock(return_value=True), + 'cp.cache_master': MagicMock(return_value=[True]) + }): + sc_instance = saltcheck.SaltCheck() + aaa = 110 + bbb = 99 + mybool = sc_instance._SaltCheck__assert_less(aaa, bbb) + self.assertNotEqual(mybool, True) + + def test__assert_less3(self): + '''test''' + with patch.dict(saltcheck.__salt__, {'config.get': MagicMock(return_value=True), + 'cp.cache_master': MagicMock(return_value=[True]) + }): + sc_instance = saltcheck.SaltCheck() + aaa = 100 + bbb = 100 + mybool = sc_instance._SaltCheck__assert_less(aaa, bbb) + self.assertNotEqual(mybool, True) + + def test__assert_less_equal1(self): + '''test''' + with patch.dict(saltcheck.__salt__, {'config.get': MagicMock(return_value=True), + 'cp.cache_master': MagicMock(return_value=[True]) + }): + sc_instance = saltcheck.SaltCheck() + aaa = 99 + bbb = 100 + mybool = sc_instance._SaltCheck__assert_less_equal(aaa, bbb) + self.assertTrue(mybool, True) + + def test__assert_less_equal2(self): + '''test''' + with patch.dict(saltcheck.__salt__, {'config.get': MagicMock(return_value=True), + 'cp.cache_master': MagicMock(return_value=[True]) + }): + sc_instance = saltcheck.SaltCheck() + aaa = 110 + bbb = 99 + mybool = sc_instance._SaltCheck__assert_less_equal(aaa, bbb) + self.assertNotEqual(mybool, True) + + def test__assert_less_equal3(self): + '''test''' + with patch.dict(saltcheck.__salt__, {'config.get': MagicMock(return_value=True), + 'cp.cache_master': MagicMock(return_value=[True]) + }): + sc_instance = saltcheck.SaltCheck() + aaa = 100 + bbb = 100 + mybool = sc_instance._SaltCheck__assert_less_equal(aaa, bbb) + self.assertEqual(mybool, 'Pass') + + def test_run_test_1(self): + '''test''' + with patch.dict(saltcheck.__salt__, {'config.get': MagicMock(return_value=True), + 'sys.list_modules': MagicMock(return_value=['test']), + 'sys.list_functions': MagicMock(return_value=['test.echo']), + 'cp.cache_master': MagicMock(return_value=[True])}): + returned = saltcheck.run_test(test={"module_and_function": "test.echo", + "assertion": "assertEqual", + "expected-return": "This works!", + "args": ["This works!"] + }) + self.assertEqual(returned, 'Pass') diff --git a/tests/unit/modules/test_scsi.py b/tests/unit/modules/test_scsi.py index b3827200551..dd327780f57 100644 --- a/tests/unit/modules/test_scsi.py +++ b/tests/unit/modules/test_scsi.py @@ -19,7 +19,7 @@ from tests.support.mock import ( # Import Salt Libs import salt.modules.scsi as scsi -import salt.utils +import salt.utils.path @skipIf(NO_MOCK, NO_MOCK_REASON) @@ -60,7 +60,7 @@ class ScsiTestCase(TestCase, LoaderModuleMockMixin): result_size['[0:0:0:0]']['size'] = '1.20TB' mock = MagicMock(return_value='/usr/bin/lsscsi') - with patch.object(salt.utils, 'which', mock): + with patch.object(salt.utils.path, 'which', mock): # get_size = True cmd_mock = MagicMock(return_value=lsscsi_size) @@ -77,7 +77,7 @@ class ScsiTestCase(TestCase, LoaderModuleMockMixin): self.assertDictEqual(scsi.ls_(get_size=False), result) mock = MagicMock(return_value=None) - with patch.object(salt.utils, 'which', mock): + with patch.object(salt.utils.path, 'which', mock): self.assertEqual(scsi.ls_(), 'scsi.ls not available - lsscsi command not found') def test_rescan_all(self): diff --git a/tests/unit/modules/test_seed.py b/tests/unit/modules/test_seed.py index 39be4a47cba..8a795889ba2 100644 --- a/tests/unit/modules/test_seed.py +++ b/tests/unit/modules/test_seed.py @@ -19,7 +19,7 @@ from tests.support.mock import ( # Import Salt Libs -import salt.utils +import salt.utils.files import salt.utils.odict import salt.modules.seed as seed @@ -39,7 +39,7 @@ class SeedTestCase(TestCase, LoaderModuleMockMixin): ddd['b'] = 'b' ddd['a'] = 'b' data = seed.mkconfig(ddd, approve_key=False) - with salt.utils.fopen(data['config']) as fic: + with salt.utils.files.fopen(data['config']) as fic: fdata = fic.read() self.assertEqual(fdata, 'b: b\na: b\nmaster: foo\n') @@ -47,14 +47,19 @@ class SeedTestCase(TestCase, LoaderModuleMockMixin): ''' Test to update and get the random script to a random place ''' - with patch.dict(seed.__salt__, - {'config.gather_bootstrap_script': MagicMock(return_value='BS_PATH/BS')}): - with patch.object(uuid, 'uuid4', return_value='UUID'): - with patch.object(os.path, 'exists', return_value=True): - with patch.object(os, 'chmod', return_value=None): - with patch.object(shutil, 'copy', return_value=None): - self.assertEqual(seed.prep_bootstrap('MPT'), ('MPT/tmp/UUID/BS', '/tmp/UUID')) - self.assertEqual(seed.prep_bootstrap('/MPT'), ('/MPT/tmp/UUID/BS', '/tmp/UUID')) + with patch.dict(seed.__salt__, {'config.gather_bootstrap_script': MagicMock(return_value=os.path.join('BS_PATH', 'BS'))}),\ + patch.object(uuid, 'uuid4', return_value='UUID'),\ + patch.object(os.path, 'exists', return_value=True),\ + patch.object(os, 'chmod', return_value=None),\ + patch.object(shutil, 'copy', return_value=None): + + expect = (os.path.join('MPT', 'tmp', 'UUID', 'BS'), + os.sep + os.path.join('tmp', 'UUID')) + self.assertEqual(seed.prep_bootstrap('MPT'), expect) + + expect = (os.sep + os.path.join('MPT', 'tmp', 'UUID', 'BS'), + os.sep + os.path.join('tmp', 'UUID')) + self.assertEqual(seed.prep_bootstrap(os.sep + 'MPT'), expect) def test_apply_(self): ''' diff --git a/tests/unit/modules/test_shadow.py b/tests/unit/modules/test_shadow.py index 0e2c98a3ea2..e152d59b9aa 100644 --- a/tests/unit/modules/test_shadow.py +++ b/tests/unit/modules/test_shadow.py @@ -7,7 +7,7 @@ from __future__ import absolute_import # Import Salt Testing libs -from salt.utils import is_linux +import salt.utils.platform from tests.support.mixins import LoaderModuleMockMixin from tests.support.unit import TestCase, skipIf @@ -19,7 +19,7 @@ except ImportError: HAS_SHADOW = False # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six _PASSWORD = 'lamepassword' @@ -41,7 +41,7 @@ _HASHES = dict( ) -@skipIf(not is_linux(), 'minion is not Linux') +@skipIf(not salt.utils.platform.is_linux(), 'minion is not Linux') class LinuxShadowTest(TestCase, LoaderModuleMockMixin): def setup_loader_modules(self): diff --git a/tests/unit/modules/test_snapper.py b/tests/unit/modules/test_snapper.py index f3c4e0fa2d4..18fded6f735 100644 --- a/tests/unit/modules/test_snapper.py +++ b/tests/unit/modules/test_snapper.py @@ -22,7 +22,7 @@ from tests.support.mock import ( ) # Import Salt libs -import salt.ext.six as six +from salt.ext import six from salt.exceptions import CommandExecutionError import salt.modules.snapper as snapper @@ -337,7 +337,7 @@ class SnapperTestCase(TestCase, LoaderModuleMockMixin): patch('salt.modules.snapper.changed_files', MagicMock(return_value=["/tmp/foo2"])), \ patch('salt.modules.snapper._is_text_file', MagicMock(return_value=True)), \ patch('os.path.isfile', MagicMock(side_effect=[False, True])), \ - patch('salt.utils.fopen', mock_open(read_data=FILE_CONTENT["/tmp/foo2"]['post'])), \ + patch('salt.utils.files.fopen', mock_open(read_data=FILE_CONTENT["/tmp/foo2"]['post'])), \ patch('salt.modules.snapper.snapper.ListConfigs', MagicMock(return_value=DBUS_RET['ListConfigs'])): if sys.version_info < (2, 7): self.assertEqual(snapper.diff(), {"/tmp/foo2": MODULE_RET['DIFF']['/tmp/foo26']}) @@ -360,7 +360,7 @@ class SnapperTestCase(TestCase, LoaderModuleMockMixin): mock_open(read_data=FILE_CONTENT["/tmp/foo"]['post']).return_value, mock_open(read_data=FILE_CONTENT["/tmp/foo2"]['post']).return_value, ] - with patch('salt.utils.fopen') as fopen_mock: + with patch('salt.utils.files.fopen') as fopen_mock: fopen_mock.side_effect = fopen_effect module_ret = { "/tmp/foo": MODULE_RET['DIFF']["/tmp/foo"], @@ -388,7 +388,7 @@ class SnapperTestCase(TestCase, LoaderModuleMockMixin): mock_open(read_data="dummy binary").return_value, mock_open(read_data="dummy binary").return_value, ] - with patch('salt.utils.fopen') as fopen_mock: + with patch('salt.utils.files.fopen') as fopen_mock: fopen_mock.side_effect = fopen_effect module_ret = { "/tmp/foo3": MODULE_RET['DIFF']["/tmp/foo3"], diff --git a/tests/unit/modules/test_ssh.py b/tests/unit/modules/test_ssh.py index fbe01a5cb9f..bd15ecbfa90 100644 --- a/tests/unit/modules/test_ssh.py +++ b/tests/unit/modules/test_ssh.py @@ -15,7 +15,8 @@ from tests.support.mock import ( ) # Import Salt Libs -import salt.utils +import salt.utils.files +import salt.utils.platform import salt.modules.ssh as ssh from salt.exceptions import CommandExecutionError @@ -90,9 +91,14 @@ class SSHAuthKeyTestCase(TestCase, LoaderModuleMockMixin): options = 'command="/usr/local/lib/ssh-helper"' email = 'github.com' empty_line = '\n' - comment_line = '# this is a comment \n' + comment_line = '# this is a comment\n' + # Write out the authorized key to a temporary file - temp_file = tempfile.NamedTemporaryFile(delete=False, mode='w+') + if salt.utils.platform.is_windows(): + temp_file = tempfile.NamedTemporaryFile(delete=False) + else: + temp_file = tempfile.NamedTemporaryFile(delete=False, mode='w+') + # Add comment temp_file.write(comment_line) # Add empty line for #41335 @@ -105,7 +111,7 @@ class SSHAuthKeyTestCase(TestCase, LoaderModuleMockMixin): ssh._replace_auth_key('foo', key, config=temp_file.name) # The previous authorized key should have been replaced by the simpler one - with salt.utils.fopen(temp_file.name) as _fh: + with salt.utils.files.fopen(temp_file.name) as _fh: file_txt = _fh.read() self.assertIn(enc, file_txt) self.assertIn(key, file_txt) @@ -116,7 +122,7 @@ class SSHAuthKeyTestCase(TestCase, LoaderModuleMockMixin): enc = 'ecdsa-sha2-nistp256' key = 'abcxyz' - with salt.utils.fopen(temp_file.name, 'a') as _fh: + with salt.utils.files.fopen(temp_file.name, 'a') as _fh: _fh.write('{0} {1}'.format(enc, key)) # Replace the simple key from before with the more complicated options + new email @@ -130,7 +136,7 @@ class SSHAuthKeyTestCase(TestCase, LoaderModuleMockMixin): ssh._replace_auth_key('foo', key, enc=enc, comment=email, options=options, config=temp_file.name) # Assert that the new line was added as-is to the file - with salt.utils.fopen(temp_file.name) as _fh: + with salt.utils.files.fopen(temp_file.name) as _fh: file_txt = _fh.read() self.assertIn(enc, file_txt) self.assertIn(key, file_txt) diff --git a/tests/unit/modules/test_state.py b/tests/unit/modules/test_state.py index b12313259a3..b6dc97af05e 100644 --- a/tests/unit/modules/test_state.py +++ b/tests/unit/modules/test_state.py @@ -19,9 +19,14 @@ from tests.support.mock import ( ) # Import Salt Libs +import salt.config +import salt.loader import salt.utils +import salt.utils.odict +import salt.utils.platform import salt.modules.state as state from salt.exceptions import SaltInvocationError +from salt.ext import six class MockState(object): @@ -37,7 +42,11 @@ class MockState(object): ''' flag = None - def __init__(self, opts, pillar=False, pillar_enc=None): + def __init__(self, + opts, + pillar_override=False, + pillar_enc=None, + initial_pillar=None): pass def verify_data(self, data): @@ -135,10 +144,10 @@ class MockState(object): opts = {'state_top': '', 'pillar': {}} - def __init__(self, opts, pillar=None, *args, **kwargs): - self.building_highstate = {} + def __init__(self, opts, pillar_override=None, *args, **kwargs): + self.building_highstate = salt.utils.odict.OrderedDict self.state = MockState.State(opts, - pillar=pillar) + pillar_override=pillar_override) def render_state(self, sls, saltenv, mods, matches, local=False): ''' @@ -339,10 +348,23 @@ class StateTestCase(TestCase, LoaderModuleMockMixin): ''' def setup_loader_modules(self): + utils = salt.loader.utils( + salt.config.DEFAULT_MINION_OPTS, + whitelist=['state'] + ) patcher = patch('salt.modules.state.salt.state', MockState()) patcher.start() self.addCleanup(patcher.stop) - return {state: {'__opts__': {'cachedir': '/D'}}} + return { + state: { + '__opts__': { + 'cachedir': '/D', + 'environment': None, + '__cli': 'salt', + }, + '__utils__': utils, + }, + } def test_running(self): ''' @@ -605,7 +627,10 @@ class StateTestCase(TestCase, LoaderModuleMockMixin): self.assertEqual(state.sls_id("apache", "http"), "A") with patch.dict(state.__opts__, {"test": "A"}): - mock = MagicMock(return_value={'test': True}) + mock = MagicMock( + return_value={'test': True, + 'environment': None} + ) with patch.object(state, '_get_opts', mock): mock = MagicMock(return_value=True) with patch.object(salt.utils, 'test_mode', mock): @@ -629,7 +654,10 @@ class StateTestCase(TestCase, LoaderModuleMockMixin): self.assertEqual(state.show_low_sls("foo"), "A") with patch.dict(state.__opts__, {"test": "A"}): - mock = MagicMock(return_value={'test': True}) + mock = MagicMock( + return_value={'test': True, + 'environment': None} + ) with patch.object(state, '_get_opts', mock): MockState.State.flag = True MockState.HighState.flag = True @@ -648,7 +676,10 @@ class StateTestCase(TestCase, LoaderModuleMockMixin): self.assertEqual(state.show_sls("foo"), "A") with patch.dict(state.__opts__, {"test": "A"}): - mock = MagicMock(return_value={'test': True}) + mock = MagicMock( + return_value={'test': True, + 'environment': None} + ) with patch.object(state, '_get_opts', mock): mock = MagicMock(return_value=True) with patch.object(salt.utils, 'test_mode', mock): @@ -779,10 +810,10 @@ class StateTestCase(TestCase, LoaderModuleMockMixin): patch('salt.modules.state.salt.payload', MockSerial): mock = MagicMock(side_effect=[True, True, False]) with patch.object(os.path, 'isfile', mock): - with patch('salt.utils.fopen', mock_open()): + with patch('salt.utils.files.fopen', mock_open()): self.assertDictEqual(state.check_request(), {'A': 'B'}) - with patch('salt.utils.fopen', mock_open()): + with patch('salt.utils.files.fopen', mock_open()): self.assertEqual(state.check_request("A"), 'B') self.assertDictEqual(state.check_request(), {}) @@ -802,9 +833,9 @@ class StateTestCase(TestCase, LoaderModuleMockMixin): with patch.object(state, 'check_request', mock): mock = MagicMock(return_value=True) with patch.object(os, 'umask', mock): - with patch.object(salt.utils, 'is_windows', mock): + with patch.object(salt.utils.platform, 'is_windows', mock): with patch.dict(state.__salt__, {'cmd.run': mock}): - with patch('salt.utils.fopen', mock_open()): + with patch('salt.utils.files.fopen', mock_open()): mock = MagicMock(return_value=True) with patch.object(os, 'umask', mock): self.assertTrue(state.request("A")) @@ -827,7 +858,6 @@ class StateTestCase(TestCase, LoaderModuleMockMixin): with patch.dict(state.__context__, {"retcode": 1}): self.assertEqual( state.sls("core,edit.vim dev", - None, None, None, True), @@ -842,13 +872,13 @@ class StateTestCase(TestCase, LoaderModuleMockMixin): with patch.dict(state.__context__, {"retcode": 5}): with patch.dict(state.__pillar__, {"_errors": "E1"}): self.assertListEqual(state.sls("core,edit.vim dev", - None, None, None, True), ret) with patch.dict(state.__opts__, {"test": None}): - mock = MagicMock(return_value={"test": ""}) + mock = MagicMock(return_value={"test": "", + "environment": None}) with patch.object(state, '_get_opts', mock): mock = MagicMock(return_value=True) with patch.object(salt.utils, @@ -860,7 +890,6 @@ class StateTestCase(TestCase, LoaderModuleMockMixin): "core,edit.vim dev", None, None, - None, True, pillar="A") @@ -873,11 +902,10 @@ class StateTestCase(TestCase, LoaderModuleMockMixin): 'isfile', mock): with patch( - 'salt.utils.fopen', + 'salt.utils.files.fopen', mock_open()): self.assertTrue( state.sls(arg, - None, None, None, True, @@ -891,7 +919,6 @@ class StateTestCase(TestCase, LoaderModuleMockMixin): ".vim dev", None, None, - None, True) ) @@ -917,18 +944,17 @@ class StateTestCase(TestCase, LoaderModuleMockMixin): with patch.object(os.path, 'join', mock): with patch.object(os, 'umask', mock): mock = MagicMock(return_value=False) - with patch.object(salt.utils, 'is_windows', mock): + with patch.object(salt.utils.platform, 'is_windows', mock): mock = MagicMock(return_value=True) with patch.object(os, 'umask', mock): with patch.object(state, '_set_retcode', mock): with patch.dict(state.__opts__, {"test": True}): - with patch('salt.utils.fopen', mock_open()): + with patch('salt.utils.files.fopen', mock_open()): self.assertTrue(state.sls("core,edit" ".vim dev", None, None, - None, True)) def test_pkg(self): @@ -951,7 +977,7 @@ class StateTestCase(TestCase, LoaderModuleMockMixin): MockTarFile.path = "" MockJson.flag = True - with patch('salt.utils.fopen', mock_open()): + with patch('salt.utils.files.fopen', mock_open()): self.assertListEqual(state.pkg("/tmp/state_pkg.tgz", 0, "md5"), @@ -959,6 +985,12 @@ class StateTestCase(TestCase, LoaderModuleMockMixin): MockTarFile.path = "" MockJson.flag = False - with patch('salt.utils.fopen', mock_open()): - self.assertTrue(state.pkg("/tmp/state_pkg.tgz", - 0, "md5")) + if six.PY2: + with patch('salt.utils.files.fopen', mock_open()), \ + patch.dict(state.__utils__, {'state.check_result': MagicMock(return_value=True)}): + self.assertTrue(state.pkg("/tmp/state_pkg.tgz", + 0, "md5")) + else: + with patch('salt.utils.files.fopen', mock_open()): + self.assertTrue(state.pkg("/tmp/state_pkg.tgz", + 0, "md5")) diff --git a/tests/unit/modules/test_status.py b/tests/unit/modules/test_status.py index 8ad8a02efbc..832ca427bbe 100644 --- a/tests/unit/modules/test_status.py +++ b/tests/unit/modules/test_status.py @@ -4,7 +4,7 @@ from __future__ import absolute_import # Import Salt Libs -import salt.utils +import salt.utils.platform import salt.modules.status as status from salt.exceptions import CommandExecutionError @@ -72,7 +72,7 @@ class StatusTestCase(TestCase, LoaderModuleMockMixin): ''' m = self._set_up_test_uptime() - with patch.multiple(salt.utils, + with patch.multiple(salt.utils.platform, is_linux=MagicMock(return_value=True), is_sunos=MagicMock(return_value=False), is_darwin=MagicMock(return_value=False), @@ -83,7 +83,7 @@ class StatusTestCase(TestCase, LoaderModuleMockMixin): with patch('time.time', MagicMock(return_value=m.now)): with patch('os.path.exists', MagicMock(return_value=True)): proc_uptime = '{0} {1}'.format(m.ut, m.idle) - with patch('salt.utils.fopen', mock_open(read_data=proc_uptime)): + with patch('salt.utils.files.fopen', mock_open(read_data=proc_uptime)): ret = status.uptime() self.assertDictEqual(ret, m.ret) @@ -97,7 +97,7 @@ class StatusTestCase(TestCase, LoaderModuleMockMixin): ''' m = self._set_up_test_uptime() m2 = self._set_up_test_uptime_sunos() - with patch.multiple(salt.utils, + with patch.multiple(salt.utils.platform, is_linux=MagicMock(return_value=False), is_sunos=MagicMock(return_value=True), is_darwin=MagicMock(return_value=False), @@ -119,7 +119,7 @@ class StatusTestCase(TestCase, LoaderModuleMockMixin): kern_boottime = ('{{ sec = {0}, usec = {1:0<6} }} Mon Oct 03 03:09:18.23 2016' ''.format(*str(m.now - m.ut).split('.'))) - with patch.multiple(salt.utils, + with patch.multiple(salt.utils.platform, is_linux=MagicMock(return_value=False), is_sunos=MagicMock(return_value=False), is_darwin=MagicMock(return_value=True), @@ -140,7 +140,7 @@ class StatusTestCase(TestCase, LoaderModuleMockMixin): ''' Test modules.status.uptime function for other platforms ''' - with patch.multiple(salt.utils, + with patch.multiple(salt.utils.platform, is_linux=MagicMock(return_value=False), is_sunos=MagicMock(return_value=False), is_darwin=MagicMock(return_value=False), diff --git a/tests/unit/modules/test_syslog_ng.py b/tests/unit/modules/test_syslog_ng.py index 7a53dc5b09c..27f8550204a 100644 --- a/tests/unit/modules/test_syslog_ng.py +++ b/tests/unit/modules/test_syslog_ng.py @@ -13,7 +13,7 @@ from tests.support.unit import skipIf, TestCase from tests.support.mock import NO_MOCK, NO_MOCK_REASON, MagicMock, patch # Import Salt libs -import salt +import salt.utils.path import salt.modules.syslog_ng as syslog_ng _VERSION = "3.6.0alpha0" @@ -261,7 +261,7 @@ class SyslogNGTestCase(TestCase, LoaderModuleMockMixin): function_args = {} installed = True - if not salt.utils.which("syslog-ng"): + if not salt.utils.path.which("syslog-ng"): installed = False if "syslog-ng-ctl" in mock_function_args: expected_output = _SYSLOG_NG_CTL_NOT_INSTALLED_RETURN_VALUE diff --git a/tests/unit/modules/test_timezone.py b/tests/unit/modules/test_timezone.py index 1214f977dac..c2fb6a4864e 100644 --- a/tests/unit/modules/test_timezone.py +++ b/tests/unit/modules/test_timezone.py @@ -19,8 +19,9 @@ from tests.support.mock import ( # Import Salt Libs from salt.exceptions import CommandExecutionError, SaltInvocationError import salt.modules.timezone as timezone -import salt.ext.six as six -import salt.utils +from salt.ext import six +import salt.utils.platform +import salt.utils.stringutils GET_ZONE_FILE = 'salt.modules.timezone._get_zone_file' GET_ETC_LOCALTIME_PATH = 'salt.modules.timezone._get_etc_localtime_path' @@ -77,7 +78,7 @@ class TimezoneTestCase(TestCase, LoaderModuleMockMixin): def create_tempfile_with_contents(self, contents): temp = NamedTemporaryFile(delete=False) if six.PY3: - temp.write(salt.utils.to_bytes(contents)) + temp.write(salt.utils.stringutils.to_bytes(contents)) else: temp.write(contents) temp.close() @@ -98,7 +99,7 @@ class TimezoneModuleTestCase(TestCase, LoaderModuleMockMixin): 'cmd.run': MagicMock(), 'cmd.retcode': MagicMock(return_value=0)}}} - @patch('salt.utils.which', MagicMock(return_value=False)) + @patch('salt.utils.path.which', MagicMock(return_value=False)) def test_get_zone_centos(self): ''' Test CentOS is recognized @@ -108,7 +109,7 @@ class TimezoneModuleTestCase(TestCase, LoaderModuleMockMixin): with patch('salt.modules.timezone._get_zone_etc_localtime', MagicMock(return_value=self.TEST_TZ)): assert timezone.get_zone() == self.TEST_TZ - @patch('salt.utils.which', MagicMock(return_value=False)) + @patch('salt.utils.path.which', MagicMock(return_value=False)) def test_get_zone_os_family_rh_suse(self): ''' Test RedHat and Suse are recognized @@ -119,7 +120,7 @@ class TimezoneModuleTestCase(TestCase, LoaderModuleMockMixin): with patch('salt.modules.timezone._get_zone_sysconfig', MagicMock(return_value=self.TEST_TZ)): assert timezone.get_zone() == self.TEST_TZ - @patch('salt.utils.which', MagicMock(return_value=False)) + @patch('salt.utils.path.which', MagicMock(return_value=False)) def test_get_zone_os_family_debian_gentoo(self): ''' Test Debian and Gentoo are recognized @@ -130,7 +131,7 @@ class TimezoneModuleTestCase(TestCase, LoaderModuleMockMixin): with patch('salt.modules.timezone._get_zone_etc_timezone', MagicMock(return_value=self.TEST_TZ)): assert timezone.get_zone() == self.TEST_TZ - @patch('salt.utils.which', MagicMock(return_value=False)) + @patch('salt.utils.path.which', MagicMock(return_value=False)) def test_get_zone_os_family_allbsd_nilinuxrt(self): ''' Test *BSD and NILinuxRT are recognized @@ -141,7 +142,7 @@ class TimezoneModuleTestCase(TestCase, LoaderModuleMockMixin): with patch('salt.modules.timezone._get_zone_etc_localtime', MagicMock(return_value=self.TEST_TZ)): assert timezone.get_zone() == self.TEST_TZ - @patch('salt.utils.which', MagicMock(return_value=False)) + @patch('salt.utils.path.which', MagicMock(return_value=False)) def test_get_zone_os_family_slowlaris(self): ''' Test Slowlaris is recognized @@ -151,7 +152,7 @@ class TimezoneModuleTestCase(TestCase, LoaderModuleMockMixin): with patch('salt.modules.timezone._get_zone_solaris', MagicMock(return_value=self.TEST_TZ)): assert timezone.get_zone() == self.TEST_TZ - @patch('salt.utils.which', MagicMock(return_value=False)) + @patch('salt.utils.path.which', MagicMock(return_value=False)) def test_get_zone_os_family_aix(self): ''' Test IBM AIX is recognized @@ -161,7 +162,8 @@ class TimezoneModuleTestCase(TestCase, LoaderModuleMockMixin): with patch('salt.modules.timezone._get_zone_aix', MagicMock(return_value=self.TEST_TZ)): assert timezone.get_zone() == self.TEST_TZ - @patch('salt.utils.which', MagicMock(return_value=False)) + @skipIf(salt.utils.platform.is_windows(), 'os.symlink not available in Windows') + @patch('salt.utils.path.which', MagicMock(return_value=False)) @patch('os.path.exists', MagicMock(return_value=True)) @patch('os.unlink', MagicMock()) @patch('os.symlink', MagicMock()) @@ -175,7 +177,8 @@ class TimezoneModuleTestCase(TestCase, LoaderModuleMockMixin): name, args, kwargs = timezone.__salt__['file.sed'].mock_calls[0] assert args == ('/etc/sysconfig/clock', '^ZONE=.*', 'ZONE="UTC"') - @patch('salt.utils.which', MagicMock(return_value=False)) + @skipIf(salt.utils.platform.is_windows(), 'os.symlink not available in Windows') + @patch('salt.utils.path.which', MagicMock(return_value=False)) @patch('os.path.exists', MagicMock(return_value=True)) @patch('os.unlink', MagicMock()) @patch('os.symlink', MagicMock()) @@ -189,7 +192,8 @@ class TimezoneModuleTestCase(TestCase, LoaderModuleMockMixin): name, args, kwargs = timezone.__salt__['file.sed'].mock_calls[0] assert args == ('/etc/sysconfig/clock', '^TIMEZONE=.*', 'TIMEZONE="UTC"') - @patch('salt.utils.which', MagicMock(return_value=False)) + @skipIf(salt.utils.platform.is_windows(), 'os.symlink not available in Windows') + @patch('salt.utils.path.which', MagicMock(return_value=False)) @patch('os.path.exists', MagicMock(return_value=True)) @patch('os.unlink', MagicMock()) @patch('os.symlink', MagicMock()) @@ -200,14 +204,15 @@ class TimezoneModuleTestCase(TestCase, LoaderModuleMockMixin): ''' with patch.dict(timezone.__grains__, {'os_family': ['Gentoo']}): _fopen = mock_open() - with patch('salt.utils.fopen', _fopen): + with patch('salt.utils.files.fopen', _fopen): assert timezone.set_zone(self.TEST_TZ) name, args, kwargs = _fopen.mock_calls[0] assert args == ('/etc/timezone', 'w') name, args, kwargs = _fopen.return_value.__enter__.return_value.write.mock_calls[0] assert args == ('UTC',) - @patch('salt.utils.which', MagicMock(return_value=False)) + @skipIf(salt.utils.platform.is_windows(), 'os.symlink not available in Windows') + @patch('salt.utils.path.which', MagicMock(return_value=False)) @patch('os.path.exists', MagicMock(return_value=True)) @patch('os.unlink', MagicMock()) @patch('os.symlink', MagicMock()) @@ -218,14 +223,15 @@ class TimezoneModuleTestCase(TestCase, LoaderModuleMockMixin): ''' with patch.dict(timezone.__grains__, {'os_family': ['Debian']}): _fopen = mock_open() - with patch('salt.utils.fopen', _fopen): + with patch('salt.utils.files.fopen', _fopen): assert timezone.set_zone(self.TEST_TZ) name, args, kwargs = _fopen.mock_calls[0] assert args == ('/etc/timezone', 'w') name, args, kwargs = _fopen.return_value.__enter__.return_value.write.mock_calls[0] assert args == ('UTC',) - @patch('salt.utils.which', MagicMock(return_value=True)) + @skipIf(salt.utils.platform.is_windows(), 'os.symlink not available in Windows') + @patch('salt.utils.path.which', MagicMock(return_value=True)) @patch('os.path.exists', MagicMock(return_value=True)) @patch('os.unlink', MagicMock()) @patch('os.symlink', MagicMock()) @@ -239,7 +245,8 @@ class TimezoneModuleTestCase(TestCase, LoaderModuleMockMixin): with patch('salt.modules.timezone._timedatectl', MagicMock(return_value={'stdout': 'rtc in local tz:yes'})): assert timezone.get_hwclock() == 'localtime' - @patch('salt.utils.which', MagicMock(return_value=False)) + @skipIf(salt.utils.platform.is_windows(), 'os.symlink not available in Windows') + @patch('salt.utils.path.which', MagicMock(return_value=False)) @patch('os.path.exists', MagicMock(return_value=True)) @patch('os.unlink', MagicMock()) @patch('os.symlink', MagicMock()) @@ -254,7 +261,8 @@ class TimezoneModuleTestCase(TestCase, LoaderModuleMockMixin): assert args == (['tail', '-n', '1', '/etc/adjtime'],) assert kwarg == {'python_shell': False} - @patch('salt.utils.which', MagicMock(return_value=False)) + @skipIf(salt.utils.platform.is_windows(), 'os.symlink not available in Windows') + @patch('salt.utils.path.which', MagicMock(return_value=False)) @patch('os.path.exists', MagicMock(return_value=True)) @patch('os.unlink', MagicMock()) @patch('os.symlink', MagicMock()) @@ -274,7 +282,7 @@ class TimezoneModuleTestCase(TestCase, LoaderModuleMockMixin): Test get hwclock on Debian :return: ''' - with patch('salt.utils.which', MagicMock(return_value=False)): + with patch('salt.utils.path.which', MagicMock(return_value=False)): with patch('os.path.exists', MagicMock(return_value=True)): with patch('os.unlink', MagicMock()): with patch('os.symlink', MagicMock()): @@ -284,7 +292,8 @@ class TimezoneModuleTestCase(TestCase, LoaderModuleMockMixin): assert args == (['tail', '-n', '1', '/etc/adjtime'],) assert kwarg == {'python_shell': False} - @patch('salt.utils.which', MagicMock(return_value=False)) + @skipIf(salt.utils.platform.is_windows(), 'os.symlink not available in Windows') + @patch('salt.utils.path.which', MagicMock(return_value=False)) @patch('os.path.exists', MagicMock(return_value=True)) @patch('os.unlink', MagicMock()) @patch('os.symlink', MagicMock()) @@ -296,10 +305,11 @@ class TimezoneModuleTestCase(TestCase, LoaderModuleMockMixin): # Incomplete with patch.dict(timezone.__grains__, {'os_family': ['Solaris']}): assert timezone.get_hwclock() == 'UTC' - with patch('salt.utils.fopen', mock_open()): + with patch('salt.utils.files.fopen', mock_open()): assert timezone.get_hwclock() == 'localtime' - @patch('salt.utils.which', MagicMock(return_value=False)) + @skipIf(salt.utils.platform.is_windows(), 'os.symlink not available in Windows') + @patch('salt.utils.path.which', MagicMock(return_value=False)) @patch('os.path.exists', MagicMock(return_value=True)) @patch('os.unlink', MagicMock()) @patch('os.symlink', MagicMock()) @@ -315,7 +325,8 @@ class TimezoneModuleTestCase(TestCase, LoaderModuleMockMixin): with patch.dict(timezone.__grains__, {'os_family': ['AIX']}): assert timezone.get_hwclock() == hwclock - @patch('salt.utils.which', MagicMock(return_value=False)) + @skipIf(salt.utils.platform.is_windows(), 'os.symlink not available in Windows') + @patch('salt.utils.path.which', MagicMock(return_value=False)) @patch('os.path.exists', MagicMock(return_value=True)) @patch('os.unlink', MagicMock()) @patch('os.symlink', MagicMock()) @@ -330,7 +341,8 @@ class TimezoneModuleTestCase(TestCase, LoaderModuleMockMixin): assert timezone.set_hwclock('forty two') assert timezone.set_hwclock('UTC') - @patch('salt.utils.which', MagicMock(return_value=False)) + @skipIf(salt.utils.platform.is_windows(), 'os.symlink not available in Windows') + @patch('salt.utils.path.which', MagicMock(return_value=False)) @patch('os.path.exists', MagicMock(return_value=True)) @patch('os.unlink', MagicMock()) @patch('os.symlink', MagicMock()) @@ -349,7 +361,8 @@ class TimezoneModuleTestCase(TestCase, LoaderModuleMockMixin): assert args == (['rtc', '-z', 'GMT'],) assert kwargs == {'python_shell': False} - @patch('salt.utils.which', MagicMock(return_value=False)) + @skipIf(salt.utils.platform.is_windows(), 'os.symlink not available in Windows') + @patch('salt.utils.path.which', MagicMock(return_value=False)) @patch('os.path.exists', MagicMock(return_value=True)) @patch('os.unlink', MagicMock()) @patch('os.symlink', MagicMock()) @@ -365,7 +378,8 @@ class TimezoneModuleTestCase(TestCase, LoaderModuleMockMixin): assert args == (['timezonectl', 'set-local-rtc', 'false'],) assert kwargs == {'python_shell': False} - @patch('salt.utils.which', MagicMock(return_value=False)) + @skipIf(salt.utils.platform.is_windows(), 'os.symlink not available in Windows') + @patch('salt.utils.path.which', MagicMock(return_value=False)) @patch('os.path.exists', MagicMock(return_value=True)) @patch('os.unlink', MagicMock()) @patch('os.symlink', MagicMock()) @@ -380,7 +394,8 @@ class TimezoneModuleTestCase(TestCase, LoaderModuleMockMixin): name, args, kwargs = timezone.__salt__['file.sed'].mock_calls[0] assert args == ('/etc/sysconfig/clock', '^ZONE=.*', 'ZONE="TEST_TIMEZONE"') - @patch('salt.utils.which', MagicMock(return_value=False)) + @skipIf(salt.utils.platform.is_windows(), 'os.symlink not available in Windows') + @patch('salt.utils.path.which', MagicMock(return_value=False)) @patch('os.path.exists', MagicMock(return_value=True)) @patch('os.unlink', MagicMock()) @patch('os.symlink', MagicMock()) @@ -395,7 +410,8 @@ class TimezoneModuleTestCase(TestCase, LoaderModuleMockMixin): name, args, kwargs = timezone.__salt__['file.sed'].mock_calls[0] assert args == ('/etc/sysconfig/clock', '^TIMEZONE=.*', 'TIMEZONE="TEST_TIMEZONE"') - @patch('salt.utils.which', MagicMock(return_value=False)) + @skipIf(salt.utils.platform.is_windows(), 'os.symlink not available in Windows') + @patch('salt.utils.path.which', MagicMock(return_value=False)) @patch('os.path.exists', MagicMock(return_value=True)) @patch('os.unlink', MagicMock()) @patch('os.symlink', MagicMock()) @@ -414,7 +430,8 @@ class TimezoneModuleTestCase(TestCase, LoaderModuleMockMixin): name, args, kwargs = timezone.__salt__['file.sed'].mock_calls[1] assert args == ('/etc/default/rcS', '^UTC=.*', 'UTC=no') - @patch('salt.utils.which', MagicMock(return_value=False)) + @skipIf(salt.utils.platform.is_windows(), 'os.symlink not available in Windows') + @patch('salt.utils.path.which', MagicMock(return_value=False)) @patch('os.path.exists', MagicMock(return_value=True)) @patch('os.unlink', MagicMock()) @patch('os.symlink', MagicMock()) diff --git a/tests/unit/modules/test_tls.py b/tests/unit/modules/test_tls.py index 42a8d7a7d6f..953d4eef102 100644 --- a/tests/unit/modules/test_tls.py +++ b/tests/unit/modules/test_tls.py @@ -188,7 +188,7 @@ class TLSAddTestCase(TestCase, LoaderModuleMockMixin): ca_path = '/tmp/test_tls' ca_name = 'test_ca' mock_opt = MagicMock(return_value=ca_path) - with patch('salt.utils.fopen', + with patch('salt.utils.files.fopen', mock_open(read_data=_TLS_TEST_DATA['ca_cert'])), \ patch.dict(tls.__salt__, {'config.option': mock_opt}), \ patch('os.path.exists', MagicMock(return_value=True)), \ @@ -269,7 +269,7 @@ class TLSAddTestCase(TestCase, LoaderModuleMockMixin): if 'extensions' not in reference: del source['extensions'] - with patch('salt.utils.fopen', + with patch('salt.utils.files.fopen', mock_open(read_data=_TLS_TEST_DATA['ca_cert'])): try: result = ignore_extensions(tls.cert_info(certp)) diff --git a/tests/unit/modules/test_twilio_notify.py b/tests/unit/modules/test_twilio_notify.py index 6aaaaae573e..370b45d6908 100644 --- a/tests/unit/modules/test_twilio_notify.py +++ b/tests/unit/modules/test_twilio_notify.py @@ -19,6 +19,17 @@ from tests.support.mock import ( # Import Salt Libs import salt.modules.twilio_notify as twilio_notify +HAS_LIBS = False +try: + import twilio + if twilio.__version__ > 5: + TWILIO_5 = False + else: + TWILIO_5 = True + HAS_LIBS = True +except ImportError: + pass + class MockTwilioRestException(Exception): ''' @@ -75,9 +86,13 @@ class MockTwilioRestClient(object): Mock TwilioRestClient class ''' def __init__(self): - self.sms = MockSMS() + if TWILIO_5: + self.sms = MockSMS() + else: + self.messages = MockMessages() +@skipIf(not HAS_LIBS, 'twilio.rest is not available') @skipIf(NO_MOCK, NO_MOCK_REASON) class TwilioNotifyTestCase(TestCase, LoaderModuleMockMixin): ''' diff --git a/tests/unit/modules/test_useradd.py b/tests/unit/modules/test_useradd.py index 2bcbee38d19..7fd457425fc 100644 --- a/tests/unit/modules/test_useradd.py +++ b/tests/unit/modules/test_useradd.py @@ -72,6 +72,7 @@ class UserAddTestCase(TestCase, LoaderModuleMockMixin): # 'getent' function tests: 2 + @skipIf(HAS_PWD is False, 'The pwd module is not available') def test_getent(self): ''' Test if user.getent already have a value @@ -359,6 +360,7 @@ class UserAddTestCase(TestCase, LoaderModuleMockMixin): # 'list_users' function tests: 1 + @skipIf(HAS_PWD is False, 'The pwd module is not available') def test_list_users(self): ''' Test if it returns a list of all users diff --git a/tests/unit/modules/test_uwsgi.py b/tests/unit/modules/test_uwsgi.py index 1957bd97285..f22cc5975ef 100644 --- a/tests/unit/modules/test_uwsgi.py +++ b/tests/unit/modules/test_uwsgi.py @@ -16,7 +16,7 @@ import salt.modules.uwsgi as uwsgi class UwsgiTestCase(TestCase, LoaderModuleMockMixin): def setup_loader_modules(self): - patcher = patch('salt.utils.which', Mock(return_value='/usr/bin/uwsgi')) + patcher = patch('salt.utils.path.which', Mock(return_value='/usr/bin/uwsgi')) patcher.start() self.addCleanup(patcher.stop) return {uwsgi: {}} diff --git a/tests/unit/modules/test_virt.py b/tests/unit/modules/test_virt.py index f616bb9391c..52e7f27e0b5 100644 --- a/tests/unit/modules/test_virt.py +++ b/tests/unit/modules/test_virt.py @@ -17,7 +17,7 @@ import salt.utils # Import third party libs import yaml -import salt.ext.six as six +from salt.ext import six @skipIf(NO_MOCK, NO_MOCK_REASON) diff --git a/tests/unit/modules/test_virtualenv.py b/tests/unit/modules/test_virtualenv.py index da49531dd97..7e86f2bcad7 100644 --- a/tests/unit/modules/test_virtualenv.py +++ b/tests/unit/modules/test_virtualenv.py @@ -28,7 +28,7 @@ class VirtualenvTestCase(TestCase, LoaderModuleMockMixin): def setup_loader_modules(self): base_virtualenv_mock = MagicMock() base_virtualenv_mock.__version__ = '1.9.1' - patcher = patch('salt.utils.which', lambda exe: exe) + patcher = patch('salt.utils.path.which', lambda exe: exe) patcher.start() self.addCleanup(patcher.stop) return { @@ -109,10 +109,9 @@ class VirtualenvTestCase(TestCase, LoaderModuleMockMixin): # Are we logging the deprecation information? self.assertIn( - 'INFO:The virtualenv \'--never-download\' option has been ' - 'deprecated in virtualenv(>=1.10), as such, the ' - '\'never_download\' option to `virtualenv.create()` has ' - 'also been deprecated and it\'s not necessary anymore.', + 'INFO:--never-download was deprecated in 1.10.0, ' + 'but reimplemented in 14.0.0. If this feature is needed, ' + 'please install a supported virtualenv version.', handler.messages ) diff --git a/tests/unit/modules/test_vsphere.py b/tests/unit/modules/test_vsphere.py index 738f7f2dc38..ce9f813094d 100644 --- a/tests/unit/modules/test_vsphere.py +++ b/tests/unit/modules/test_vsphere.py @@ -11,7 +11,8 @@ from __future__ import absolute_import # Import Salt Libs import salt.modules.vsphere as vsphere -from salt.exceptions import CommandExecutionError, VMwareSaltError +from salt.exceptions import CommandExecutionError, VMwareSaltError, \ + ArgumentValueError # Import Salt Testing Libs from tests.support.mixins import LoaderModuleMockMixin @@ -20,7 +21,8 @@ from tests.support.mock import ( MagicMock, patch, NO_MOCK, - NO_MOCK_REASON + NO_MOCK_REASON, + call ) # Globals @@ -618,9 +620,29 @@ class _GetProxyConnectionDetailsTestCase(TestCase, LoaderModuleMockMixin): 'mechanism': 'fake_mechanism', 'principal': 'fake_principal', 'domain': 'fake_domain'} + self.esxdatacenter_details = {'vcenter': 'fake_vcenter', + 'datacenter': 'fake_dc', + 'username': 'fake_username', + 'password': 'fake_password', + 'protocol': 'fake_protocol', + 'port': 'fake_port', + 'mechanism': 'fake_mechanism', + 'principal': 'fake_principal', + 'domain': 'fake_domain'} + self.esxcluster_details = {'vcenter': 'fake_vcenter', + 'datacenter': 'fake_dc', + 'cluster': 'fake_cluster', + 'username': 'fake_username', + 'password': 'fake_password', + 'protocol': 'fake_protocol', + 'port': 'fake_port', + 'mechanism': 'fake_mechanism', + 'principal': 'fake_principal', + 'domain': 'fake_domain'} def tearDown(self): - for attrname in ('esxi_host_details', 'esxi_vcenter_details'): + for attrname in ('esxi_host_details', 'esxi_vcenter_details', + 'esxdatacenter_details', 'esxcluster_details'): try: delattr(self, attrname) except AttributeError: @@ -637,6 +659,28 @@ class _GetProxyConnectionDetailsTestCase(TestCase, LoaderModuleMockMixin): 'fake_protocol', 'fake_port', 'fake_mechanism', 'fake_principal', 'fake_domain'), ret) + def test_esxdatacenter_proxy_details(self): + with patch('salt.modules.vsphere.get_proxy_type', + MagicMock(return_value='esxdatacenter')): + with patch.dict(vsphere.__salt__, + {'esxdatacenter.get_details': MagicMock( + return_value=self.esxdatacenter_details)}): + ret = vsphere._get_proxy_connection_details() + self.assertEqual(('fake_vcenter', 'fake_username', 'fake_password', + 'fake_protocol', 'fake_port', 'fake_mechanism', + 'fake_principal', 'fake_domain'), ret) + + def test_esxcluster_proxy_details(self): + with patch('salt.modules.vsphere.get_proxy_type', + MagicMock(return_value='esxcluster')): + with patch.dict(vsphere.__salt__, + {'esxcluster.get_details': MagicMock( + return_value=self.esxcluster_details)}): + ret = vsphere._get_proxy_connection_details() + self.assertEqual(('fake_vcenter', 'fake_username', 'fake_password', + 'fake_protocol', 'fake_port', 'fake_mechanism', + 'fake_principal', 'fake_domain'), ret) + def test_esxi_proxy_vcenter_details(self): with patch('salt.modules.vsphere.get_proxy_type', MagicMock(return_value='esxi')): @@ -845,8 +889,8 @@ class GetServiceInstanceViaProxyTestCase(TestCase, LoaderModuleMockMixin): } } - def test_supported_proxes(self): - supported_proxies = ['esxi'] + def test_supported_proxies(self): + supported_proxies = ['esxi', 'esxcluster', 'esxdatacenter'] for proxy_type in supported_proxies: with patch('salt.modules.vsphere.get_proxy_type', MagicMock(return_value=proxy_type)): @@ -888,8 +932,8 @@ class DisconnectTestCase(TestCase, LoaderModuleMockMixin): } } - def test_supported_proxes(self): - supported_proxies = ['esxi'] + def test_supported_proxies(self): + supported_proxies = ['esxi', 'esxcluster', 'esxdatacenter'] for proxy_type in supported_proxies: with patch('salt.modules.vsphere.get_proxy_type', MagicMock(return_value=proxy_type)): @@ -929,8 +973,8 @@ class TestVcenterConnectionTestCase(TestCase, LoaderModuleMockMixin): } } - def test_supported_proxes(self): - supported_proxies = ['esxi'] + def test_supported_proxies(self): + supported_proxies = ['esxi', 'esxcluster', 'esxdatacenter'] for proxy_type in supported_proxies: with patch('salt.modules.vsphere.get_proxy_type', MagicMock(return_value=proxy_type)): @@ -977,3 +1021,310 @@ class TestVcenterConnectionTestCase(TestCase, LoaderModuleMockMixin): MagicMock(return_value=False)): res = vsphere.test_vcenter_connection() self.assertEqual(res, False) + + +@skipIf(NO_MOCK, NO_MOCK_REASON) +class ListDatacentersViaProxyTestCase(TestCase, LoaderModuleMockMixin): + '''Tests for salt.modules.vsphere.list_datacenters_via_proxy''' + def setup_loader_modules(self): + self.mock_si = MagicMock() + self.addCleanup(delattr, self, 'mock_si') + patcher = patch('salt.utils.vmware.get_service_instance', + MagicMock(return_value=self.mock_si)) + patcher.start() + self.addCleanup(patcher.stop) + patcher = patch('salt.utils.vmware.get_datacenters', MagicMock()) + patcher.start() + self.addCleanup(patcher.stop) + patcher = patch('salt.utils.vmware.get_managed_object_name', + MagicMock()) + patcher.start() + self.addCleanup(patcher.stop) + return { + vsphere: { + '__virtual__': MagicMock(return_value='vsphere'), + '_get_proxy_connection_details': MagicMock(), + 'get_proxy_type': MagicMock(return_value='esxdatacenter') + } + } + + def test_supported_proxies(self): + supported_proxies = ['esxcluster', 'esxdatacenter'] + for proxy_type in supported_proxies: + with patch('salt.modules.vsphere.get_proxy_type', + MagicMock(return_value=proxy_type)): + vsphere.list_datacenters_via_proxy() + + def test_default_params(self): + mock_get_datacenters = MagicMock() + with patch('salt.utils.vmware.get_datacenters', + mock_get_datacenters): + vsphere.list_datacenters_via_proxy() + mock_get_datacenters.assert_called_once_with(self.mock_si, + get_all_datacenters=True) + + def test_defined_service_instance(self): + mock_si = MagicMock() + mock_get_datacenters = MagicMock() + with patch('salt.utils.vmware.get_datacenters', + mock_get_datacenters): + vsphere.list_datacenters_via_proxy(service_instance=mock_si) + mock_get_datacenters.assert_called_once_with(mock_si, + get_all_datacenters=True) + + def test_defined_datacenter_names(self): + mock_datacenters = MagicMock() + mock_get_datacenters = MagicMock() + with patch('salt.utils.vmware.get_datacenters', + mock_get_datacenters): + vsphere.list_datacenters_via_proxy(mock_datacenters) + mock_get_datacenters.assert_called_once_with(self.mock_si, + mock_datacenters) + + def test_get_managed_object_name_calls(self): + mock_get_managed_object_name = MagicMock() + mock_dcs = [MagicMock(), MagicMock()] + with patch('salt.utils.vmware.get_datacenters', + MagicMock(return_value=mock_dcs)): + with patch('salt.utils.vmware.get_managed_object_name', + mock_get_managed_object_name): + vsphere.list_datacenters_via_proxy() + mock_get_managed_object_name.assert_has_calls([call(mock_dcs[0]), + call(mock_dcs[1])]) + + def test_returned_array(self): + with patch('salt.utils.vmware.get_datacenters', + MagicMock(return_value=[MagicMock(), MagicMock()])): + # 2 datacenters + with patch('salt.utils.vmware.get_managed_object_name', + MagicMock(side_effect=['fake_dc1', 'fake_dc2', + 'fake_dc3'])): + # 3 possible names + res = vsphere.list_datacenters_via_proxy() + + # Just the first two names are in the result + self.assertEqual(res, [{'name': 'fake_dc1'}, {'name': 'fake_dc2'}]) + + +@skipIf(NO_MOCK, NO_MOCK_REASON) +class CreateDatacenterTestCase(TestCase, LoaderModuleMockMixin): + '''Tests for salt.modules.vsphere.create_datacenter''' + def setup_loader_modules(self): + self.mock_si = MagicMock() + self.addCleanup(delattr, self, 'mock_si') + patcher = patch('salt.utils.vmware.get_service_instance', MagicMock(return_value=self.mock_si)) + patcher.start() + self.addCleanup(patcher.stop) + patcher = patch('salt.utils.vmware.create_datacenter', MagicMock()) + patcher.start() + self.addCleanup(patcher.stop) + return { + vsphere: { + '__virtual__': MagicMock(return_value='vsphere'), + '_get_proxy_connection_details': MagicMock(), + 'get_proxy_type': MagicMock(return_value='esxdatacenter') + } + } + + def test_supported_proxies(self): + supported_proxies = ['esxdatacenter'] + for proxy_type in supported_proxies: + with patch('salt.modules.vsphere.get_proxy_type', + MagicMock(return_value=proxy_type)): + vsphere.create_datacenter('fake_dc1') + + def test_default_service_instance(self): + mock_create_datacenter = MagicMock() + with patch('salt.utils.vmware.create_datacenter', + mock_create_datacenter): + vsphere.create_datacenter('fake_dc1') + mock_create_datacenter.assert_called_once_with(self.mock_si, + 'fake_dc1') + + def test_defined_service_instance(self): + mock_si = MagicMock() + mock_create_datacenter = MagicMock() + with patch('salt.utils.vmware.create_datacenter', + mock_create_datacenter): + vsphere.create_datacenter('fake_dc1', service_instance=mock_si) + mock_create_datacenter.assert_called_once_with(mock_si, 'fake_dc1') + + def test_returned_value(self): + res = vsphere.create_datacenter('fake_dc1') + self.assertEqual(res, {'create_datacenter': True}) + + +@skipIf(NO_MOCK, NO_MOCK_REASON) +class ListClusterTestCase(TestCase, LoaderModuleMockMixin): + '''Tests for salt.modules.vsphere.list_cluster''' + def setup_loader_modules(self): + return { + vsphere: { + '__virtual__': MagicMock(return_value='vsphere'), + '_get_proxy_connection_details': MagicMock(), + '__salt__': {} + } + } + + def setUp(self): + attrs = (('mock_si', MagicMock()), + ('mock_dc', MagicMock()), + ('mock_cl', MagicMock()), + ('mock__get_cluster_dict', MagicMock())) + for attr, mock_obj in attrs: + setattr(self, attr, mock_obj) + self.addCleanup(delattr, self, attr) + attrs = (('mock_get_cluster', MagicMock(return_value=self.mock_cl)),) + for attr, mock_obj in attrs: + setattr(self, attr, mock_obj) + self.addCleanup(delattr, self, attr) + patches = ( + ('salt.utils.vmware.get_service_instance', + MagicMock(return_value=self.mock_si)), + ('salt.modules.vsphere.get_proxy_type', + MagicMock(return_value='esxcluster')), + ('salt.modules.vsphere._get_proxy_target', + MagicMock(return_value=self.mock_cl)), + ('salt.utils.vmware.get_cluster', self.mock_get_cluster), + ('salt.modules.vsphere._get_cluster_dict', + self.mock__get_cluster_dict)) + for module, mock_obj in patches: + patcher = patch(module, mock_obj) + patcher.start() + self.addCleanup(patcher.stop) + # Patch __salt__ dunder + patcher = patch.dict(vsphere.__salt__, + {'esxcluster.get_details': + MagicMock(return_value={'cluster': 'cl'})}) + patcher.start() + self.addCleanup(patcher.stop) + + def test_supported_proxies(self): + supported_proxies = ['esxcluster', 'esxdatacenter'] + for proxy_type in supported_proxies: + with patch('salt.modules.vsphere.get_proxy_type', + MagicMock(return_value=proxy_type)): + vsphere.list_cluster(cluster='cl') + + def test_default_service_instance(self): + mock__get_proxy_target = MagicMock() + with patch('salt.modules.vsphere._get_proxy_target', + mock__get_proxy_target): + vsphere.list_cluster() + mock__get_proxy_target.assert_called_once_with(self.mock_si) + + def test_defined_service_instance(self): + mock_si = MagicMock() + mock__get_proxy_target = MagicMock() + with patch('salt.modules.vsphere._get_proxy_target', + mock__get_proxy_target): + vsphere.list_cluster(service_instance=mock_si) + mock__get_proxy_target.assert_called_once_with(mock_si) + + def test_no_cluster_raises_argument_value_error(self): + with patch('salt.modules.vsphere.get_proxy_type', + MagicMock(return_value='esxdatacenter')): + with patch('salt.modules.vsphere._get_proxy_target', MagicMock()): + with self.assertRaises(ArgumentValueError) as excinfo: + vsphere.list_cluster() + self.assertEqual(excinfo.exception.strerror, + '\'cluster\' needs to be specified') + + def test_get_cluster_call(self): + with patch('salt.modules.vsphere.get_proxy_type', + MagicMock(return_value='esxdatacenter')): + with patch('salt.modules.vsphere._get_proxy_target', + MagicMock(return_value=self.mock_dc)): + vsphere.list_cluster(cluster='cl') + self.mock_get_cluster.assert_called_once_with(self.mock_dc, 'cl') + + def test__get_cluster_dict_call(self): + vsphere.list_cluster() + self.mock__get_cluster_dict.assert_called_once_with('cl', self.mock_cl) + + +@skipIf(NO_MOCK, NO_MOCK_REASON) +class _GetProxyTargetTestCase(TestCase, LoaderModuleMockMixin): + '''Tests for salt.modules.vsphere._get_proxy_target''' + def setup_loader_modules(self): + return { + vsphere: { + '__virtual__': MagicMock(return_value='vsphere'), + '_get_proxy_connection_details': MagicMock(), + 'get_proxy_type': MagicMock(return_value='esxdatacenter') + } + } + + def setUp(self): + attrs = (('mock_si', MagicMock()), + ('mock_dc', MagicMock()), + ('mock_cl', MagicMock())) + for attr, mock_obj in attrs: + setattr(self, attr, mock_obj) + self.addCleanup(delattr, self, attr) + attrs = (('mock_get_datacenter', MagicMock(return_value=self.mock_dc)), + ('mock_get_cluster', MagicMock(return_value=self.mock_cl))) + for attr, mock_obj in attrs: + setattr(self, attr, mock_obj) + self.addCleanup(delattr, self, attr) + patches = ( + ('salt.modules.vsphere.get_proxy_type', + MagicMock(return_value='esxcluster')), + ('salt.utils.vmware.is_connection_to_a_vcenter', + MagicMock(return_value=True)), + ('salt.modules.vsphere._get_esxcluster_proxy_details', + MagicMock(return_value=(None, None, None, None, None, None, None, + None, 'datacenter', 'cluster'))), + ('salt.modules.vsphere._get_esxdatacenter_proxy_details', + MagicMock(return_value=(None, None, None, None, None, None, None, + None, 'datacenter'))), + ('salt.utils.vmware.get_datacenter', self.mock_get_datacenter), + ('salt.utils.vmware.get_cluster', self.mock_get_cluster)) + for module, mock_obj in patches: + patcher = patch(module, mock_obj) + patcher.start() + self.addCleanup(patcher.stop) + + def test_supported_proxies(self): + supported_proxies = ['esxcluster', 'esxdatacenter'] + for proxy_type in supported_proxies: + with patch('salt.modules.vsphere.get_proxy_type', + MagicMock(return_value=proxy_type)): + vsphere._get_proxy_target(self.mock_si) + + def test_connected_to_esxi(self): + with patch('salt.utils.vmware.is_connection_to_a_vcenter', + MagicMock(return_value=False)): + with self.assertRaises(CommandExecutionError) as excinfo: + vsphere._get_proxy_target(self.mock_si) + self.assertEqual(excinfo.exception.strerror, + '\'_get_proxy_target\' not supported when ' + 'connected via the ESXi host') + + def test_get_cluster_call(self): + #with patch('salt.modules.vsphere.get_proxy_type', + # MagicMock(return_value='esxcluster')): + vsphere._get_proxy_target(self.mock_si) + self.mock_get_datacenter.assert_called_once_with(self.mock_si, + 'datacenter') + self.mock_get_cluster.assert_called_once_with(self.mock_dc, 'cluster') + + def test_esxcluster_proxy_return(self): + with patch('salt.modules.vsphere.get_proxy_type', + MagicMock(return_value='esxcluster')): + ret = vsphere._get_proxy_target(self.mock_si) + self.assertEqual(ret, self.mock_cl) + + def test_get_datacenter_call(self): + with patch('salt.modules.vsphere.get_proxy_type', + MagicMock(return_value='esxdatacenter')): + vsphere._get_proxy_target(self.mock_si) + self.mock_get_datacenter.assert_called_once_with(self.mock_si, + 'datacenter') + self.assertEqual(self.mock_get_cluster.call_count, 0) + + def test_esxdatacenter_proxy_return(self): + with patch('salt.modules.vsphere.get_proxy_type', + MagicMock(return_value='esxdatacenter')): + ret = vsphere._get_proxy_target(self.mock_si) + self.assertEqual(ret, self.mock_dc) diff --git a/tests/unit/modules/test_win_certutil.py b/tests/unit/modules/test_win_certutil.py index 8289a2333e8..28d76d6145d 100644 --- a/tests/unit/modules/test_win_certutil.py +++ b/tests/unit/modules/test_win_certutil.py @@ -24,34 +24,54 @@ class CertUtilTestCase(TestCase, LoaderModuleMockMixin): ''' Test getting the serial number from a certificate ''' - expected = 'XYZABC' - mock = MagicMock(return_value='CertInfo\r\nSerial: XYZABC\r\nOtherStuff') + expected = '180720d39cd2db3244ba037417241e90' + mock = MagicMock(return_value=( + 'CertInfo\r\n' + 'Cert Serial Number: 180720d39cd2db3244ba037417241e90\r\n' + '\r\n' + 'OtherStuff')) with patch.dict(certutil.__salt__, {'cmd.run': mock}): out = certutil.get_cert_serial('/path/to/cert.cer') - mock.assert_called_once_with('certutil.exe -verify /path/to/cert.cer') + mock.assert_called_once_with( + 'certutil.exe -silent -verify /path/to/cert.cer') self.assertEqual(expected, out) def test_get_serials(self): ''' - Test getting the all the serial numbers from a store + Test getting all the serial numbers from a store ''' - expected = ['XYZABC', '123456'] - mock = MagicMock(return_value='CertInfo\r\nSerial Number: XYZABC\r\nSerial Number: 123456\r\n') + expected = ['180720d39cd2db3244ba037417241e90', + '1768ac4e5b72bf1d0df0df118b34b959'] + mock = MagicMock(return_value=( + 'CertInfo\r\n' + '================ Certificate 0 ================\r\n' + 'Serial Number: 180720d39cd2db3244ba037417241e90\r\n' + 'OtherStuff\r\n' + '\r\n' + '================ Certificate 1 ================\r\n' + 'Serial Number: 1768ac4e5b72bf1d0df0df118b34b959\r\n' + 'OtherStuff')) with patch.dict(certutil.__salt__, {'cmd.run': mock}): out = certutil.get_stored_cert_serials('TrustedPublisher') - mock.assert_called_once_with('certutil.exe -store TrustedPublisher') + mock.assert_called_once_with( + 'certutil.exe -store TrustedPublisher') self.assertEqual(expected, out) def test_add_store(self): ''' Test adding a certificate to a specific store ''' - cmd_mock = MagicMock(return_value='CertInfo\r\nSerial: XYZABC\r\nOtherStuff') + cmd_mock = MagicMock(return_value=( + 'CertInfo\r\n' + '================ Certificate 0 ================\r\n' + 'Serial Number: 180720d39cd2db3244ba037417241e90\r\n' + 'OtherStuff')) cache_mock = MagicMock(return_value='/tmp/cert.cer') with patch.dict(certutil.__salt__, {'cmd.run': cmd_mock, 'cp.cache_file': cache_mock}): certutil.add_store('salt://path/to/file', 'TrustedPublisher') - cmd_mock.assert_called_once_with('certutil.exe -addstore TrustedPublisher /tmp/cert.cer') + cmd_mock.assert_called_once_with( + 'certutil.exe -addstore TrustedPublisher /tmp/cert.cer') cache_mock.assert_called_once_with('salt://path/to/file', 'base') def test_del_store(self): @@ -59,11 +79,16 @@ class CertUtilTestCase(TestCase, LoaderModuleMockMixin): Test removing a certificate to a specific store ''' with patch('salt.modules.win_certutil.get_cert_serial') as cert_serial_mock: - cmd_mock = MagicMock(return_value='CertInfo\r\nSerial: XYZABC\r\nOtherStuff') + cmd_mock = MagicMock(return_value=( + 'CertInfo\r\n' + '================ Certificate 0 ================\r\n' + 'Serial Number: 180720d39cd2db3244ba037417241e90\r\n' + 'OtherStuff')) cache_mock = MagicMock(return_value='/tmp/cert.cer') - cert_serial_mock.return_value = "ABCDEF" + cert_serial_mock.return_value = 'ABCDEF' with patch.dict(certutil.__salt__, {'cmd.run': cmd_mock, 'cp.cache_file': cache_mock}): certutil.del_store('salt://path/to/file', 'TrustedPublisher') - cmd_mock.assert_called_once_with('certutil.exe -delstore TrustedPublisher ABCDEF') + cmd_mock.assert_called_once_with( + 'certutil.exe -delstore TrustedPublisher ABCDEF') cache_mock.assert_called_once_with('salt://path/to/file', 'base') diff --git a/tests/unit/modules/test_win_service.py b/tests/unit/modules/test_win_service.py index e179d58d2f9..79ac4d04a0d 100644 --- a/tests/unit/modules/test_win_service.py +++ b/tests/unit/modules/test_win_service.py @@ -214,7 +214,8 @@ class WinServiceTestCase(TestCase, LoaderModuleMockMixin): Test to enable the named service to start at boot ''' mock_modify = MagicMock(return_value=True) - mock_info = MagicMock(return_value={'StartType': 'Auto'}) + mock_info = MagicMock(return_value={'StartType': 'Auto', + 'StartTypeDelayed': False}) with patch.object(win_service, 'modify', mock_modify): with patch.object(win_service, 'info', mock_info): self.assertTrue(win_service.enable('spongebob')) diff --git a/tests/unit/modules/test_win_status.py b/tests/unit/modules/test_win_status.py index 3c072b63c05..be46f428ba1 100644 --- a/tests/unit/modules/test_win_status.py +++ b/tests/unit/modules/test_win_status.py @@ -6,11 +6,11 @@ import sys import types # Import Salt libs -import salt.ext.six as six +from salt.ext import six # Import Salt Testing libs from tests.support.unit import skipIf, TestCase -from tests.support.mock import NO_MOCK, Mock, patch, ANY +from tests.support.mock import NO_MOCK, NO_MOCK_REASON, Mock, patch, ANY # wmi and pythoncom modules are platform specific... wmi = types.ModuleType('wmi') @@ -29,7 +29,9 @@ if NO_MOCK is False: import salt.modules.win_status as status -@skipIf(NO_MOCK or sys.stdin.encoding != 'UTF8', 'Mock is not installed or encoding not supported') +@skipIf(NO_MOCK, NO_MOCK_REASON) +@skipIf(sys.stdin.encoding != 'UTF-8', 'UTF-8 encoding required for this test is not supported') +@skipIf(status.HAS_WMI is False, 'This test requires Windows') class TestProcsBase(TestCase): def __init__(self, *args, **kwargs): TestCase.__init__(self, *args, **kwargs) diff --git a/tests/unit/modules/test_win_system.py b/tests/unit/modules/test_win_system.py index 3506a4ddf63..8b8a05356f7 100644 --- a/tests/unit/modules/test_win_system.py +++ b/tests/unit/modules/test_win_system.py @@ -67,30 +67,31 @@ class WinSystemTestCase(TestCase, LoaderModuleMockMixin): ''' Test to reboot the system ''' - mock = MagicMock(return_value='salt') - with patch.dict(win_system.__salt__, {'cmd.run': mock}): - self.assertEqual(win_system.reboot(), 'salt') - mock.assert_called_once_with(['shutdown', '/r', '/t', '300'], python_shell=False) + with patch('salt.modules.win_system.shutdown', + MagicMock(return_value=True)) as shutdown: + self.assertEqual(win_system.reboot(), True) @skipIf(not win_system.HAS_WIN32NET_MODS, 'this test needs the w32net library') def test_reboot_with_timeout_in_minutes(self): ''' Test to reboot the system with a timeout ''' - mock = MagicMock(return_value='salt') - with patch.dict(win_system.__salt__, {'cmd.run': mock}): - self.assertEqual(win_system.reboot(5, in_seconds=False), 'salt') - mock.assert_called_once_with(['shutdown', '/r', '/t', '300'], python_shell=False) + with patch('salt.modules.win_system.shutdown', + MagicMock(return_value=True)) as shutdown: + self.assertEqual(win_system.reboot(5, in_seconds=False), True) + shutdown.assert_called_with(timeout=5, in_seconds=False, reboot=True, + only_on_pending_reboot=False) @skipIf(not win_system.HAS_WIN32NET_MODS, 'this test needs the w32net library') def test_reboot_with_timeout_in_seconds(self): ''' Test to reboot the system with a timeout ''' - mock = MagicMock(return_value='salt') - with patch.dict(win_system.__salt__, {'cmd.run': mock}): - self.assertEqual(win_system.reboot(5, in_seconds=True), 'salt') - mock.assert_called_once_with(['shutdown', '/r', '/t', '5'], python_shell=False) + with patch('salt.modules.win_system.shutdown', + MagicMock(return_value=True)) as shutdown: + self.assertEqual(win_system.reboot(5, in_seconds=True), True) + shutdown.assert_called_with(timeout=5, in_seconds=True, reboot=True, + only_on_pending_reboot=False) @skipIf(not win_system.HAS_WIN32NET_MODS, 'this test needs the w32net library') def test_reboot_with_wait(self): @@ -98,50 +99,49 @@ class WinSystemTestCase(TestCase, LoaderModuleMockMixin): Test to reboot the system with a timeout and wait for it to finish ''' - mock = MagicMock(return_value='salt') - sleep_mock = MagicMock(return_value='salt') - with patch.dict(win_system.__salt__, {'cmd.run': mock}): - with patch('time.sleep', sleep_mock): - self.assertEqual(win_system.reboot(wait_for_reboot=True), 'salt') - mock.assert_called_once_with(['shutdown', '/r', '/t', '300'], python_shell=False) - sleep_mock.assert_called_once_with(330) + with patch('salt.modules.win_system.shutdown', + MagicMock(return_value=True)), \ + patch('salt.modules.win_system.time.sleep', + MagicMock()) as time: + self.assertEqual(win_system.reboot(wait_for_reboot=True), True) + time.assert_called_with(330) @skipIf(not win_system.HAS_WIN32NET_MODS, 'this test needs the w32net library') def test_shutdown(self): ''' Test to shutdown a running system ''' - mock = MagicMock(return_value='salt') - with patch.dict(win_system.__salt__, {'cmd.run': mock}): - self.assertEqual(win_system.shutdown(), 'salt') + with patch('salt.modules.win_system.win32api.InitiateSystemShutdown', + MagicMock()): + self.assertEqual(win_system.shutdown(), True) @skipIf(not win_system.HAS_WIN32NET_MODS, 'this test needs the w32net library') def test_shutdown_hard(self): ''' Test to shutdown a running system with no timeout or warning ''' - mock = MagicMock(return_value='salt') - with patch.dict(win_system.__salt__, {'cmd.run': mock}): - self.assertEqual(win_system.shutdown_hard(), 'salt') + with patch('salt.modules.win_system.shutdown', + MagicMock(return_value=True)) as shutdown: + self.assertEqual(win_system.shutdown_hard(), True) + shutdown.assert_called_with(timeout=0) @skipIf(not win_system.HAS_WIN32NET_MODS, 'this test needs the w32net library') def test_set_computer_name(self): ''' Test to set the Windows computer name ''' - mock = MagicMock(side_effect=[{'Computer Name': {'Current': ""}, - 'ReturnValue = 0;': True}, - {'Computer Name': {'Current': 'salt'}}]) - with patch.dict(win_system.__salt__, {'cmd.run': mock}): - mock = MagicMock(return_value='salt') - with patch.object(win_system, 'get_computer_name', mock): - mock = MagicMock(return_value=True) - with patch.object(win_system, - 'get_pending_computer_name', mock): - self.assertDictEqual(win_system.set_computer_name("salt"), + with patch('salt.modules.win_system.windll.kernel32.SetComputerNameExW', + MagicMock(return_value=True)): + with patch.object(win_system, 'get_computer_name', + MagicMock(return_value='salt')): + with patch.object(win_system, 'get_pending_computer_name', + MagicMock(return_value='salt_new')): + self.assertDictEqual(win_system.set_computer_name("salt_new"), {'Computer Name': {'Current': 'salt', - 'Pending': True}}) - + 'Pending': 'salt_new'}}) + # Test set_computer_name failure + with patch('salt.modules.win_system.windll.kernel32.SetComputerNameExW', + MagicMock(return_value=False)): self.assertFalse(win_system.set_computer_name("salt")) @skipIf(not win_system.HAS_WIN32NET_MODS, 'this test needs the w32net library') @@ -149,25 +149,25 @@ class WinSystemTestCase(TestCase, LoaderModuleMockMixin): ''' Test to get a pending computer name. ''' - mock = MagicMock(return_value='salt') - with patch.object(win_system, 'get_computer_name', mock): - mock = MagicMock(side_effect=['salt0', - 'ComputerName REG_SZ (salt)']) - with patch.dict(win_system.__salt__, {'cmd.run': mock}): + with patch.object(win_system, 'get_computer_name', + MagicMock(return_value='salt')): + reg_mock = MagicMock(return_value={'vdata': 'salt'}) + with patch.dict(win_system.__salt__, {'reg.read_value': reg_mock}): self.assertFalse(win_system.get_pending_computer_name()) + reg_mock = MagicMock(return_value={'vdata': 'salt_pending'}) + with patch.dict(win_system.__salt__, {'reg.read_value': reg_mock}): self.assertEqual(win_system.get_pending_computer_name(), - '(salt)') + 'salt_pending') @skipIf(not win_system.HAS_WIN32NET_MODS, 'this test needs the w32net library') def test_get_computer_name(self): ''' Test to get the Windows computer name ''' - mock = MagicMock(side_effect=['Server Name Salt', 'Salt']) - with patch.dict(win_system.__salt__, {'cmd.run': mock}): - self.assertEqual(win_system.get_computer_name(), 'Salt') - + with patch('salt.modules.win_system.win32api.GetComputerNameEx', + MagicMock(side_effect=['computer name', ''])): + self.assertEqual(win_system.get_computer_name(), 'computer name') self.assertFalse(win_system.get_computer_name()) @skipIf(not win_system.HAS_WIN32NET_MODS, 'this test needs the w32net library') @@ -189,10 +189,10 @@ class WinSystemTestCase(TestCase, LoaderModuleMockMixin): ''' Test to get the Windows computer description ''' - mock = MagicMock(side_effect=['Server Comment Salt', 'Salt']) - with patch.dict(win_system.__salt__, {'cmd.run': mock}): - self.assertEqual(win_system.get_computer_desc(), 'Salt') - + with patch('salt.modules.win_system.get_system_info', + MagicMock(side_effect=[{'description': 'salt description'}, + {'description': None}])): + self.assertEqual(win_system.get_computer_desc(), 'salt description') self.assertFalse(win_system.get_computer_desc()) @skipIf(not win_system.HAS_WIN32NET_MODS, 'this test needs w32net and other windows libraries') @@ -200,17 +200,20 @@ class WinSystemTestCase(TestCase, LoaderModuleMockMixin): ''' Test to join a computer to an Active Directory domain ''' - mock = MagicMock(side_effect=[{'ReturnValue = 0;': True}, - {'Salt': True}]) - with patch.dict(win_system.__salt__, {'cmd.run': mock}): - self.assertDictEqual(win_system.join_domain("saltstack", - "salt", - "salt@123"), - {'Domain': 'saltstack'}) + with patch('salt.modules.win_system._join_domain', + MagicMock(return_value=0)): + with patch('salt.modules.win_system.get_domain_workgroup', + MagicMock(return_value={'Workgroup': 'Workgroup'})): + self.assertDictEqual( + win_system.join_domain( + "saltstack", "salt", "salt@123"), + {'Domain': 'saltstack', 'Restart': False}) - self.assertFalse(win_system.join_domain("saltstack", - "salt", - "salt@123")) + with patch('salt.modules.win_system.get_domain_workgroup', + MagicMock(return_value={'Domain': 'saltstack'})): + self.assertEqual( + win_system.join_domain("saltstack", "salt", "salt@123"), + 'Already joined to saltstack') def test_get_system_time(self): ''' @@ -230,13 +233,10 @@ class WinSystemTestCase(TestCase, LoaderModuleMockMixin): ''' Test to set system time ''' - mock = MagicMock(side_effect=[False, True]) - with patch.object(win_system, '_validate_time', mock): + with patch('salt.modules.win_system.set_system_date_time', + MagicMock(side_effect=[False, True])): self.assertFalse(win_system.set_system_time("11:31:15 AM")) - - mock = MagicMock(return_value=True) - with patch.dict(win_system.__salt__, {'cmd.retcode': mock}): - self.assertFalse(win_system.set_system_time("11:31:15 AM")) + self.assertTrue(win_system.set_system_time("11:31:15 AM")) def test_get_system_date(self): ''' @@ -250,13 +250,10 @@ class WinSystemTestCase(TestCase, LoaderModuleMockMixin): ''' Test to set system date ''' - mock = MagicMock(side_effect=[False, True]) - with patch.object(win_system, '_validate_date', mock): + with patch('salt.modules.win_system.set_system_date_time', + MagicMock(side_effect=[False, True])): self.assertFalse(win_system.set_system_date("03-28-13")) - - mock = MagicMock(return_value=True) - with patch.dict(win_system.__salt__, {'cmd.retcode': mock}): - self.assertFalse(win_system.set_system_date("03-28-13")) + self.assertTrue(win_system.set_system_date("03-28-13")) def test_start_time_service(self): ''' diff --git a/tests/unit/modules/test_xapi.py b/tests/unit/modules/test_xapi.py index 0218ecb270e..027301af3a2 100644 --- a/tests/unit/modules/test_xapi.py +++ b/tests/unit/modules/test_xapi.py @@ -367,14 +367,14 @@ class XapiTestCase(TestCase, LoaderModuleMockMixin): self.assertFalse(xapi.is_hyper()) with patch.dict(xapi.__grains__, {'virtual_subtype': 'Xen Dom0'}): - with patch('salt.utils.fopen', mock_open(read_data="salt")): + with patch('salt.utils.files.fopen', mock_open(read_data="salt")): self.assertFalse(xapi.is_hyper()) - with patch('salt.utils.fopen', mock_open()) as mock_read: + with patch('salt.utils.files.fopen', mock_open()) as mock_read: mock_read.side_effect = IOError self.assertFalse(xapi.is_hyper()) - with patch('salt.utils.fopen', mock_open(read_data="xen_")): + with patch('salt.utils.files.fopen', mock_open(read_data="xen_")): with patch.dict(xapi.__grains__, {'ps': 'salt'}): mock = MagicMock(return_value={'xenstore': 'salt'}) with patch.dict(xapi.__salt__, {'cmd.run': mock}): diff --git a/tests/unit/modules/test_yumpkg.py b/tests/unit/modules/test_yumpkg.py new file mode 100644 index 00000000000..84e0a0ac52f --- /dev/null +++ b/tests/unit/modules/test_yumpkg.py @@ -0,0 +1,188 @@ +# -*- coding: utf-8 -*- + +# Import Python Libs +from __future__ import absolute_import +import os + +# Import Salt Testing Libs +from tests.support.mixins import LoaderModuleMockMixin +from tests.support.unit import TestCase, skipIf +from tests.support.mock import ( + MagicMock, + patch, + NO_MOCK, + NO_MOCK_REASON +) + +# Import Salt libs +import salt.modules.yumpkg as yumpkg +import salt.modules.pkg_resource as pkg_resource + + +@skipIf(NO_MOCK, NO_MOCK_REASON) +class YumTestCase(TestCase, LoaderModuleMockMixin): + ''' + Test cases for salt.modules.yumpkg + ''' + def setup_loader_modules(self): + return {yumpkg: {'rpm': None}} + + def test_list_pkgs(self): + ''' + Test packages listing. + + :return: + ''' + def _add_data(data, key, value): + data.setdefault(key, []).append(value) + + rpm_out = [ + 'python-urlgrabber_|-(none)_|-3.10_|-8.el7_|-noarch_|-(none)_|-1487838471', + 'alsa-lib_|-(none)_|-1.1.1_|-1.el7_|-x86_64_|-(none)_|-1487838475', + 'gnupg2_|-(none)_|-2.0.22_|-4.el7_|-x86_64_|-(none)_|-1487838477', + 'rpm-python_|-(none)_|-4.11.3_|-21.el7_|-x86_64_|-(none)_|-1487838477', + 'pygpgme_|-(none)_|-0.3_|-9.el7_|-x86_64_|-(none)_|-1487838478', + 'yum_|-(none)_|-3.4.3_|-150.el7.centos_|-noarch_|-(none)_|-1487838479', + 'lzo_|-(none)_|-2.06_|-8.el7_|-x86_64_|-(none)_|-1487838479', + 'qrencode-libs_|-(none)_|-3.4.1_|-3.el7_|-x86_64_|-(none)_|-1487838480', + 'ustr_|-(none)_|-1.0.4_|-16.el7_|-x86_64_|-(none)_|-1487838480', + 'shadow-utils_|-2_|-4.1.5.1_|-24.el7_|-x86_64_|-(none)_|-1487838481', + 'util-linux_|-(none)_|-2.23.2_|-33.el7_|-x86_64_|-(none)_|-1487838484', + 'openssh_|-(none)_|-6.6.1p1_|-33.el7_3_|-x86_64_|-(none)_|-1487838485', + 'virt-what_|-(none)_|-1.13_|-8.el7_|-x86_64_|-(none)_|-1487838486', + ] + with patch.dict(yumpkg.__grains__, {'osarch': 'x86_64'}), \ + patch.dict(yumpkg.__salt__, {'cmd.run': MagicMock(return_value=os.linesep.join(rpm_out))}), \ + patch.dict(yumpkg.__salt__, {'pkg_resource.add_pkg': _add_data}), \ + patch.dict(yumpkg.__salt__, {'pkg_resource.format_pkg_list': pkg_resource.format_pkg_list}), \ + patch.dict(yumpkg.__salt__, {'pkg_resource.stringify': MagicMock()}): + pkgs = yumpkg.list_pkgs(versions_as_list=True) + for pkg_name, pkg_version in { + 'python-urlgrabber': '3.10-8.el7', + 'alsa-lib': '1.1.1-1.el7', + 'gnupg2': '2.0.22-4.el7', + 'rpm-python': '4.11.3-21.el7', + 'pygpgme': '0.3-9.el7', + 'yum': '3.4.3-150.el7.centos', + 'lzo': '2.06-8.el7', + 'qrencode-libs': '3.4.1-3.el7', + 'ustr': '1.0.4-16.el7', + 'shadow-utils': '2:4.1.5.1-24.el7', + 'util-linux': '2.23.2-33.el7', + 'openssh': '6.6.1p1-33.el7_3', + 'virt-what': '1.13-8.el7'}.items(): + self.assertTrue(pkgs.get(pkg_name)) + self.assertEqual(pkgs[pkg_name], [pkg_version]) + + def test_list_pkgs_with_attr(self): + ''' + Test packages listing with the attr parameter + + :return: + ''' + def _add_data(data, key, value): + data.setdefault(key, []).append(value) + + rpm_out = [ + 'python-urlgrabber_|-(none)_|-3.10_|-8.el7_|-noarch_|-(none)_|-1487838471', + 'alsa-lib_|-(none)_|-1.1.1_|-1.el7_|-x86_64_|-(none)_|-1487838475', + 'gnupg2_|-(none)_|-2.0.22_|-4.el7_|-x86_64_|-(none)_|-1487838477', + 'rpm-python_|-(none)_|-4.11.3_|-21.el7_|-x86_64_|-(none)_|-1487838477', + 'pygpgme_|-(none)_|-0.3_|-9.el7_|-x86_64_|-(none)_|-1487838478', + 'yum_|-(none)_|-3.4.3_|-150.el7.centos_|-noarch_|-(none)_|-1487838479', + 'lzo_|-(none)_|-2.06_|-8.el7_|-x86_64_|-(none)_|-1487838479', + 'qrencode-libs_|-(none)_|-3.4.1_|-3.el7_|-x86_64_|-(none)_|-1487838480', + 'ustr_|-(none)_|-1.0.4_|-16.el7_|-x86_64_|-(none)_|-1487838480', + 'shadow-utils_|-2_|-4.1.5.1_|-24.el7_|-x86_64_|-(none)_|-1487838481', + 'util-linux_|-(none)_|-2.23.2_|-33.el7_|-x86_64_|-(none)_|-1487838484', + 'openssh_|-(none)_|-6.6.1p1_|-33.el7_3_|-x86_64_|-(none)_|-1487838485', + 'virt-what_|-(none)_|-1.13_|-8.el7_|-x86_64_|-(none)_|-1487838486', + ] + with patch.dict(yumpkg.__grains__, {'osarch': 'x86_64'}), \ + patch.dict(yumpkg.__salt__, {'cmd.run': MagicMock(return_value=os.linesep.join(rpm_out))}), \ + patch.dict(yumpkg.__salt__, {'pkg_resource.add_pkg': _add_data}), \ + patch.dict(yumpkg.__salt__, {'pkg_resource.format_pkg_list': pkg_resource.format_pkg_list}), \ + patch.dict(yumpkg.__salt__, {'pkg_resource.stringify': MagicMock()}): + pkgs = yumpkg.list_pkgs(attr=['epoch', 'release', 'arch', 'install_date_time_t']) + for pkg_name, pkg_attr in { + 'python-urlgrabber': { + 'version': '3.10', + 'release': '8.el7', + 'arch': 'noarch', + 'install_date_time_t': 1487838471, + }, + 'alsa-lib': { + 'version': '1.1.1', + 'release': '1.el7', + 'arch': 'x86_64', + 'install_date_time_t': 1487838475, + }, + 'gnupg2': { + 'version': '2.0.22', + 'release': '4.el7', + 'arch': 'x86_64', + 'install_date_time_t': 1487838477, + }, + 'rpm-python': { + 'version': '4.11.3', + 'release': '21.el7', + 'arch': 'x86_64', + 'install_date_time_t': 1487838477, + }, + 'pygpgme': { + 'version': '0.3', + 'release': '9.el7', + 'arch': 'x86_64', + 'install_date_time_t': 1487838478, + }, + 'yum': { + 'version': '3.4.3', + 'release': '150.el7.centos', + 'arch': 'noarch', + 'install_date_time_t': 1487838479, + }, + 'lzo': { + 'version': '2.06', + 'release': '8.el7', + 'arch': 'x86_64', + 'install_date_time_t': 1487838479, + }, + 'qrencode-libs': { + 'version': '3.4.1', + 'release': '3.el7', + 'arch': 'x86_64', + 'install_date_time_t': 1487838480, + }, + 'ustr': { + 'version': '1.0.4', + 'release': '16.el7', + 'arch': 'x86_64', + 'install_date_time_t': 1487838480, + }, + 'shadow-utils': { + 'epoch': '2', + 'version': '4.1.5.1', + 'release': '24.el7', + 'arch': 'x86_64', + 'install_date_time_t': 1487838481, + }, + 'util-linux': { + 'version': '2.23.2', + 'release': '33.el7', + 'arch': 'x86_64', + 'install_date_time_t': 1487838484, + }, + 'openssh': { + 'version': '6.6.1p1', + 'release': '33.el7_3', + 'arch': 'x86_64', + 'install_date_time_t': 1487838485, + }, + 'virt-what': { + 'version': '1.13', + 'release': '8.el7', + 'install_date_time_t': 1487838486, + 'arch': 'x86_64', + }}.items(): + self.assertTrue(pkgs.get(pkg_name)) + self.assertEqual(pkgs[pkg_name], [pkg_attr]) diff --git a/tests/unit/modules/test_zcbuildout.py b/tests/unit/modules/test_zcbuildout.py index c67cec9db39..a8b66164cf2 100644 --- a/tests/unit/modules/test_zcbuildout.py +++ b/tests/unit/modules/test_zcbuildout.py @@ -9,7 +9,7 @@ import shutil # Import 3rd-party libs # pylint: disable=import-error,no-name-in-module,redefined-builtin -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves.urllib.error import URLError from salt.ext.six.moves.urllib.request import urlopen # pylint: enable=import-error,no-name-in-module,redefined-builtin @@ -21,7 +21,8 @@ from tests.support.unit import TestCase, skipIf from tests.support.helpers import requires_network, skip_if_binaries_missing # Import Salt libs -import salt.utils +import salt.utils.files +import salt.utils.path import salt.modules.zcbuildout as buildout import salt.modules.cmdmod as cmd @@ -47,7 +48,7 @@ log = logging.getLogger(__name__) def download_to(url, dest): - with salt.utils.fopen(dest, 'w') as fic: + with salt.utils.files.fopen(dest, 'w') as fic: fic.write(urlopen(url, timeout=10).read()) @@ -87,7 +88,7 @@ class Base(TestCase, LoaderModuleMockMixin): '{0} --no-site-packages {1};' '{1}/bin/pip install -U setuptools; ' '{1}/bin/easy_install -U distribute;').format( - salt.utils.which_bin(KNOWN_VIRTUALENV_BINARY_NAMES), + salt.utils.path.which_bin(KNOWN_VIRTUALENV_BINARY_NAMES), cls.ppy_st ) ) @@ -120,7 +121,7 @@ class Base(TestCase, LoaderModuleMockMixin): @skipIf(True, 'These tests are not running reliably') -@skipIf(salt.utils.which_bin(KNOWN_VIRTUALENV_BINARY_NAMES) is None, +@skipIf(salt.utils.path.which_bin(KNOWN_VIRTUALENV_BINARY_NAMES) is None, 'The \'virtualenv\' packaged needs to be installed') @skip_if_binaries_missing(['tar']) class BuildoutTestCase(Base): @@ -294,14 +295,14 @@ class BuildoutTestCase(Base): bpy = os.path.join(b_dir, 'bootstrap.py') buildout.upgrade_bootstrap(b_dir) time1 = os.stat(bpy).st_mtime - with salt.utils.fopen(bpy) as fic: + with salt.utils.files.fopen(bpy) as fic: data = fic.read() self.assertTrue('setdefaulttimeout(2)' in data) flag = os.path.join(b_dir, '.buildout', '2.updated_bootstrap') self.assertTrue(os.path.exists(flag)) buildout.upgrade_bootstrap(b_dir, buildout_ver=1) time2 = os.stat(bpy).st_mtime - with salt.utils.fopen(bpy) as fic: + with salt.utils.files.fopen(bpy) as fic: data = fic.read() self.assertTrue('setdefaulttimeout(2)' in data) flag = os.path.join(b_dir, '.buildout', '1.updated_bootstrap') @@ -312,7 +313,7 @@ class BuildoutTestCase(Base): self.assertEqual(time2, time3) -@skipIf(salt.utils.which_bin(KNOWN_VIRTUALENV_BINARY_NAMES) is None, +@skipIf(salt.utils.path.which_bin(KNOWN_VIRTUALENV_BINARY_NAMES) is None, 'The \'virtualenv\' packaged needs to be installed') @skipIf(True, 'These tests are not running reliably') class BuildoutOnlineTestCase(Base): @@ -328,14 +329,14 @@ class BuildoutOnlineTestCase(Base): try: ret20 = buildout._Popen(( '{0} --no-site-packages --no-setuptools --no-pip {1}'.format( - salt.utils.which_bin(KNOWN_VIRTUALENV_BINARY_NAMES), + salt.utils.path.which_bin(KNOWN_VIRTUALENV_BINARY_NAMES), cls.ppy_dis ) )) except buildout._BuildoutError: ret20 = buildout._Popen(( '{0} --no-site-packages {1}'.format( - salt.utils.which_bin(KNOWN_VIRTUALENV_BINARY_NAMES), + salt.utils.path.which_bin(KNOWN_VIRTUALENV_BINARY_NAMES), cls.ppy_dis )) ) @@ -356,14 +357,14 @@ class BuildoutOnlineTestCase(Base): try: ret3 = buildout._Popen(( '{0} --no-site-packages --no-setuptools --no-pip {1}'.format( - salt.utils.which_bin(KNOWN_VIRTUALENV_BINARY_NAMES), + salt.utils.path.which_bin(KNOWN_VIRTUALENV_BINARY_NAMES), cls.ppy_blank ) )) except buildout._BuildoutError: ret3 = buildout._Popen(( '{0} --no-site-packages {1}'.format( - salt.utils.which_bin(KNOWN_VIRTUALENV_BINARY_NAMES), + salt.utils.path.which_bin(KNOWN_VIRTUALENV_BINARY_NAMES), cls.ppy_blank ) )) diff --git a/tests/unit/modules/test_zfs.py b/tests/unit/modules/test_zfs.py index 5d421b7288f..3bc0c614f5b 100644 --- a/tests/unit/modules/test_zfs.py +++ b/tests/unit/modules/test_zfs.py @@ -340,7 +340,7 @@ class ZfsTestCase(TestCase, LoaderModuleMockMixin): ''' Tests zfs bookmark success ''' - with patch('salt.utils.which', MagicMock(return_value='/usr/bin/man')): + with patch('salt.utils.path.which', MagicMock(return_value='/usr/bin/man')): res = {'myzpool/mydataset@yesterday': 'bookmarked as myzpool/mydataset#important'} ret = {'pid': 20990, 'retcode': 0, 'stderr': '', 'stdout': ''} mock_cmd = MagicMock(return_value=ret) diff --git a/tests/unit/modules/test_zypper.py b/tests/unit/modules/test_zypper.py index a02ce4cb733..a6a0c887945 100644 --- a/tests/unit/modules/test_zypper.py +++ b/tests/unit/modules/test_zypper.py @@ -21,13 +21,14 @@ from tests.support.mock import ( ) # Import Salt libs -import salt.utils +import salt.utils.files import salt.modules.zypper as zypper +import salt.modules.pkg_resource as pkg_resource from salt.exceptions import CommandExecutionError # Import 3rd-party libs from salt.ext.six.moves import configparser -import salt.ext.six as six +from salt.ext import six import salt.utils.pkg @@ -46,7 +47,7 @@ def get_test_data(filename): ''' Return static test data ''' - with salt.utils.fopen( + with salt.utils.files.fopen( os.path.join( os.path.join( os.path.dirname(os.path.abspath(__file__)), 'zypp'), filename)) as rfh: @@ -391,17 +392,23 @@ class ZypperTestCase(TestCase, LoaderModuleMockMixin): with patch('salt.modules.zypper.list_pkgs', MagicMock(side_effect=[{"vim": "1.1"}, {"vim": "1.1"}])): ret = zypper.upgrade(dist_upgrade=True, dryrun=True) zypper_mock.assert_any_call('dist-upgrade', '--auto-agree-with-licenses', '--dry-run') - zypper_mock.assert_any_call('dist-upgrade', '--auto-agree-with-licenses', '--dry-run', '--debug-solver') + zypper_mock.assert_any_call('dist-upgrade', '--auto-agree-with-licenses', + '--dry-run', '--debug-solver') with patch('salt.modules.zypper.list_pkgs', MagicMock(side_effect=[{"vim": "1.1"}, {"vim": "1.1"}])): - ret = zypper.upgrade(dist_upgrade=True, dryrun=True, fromrepo=["Dummy", "Dummy2"], novendorchange=True) - zypper_mock.assert_any_call('dist-upgrade', '--auto-agree-with-licenses', '--dry-run', '--from', "Dummy", '--from', 'Dummy2', '--no-allow-vendor-change') - zypper_mock.assert_any_call('dist-upgrade', '--auto-agree-with-licenses', '--dry-run', '--from', "Dummy", '--from', 'Dummy2', '--no-allow-vendor-change', '--debug-solver') + ret = zypper.upgrade(dist_upgrade=True, dryrun=True, + fromrepo=["Dummy", "Dummy2"], novendorchange=True) + zypper_mock.assert_any_call('dist-upgrade', '--auto-agree-with-licenses', '--dry-run', + '--from', "Dummy", '--from', 'Dummy2', '--no-allow-vendor-change') + zypper_mock.assert_any_call('dist-upgrade', '--auto-agree-with-licenses', '--dry-run', + '--from', "Dummy", '--from', 'Dummy2', '--no-allow-vendor-change', + '--debug-solver') with patch('salt.modules.zypper.list_pkgs', MagicMock(side_effect=[{"vim": "1.1"}, {"vim": "1.2"}])): ret = zypper.upgrade(dist_upgrade=True, fromrepo=["Dummy", "Dummy2"], novendorchange=True) self.assertDictEqual(ret, {"vim": {"old": "1.1", "new": "1.2"}}) - zypper_mock.assert_any_call('dist-upgrade', '--auto-agree-with-licenses', '--from', "Dummy", '--from', 'Dummy2', '--no-allow-vendor-change') + zypper_mock.assert_any_call('dist-upgrade', '--auto-agree-with-licenses', '--from', "Dummy", + '--from', 'Dummy2', '--no-allow-vendor-change') def test_upgrade_kernel(self): ''' @@ -412,7 +419,8 @@ class ZypperTestCase(TestCase, LoaderModuleMockMixin): with patch.dict(zypper.__grains__, {'osrelease_info': [12, 1]}), \ patch('salt.modules.zypper.refresh_db', MagicMock(return_value=True)), \ patch('salt.modules.zypper._systemd_scope', MagicMock(return_value=False)): - with patch.dict(zypper.__salt__, {'pkg_resource.parse_targets': MagicMock(return_value=(['kernel-default'], None))}): + with patch.dict(zypper.__salt__, {'pkg_resource.parse_targets': MagicMock(return_value=(['kernel-default'], + None))}): with patch('salt.modules.zypper.__zypper__.noraise.call', MagicMock()): with patch('salt.modules.zypper.list_pkgs', MagicMock(side_effect=[ {"kernel-default": "3.12.49-11.1"}, {"kernel-default": "3.12.49-11.1,3.12.51-60.20.2"}])): @@ -455,7 +463,8 @@ Repository 'DUMMY' not found by its alias, number, or URI. ret = zypper.upgrade(dist_upgrade=True, fromrepo=["DUMMY"]) self.assertEqual(cmd_exc.exception.info['changes'], {}) self.assertEqual(cmd_exc.exception.info['result']['stdout'], zypper_out) - zypper_mock.noraise.call.assert_called_with('dist-upgrade', '--auto-agree-with-licenses', '--from', 'DUMMY') + zypper_mock.noraise.call.assert_called_with('dist-upgrade', '--auto-agree-with-licenses', + '--from', 'DUMMY') def test_upgrade_available(self): ''' @@ -478,30 +487,92 @@ Repository 'DUMMY' not found by its alias, number, or URI. :return: ''' def _add_data(data, key, value): - data[key] = value + data.setdefault(key, []).append(value) rpm_out = [ - 'protobuf-java_|-2.6.1_|-3.1.develHead_|-', - 'yast2-ftp-server_|-3.1.8_|-8.1_|-', - 'jose4j_|-0.4.4_|-2.1.develHead_|-', - 'apache-commons-cli_|-1.2_|-1.233_|-', - 'jakarta-commons-discovery_|-0.4_|-129.686_|-', - 'susemanager-build-keys-web_|-12.0_|-5.1.develHead_|-', + 'protobuf-java_|-2.6.1_|-3.1.develHead_|-noarch_|-_|-1499257756', + 'yast2-ftp-server_|-3.1.8_|-8.1_|-x86_64_|-_|-1499257798', + 'jose4j_|-0.4.4_|-2.1.develHead_|-noarch_|-_|-1499257756', + 'apache-commons-cli_|-1.2_|-1.233_|-noarch_|-_|-1498636510', + 'jakarta-commons-discovery_|-0.4_|-129.686_|-noarch_|-_|-1498636511', + 'susemanager-build-keys-web_|-12.0_|-5.1.develHead_|-noarch_|-_|-1498636510', ] - with patch.dict(zypper.__salt__, {'cmd.run': MagicMock(return_value=os.linesep.join(rpm_out))}): - with patch.dict(zypper.__salt__, {'pkg_resource.add_pkg': _add_data}): - with patch.dict(zypper.__salt__, {'pkg_resource.sort_pkglist': MagicMock()}): - with patch.dict(zypper.__salt__, {'pkg_resource.stringify': MagicMock()}): - pkgs = zypper.list_pkgs() - for pkg_name, pkg_version in { - 'jakarta-commons-discovery': '0.4-129.686', - 'yast2-ftp-server': '3.1.8-8.1', - 'protobuf-java': '2.6.1-3.1.develHead', - 'susemanager-build-keys-web': '12.0-5.1.develHead', - 'apache-commons-cli': '1.2-1.233', - 'jose4j': '0.4.4-2.1.develHead'}.items(): - self.assertTrue(pkgs.get(pkg_name)) - self.assertEqual(pkgs[pkg_name], pkg_version) + with patch.dict(zypper.__salt__, {'cmd.run': MagicMock(return_value=os.linesep.join(rpm_out))}), \ + patch.dict(zypper.__salt__, {'pkg_resource.add_pkg': _add_data}), \ + patch.dict(zypper.__salt__, {'pkg_resource.format_pkg_list': pkg_resource.format_pkg_list}), \ + patch.dict(zypper.__salt__, {'pkg_resource.stringify': MagicMock()}): + pkgs = zypper.list_pkgs(versions_as_list=True) + for pkg_name, pkg_version in { + 'jakarta-commons-discovery': '0.4-129.686', + 'yast2-ftp-server': '3.1.8-8.1', + 'protobuf-java': '2.6.1-3.1.develHead', + 'susemanager-build-keys-web': '12.0-5.1.develHead', + 'apache-commons-cli': '1.2-1.233', + 'jose4j': '0.4.4-2.1.develHead'}.items(): + self.assertTrue(pkgs.get(pkg_name)) + self.assertEqual(pkgs[pkg_name], [pkg_version]) + + def test_list_pkgs_with_attr(self): + ''' + Test packages listing with the attr parameter + + :return: + ''' + def _add_data(data, key, value): + data.setdefault(key, []).append(value) + + rpm_out = [ + 'protobuf-java_|-2.6.1_|-3.1.develHead_|-noarch_|-_|-1499257756', + 'yast2-ftp-server_|-3.1.8_|-8.1_|-x86_64_|-_|-1499257798', + 'jose4j_|-0.4.4_|-2.1.develHead_|-noarch_|-_|-1499257756', + 'apache-commons-cli_|-1.2_|-1.233_|-noarch_|-_|-1498636510', + 'jakarta-commons-discovery_|-0.4_|-129.686_|-noarch_|-_|-1498636511', + 'susemanager-build-keys-web_|-12.0_|-5.1.develHead_|-noarch_|-_|-1498636510', + ] + with patch.dict(zypper.__salt__, {'cmd.run': MagicMock(return_value=os.linesep.join(rpm_out))}), \ + patch.dict(zypper.__salt__, {'pkg_resource.add_pkg': _add_data}), \ + patch.dict(zypper.__salt__, {'pkg_resource.format_pkg_list': pkg_resource.format_pkg_list}), \ + patch.dict(zypper.__salt__, {'pkg_resource.stringify': MagicMock()}): + pkgs = zypper.list_pkgs(attr=['epoch', 'release', 'arch', 'install_date_time_t']) + for pkg_name, pkg_attr in { + 'jakarta-commons-discovery': { + 'version': '0.4', + 'release': '129.686', + 'arch': 'noarch', + 'install_date_time_t': 1498636511, + }, + 'yast2-ftp-server': { + 'version': '3.1.8', + 'release': '8.1', + 'arch': 'x86_64', + 'install_date_time_t': 1499257798, + }, + 'protobuf-java': { + 'version': '2.6.1', + 'release': '3.1.develHead', + 'install_date_time_t': 1499257756, + 'arch': 'noarch', + }, + 'susemanager-build-keys-web': { + 'version': '12.0', + 'release': '5.1.develHead', + 'arch': 'noarch', + 'install_date_time_t': 1498636510, + }, + 'apache-commons-cli': { + 'version': '1.2', + 'release': '1.233', + 'arch': 'noarch', + 'install_date_time_t': 1498636510, + }, + 'jose4j': { + 'arch': 'noarch', + 'version': '0.4.4', + 'release': '2.1.develHead', + 'install_date_time_t': 1499257756, + }}.items(): + self.assertTrue(pkgs.get(pkg_name)) + self.assertEqual(pkgs[pkg_name], [pkg_attr]) def test_list_patches(self): ''' @@ -548,7 +619,8 @@ Repository 'DUMMY' not found by its alias, number, or URI. } } - with patch.dict(zypper.__salt__, {'lowpkg.bin_pkg_info': MagicMock(return_value={'name': 'test-package', 'version': '1.0'})}): + with patch.dict(zypper.__salt__, {'lowpkg.bin_pkg_info': MagicMock(return_value={'name': 'test-package', + 'version': '1.0'})}): list_downloaded = zypper.list_downloaded() self.assertEqual(len(list_downloaded), 1) self.assertDictEqual(list_downloaded, DOWNLOADED_RET) @@ -579,14 +651,19 @@ Repository 'DUMMY' not found by its alias, number, or URI. self.assertEqual(zypper.download("nmap", "foo"), test_out) @patch('salt.modules.zypper._systemd_scope', MagicMock(return_value=False)) - @patch('salt.modules.zypper.list_downloaded', MagicMock(side_effect=[{}, {'vim': {'1.1': {'path': '/foo/bar/test.rpm', 'size': 1234, 'creation_date_time_t': 1234567890, 'creation_date_time': '2009-02-13T23:31:30'}}}])) + @patch('salt.modules.zypper.list_downloaded', + MagicMock(side_effect=[{}, {'vim': {'1.1': {'path': '/foo/bar/test.rpm', + 'size': 1234, + 'creation_date_time_t': 1234567890, + 'creation_date_time': '2009-02-13T23:31:30'}}}])) def test_install_with_downloadonly(self): ''' Test a package installation with downloadonly=True. :return: ''' - with patch.dict(zypper.__salt__, {'pkg_resource.parse_targets': MagicMock(return_value=({'vim': None}, 'repository'))}): + with patch.dict(zypper.__salt__, + {'pkg_resource.parse_targets': MagicMock(return_value=({'vim': None}, 'repository'))}): with patch('salt.modules.zypper.__zypper__.noraise.call', MagicMock()) as zypper_mock: ret = zypper.install(pkgs=['vim'], downloadonly=True) zypper_mock.assert_called_once_with( @@ -597,17 +674,26 @@ Repository 'DUMMY' not found by its alias, number, or URI. '--download-only', 'vim' ) - self.assertDictEqual(ret, {'vim': {'new': {'1.1': {'path': '/foo/bar/test.rpm', 'size': 1234, 'creation_date_time_t': 1234567890, 'creation_date_time': '2009-02-13T23:31:30'}}, 'old': ''}}) + self.assertDictEqual(ret, {'vim': {'new': {'1.1': {'path': '/foo/bar/test.rpm', + 'size': 1234, + 'creation_date_time_t': 1234567890, + 'creation_date_time': '2009-02-13T23:31:30'}}, + 'old': ''}}) @patch('salt.modules.zypper._systemd_scope', MagicMock(return_value=False)) - @patch('salt.modules.zypper.list_downloaded', MagicMock(return_value={'vim': {'1.1': {'path': '/foo/bar/test.rpm', 'size': 1234, 'creation_date_time_t': 1234567890, 'creation_date_time': '2017-01-01T11:00:00'}}})) + @patch('salt.modules.zypper.list_downloaded', + MagicMock(return_value={'vim': {'1.1': {'path': '/foo/bar/test.rpm', + 'size': 1234, + 'creation_date_time_t': 1234567890, + 'creation_date_time': '2017-01-01T11:00:00'}}})) def test_install_with_downloadonly_already_downloaded(self): ''' Test a package installation with downloadonly=True when package is already downloaded. :return: ''' - with patch.dict(zypper.__salt__, {'pkg_resource.parse_targets': MagicMock(return_value=({'vim': None}, 'repository'))}): + with patch.dict(zypper.__salt__, {'pkg_resource.parse_targets': MagicMock(return_value=({'vim': None}, + 'repository'))}): with patch('salt.modules.zypper.__zypper__.noraise.call', MagicMock()) as zypper_mock: ret = zypper.install(pkgs=['vim'], downloadonly=True) zypper_mock.assert_called_once_with( @@ -621,7 +707,8 @@ Repository 'DUMMY' not found by its alias, number, or URI. self.assertDictEqual(ret, {}) @patch('salt.modules.zypper._systemd_scope', MagicMock(return_value=False)) - @patch('salt.modules.zypper._get_patches', MagicMock(return_value={'SUSE-PATCH-1234': {'installed': False, 'summary': 'test'}})) + @patch('salt.modules.zypper._get_patches', + MagicMock(return_value={'SUSE-PATCH-1234': {'installed': False, 'summary': 'test'}})) @patch('salt.modules.zypper.list_pkgs', MagicMock(side_effect=[{"vim": "1.1"}, {"vim": "1.2"}])) def test_install_advisory_patch_ok(self): ''' @@ -629,7 +716,9 @@ Repository 'DUMMY' not found by its alias, number, or URI. :return: ''' - with patch.dict(zypper.__salt__, {'pkg_resource.parse_targets': MagicMock(return_value=({'SUSE-PATCH-1234': None}, 'advisory'))}): + with patch.dict(zypper.__salt__, + {'pkg_resource.parse_targets': MagicMock(return_value=({'SUSE-PATCH-1234': None}, + 'advisory'))}): with patch('salt.modules.zypper.__zypper__.noraise.call', MagicMock()) as zypper_mock: ret = zypper.install(advisory_ids=['SUSE-PATCH-1234']) zypper_mock.assert_called_once_with( @@ -642,7 +731,8 @@ Repository 'DUMMY' not found by its alias, number, or URI. self.assertDictEqual(ret, {"vim": {"old": "1.1", "new": "1.2"}}) @patch('salt.modules.zypper._systemd_scope', MagicMock(return_value=False)) - @patch('salt.modules.zypper._get_patches', MagicMock(return_value={'SUSE-PATCH-1234': {'installed': False, 'summary': 'test'}})) + @patch('salt.modules.zypper._get_patches', + MagicMock(return_value={'SUSE-PATCH-1234': {'installed': False, 'summary': 'test'}})) @patch('salt.modules.zypper.list_pkgs', MagicMock(return_value={"vim": "1.1"})) def test_install_advisory_patch_failure(self): ''' @@ -650,7 +740,8 @@ Repository 'DUMMY' not found by its alias, number, or URI. :return: ''' - with patch.dict(zypper.__salt__, {'pkg_resource.parse_targets': MagicMock(return_value=({'SUSE-PATCH-XXX': None}, 'advisory'))}): + with patch.dict(zypper.__salt__, + {'pkg_resource.parse_targets': MagicMock(return_value=({'SUSE-PATCH-XXX': None}, 'advisory'))}): with patch('salt.modules.zypper.__zypper__.noraise.call', MagicMock()) as zypper_mock: with self.assertRaisesRegex(CommandExecutionError, '^Advisory id "SUSE-PATCH-XXX" not found$'): zypper.install(advisory_ids=['SUSE-PATCH-XXX']) @@ -1069,7 +1160,7 @@ Repository 'DUMMY' not found by its alias, number, or URI. """ _zpr = MagicMock() _zpr.nolock.xml.call = MagicMock(return_value=minidom.parseString(xmldoc)) - assert isinstance(zypper.Wildcard(_zpr)('libzypp', '*.1'), str) + assert isinstance(zypper.Wildcard(_zpr)('libzypp', '*.1'), six.string_types) def test_wildcard_to_query_condition_preservation(self): ''' diff --git a/tests/unit/netapi/rest_tornado/test_handlers.py b/tests/unit/netapi/rest_tornado/test_handlers.py index 555528e0765..23bf5188ca2 100644 --- a/tests/unit/netapi/rest_tornado/test_handlers.py +++ b/tests/unit/netapi/rest_tornado/test_handlers.py @@ -38,7 +38,7 @@ except ImportError: class AsyncHTTPTestCase(object): pass -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves.urllib.parse import urlencode, urlparse # pylint: disable=no-name-in-module # pylint: enable=import-error diff --git a/tests/unit/netapi/rest_tornado/test_utils.py b/tests/unit/netapi/rest_tornado/test_utils.py index 76b5d660766..5df66cb2d14 100644 --- a/tests/unit/netapi/rest_tornado/test_utils.py +++ b/tests/unit/netapi/rest_tornado/test_utils.py @@ -31,7 +31,8 @@ except ImportError: HAS_TORNADO = False # Import utility lib from tests -from tests.unit.utils.test_event import eventpublisher_process, event, SOCK_DIR # pylint: disable=import-error +import salt.utils.event +from tests.unit.utils.test_event import eventpublisher_process, SOCK_DIR # pylint: disable=import-error @skipIf(HAS_TORNADO is False, 'The tornado package needs to be installed') @@ -85,7 +86,7 @@ class TestEventListener(AsyncTestCase): Test getting a few events ''' with eventpublisher_process(): - me = event.MasterEvent(SOCK_DIR) + me = salt.utils.event.MasterEvent(SOCK_DIR) event_listener = saltnado.EventListener({}, # we don't use mod_opts, don't save? {'sock_dir': SOCK_DIR, 'transport': 'zeromq'}) @@ -105,7 +106,7 @@ class TestEventListener(AsyncTestCase): Test subscribing events using set_event_handler ''' with eventpublisher_process(): - me = event.MasterEvent(SOCK_DIR) + me = salt.utils.event.MasterEvent(SOCK_DIR) event_listener = saltnado.EventListener({}, # we don't use mod_opts, don't save? {'sock_dir': SOCK_DIR, 'transport': 'zeromq'}) diff --git a/tests/unit/output/test_highstate_out.py b/tests/unit/output/test_highstate_out.py index b66c21311bb..d02f1076fa6 100644 --- a/tests/unit/output/test_highstate_out.py +++ b/tests/unit/output/test_highstate_out.py @@ -12,10 +12,11 @@ from tests.support.unit import TestCase # Import Salt Libs import salt.utils +import salt.utils.stringutils import salt.output.highstate as highstate # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six class JsonTestCase(TestCase, LoaderModuleMockMixin): @@ -100,9 +101,9 @@ class JsonTestCase(TestCase, LoaderModuleMockMixin): continue entry = entry[key] if six.PY2: - entry['comment'] = salt.utils.to_unicode(entry['comment']) + entry['comment'] = salt.utils.stringutils.to_unicode(entry['comment']) else: - entry['comment'] = salt.utils.to_bytes(entry['comment']) + entry['comment'] = salt.utils.stringutils.to_bytes(entry['comment']) ret = highstate.output(self.data) self.assertIn('Succeeded: 1 (changed=1)', ret) self.assertIn('Failed: 0', ret) diff --git a/tests/unit/output/test_json_out.py b/tests/unit/output/test_json_out.py index 13c96596071..0e0be5f4fae 100644 --- a/tests/unit/output/test_json_out.py +++ b/tests/unit/output/test_json_out.py @@ -16,7 +16,7 @@ from tests.support.mock import patch import salt.output.json_out as json_out # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six class JsonTestCase(TestCase, LoaderModuleMockMixin): diff --git a/tests/unit/pillar/test_consul.py b/tests/unit/pillar/test_consul.py index efe8d628e05..6d26b75d328 100644 --- a/tests/unit/pillar/test_consul.py +++ b/tests/unit/pillar/test_consul.py @@ -11,6 +11,9 @@ from tests.support.mock import NO_MOCK, NO_MOCK_REASON, MagicMock, patch # Import Salt Libs import salt.pillar.consul_pillar as consul_pillar +# Import 3rd-party libs +from salt.ext import six + OPTS = {'consul_config': {'consul.port': 8500, 'consul.host': '172.17.0.15'}} PILLAR_DATA = [ @@ -66,7 +69,7 @@ class ConsulPillarTestCase(TestCase, LoaderModuleMockMixin): with patch.dict(consul_pillar.__salt__, {'grains.get': MagicMock(return_value=({}))}): with patch.object(consul_pillar, 'consul_fetch', MagicMock(return_value=('2232', PILLAR_DATA))): pillar_data = consul_pillar.ext_pillar('testminion', {}, 'consul_config root=test-shared/') - assert isinstance(pillar_data[u'user'][u'dontsplit'], str) + assert isinstance(pillar_data[u'user'][u'dontsplit'], six.string_types) def test_dict_merge(self): test_dict = {} diff --git a/tests/unit/pillar/test_git.py b/tests/unit/pillar/test_git.py deleted file mode 100644 index 1ea9185c848..00000000000 --- a/tests/unit/pillar/test_git.py +++ /dev/null @@ -1,186 +0,0 @@ -# -*- coding: utf-8 -*- -'''test for pillar git_pillar.py - - - :codeauthor: :email:`Georges Racinet (gracinet@anybox.fr)` - -Based on joint work with Paul Tonelli about hg_pillar integration. - -''' - -# Import python libs -from __future__ import absolute_import - -import os -import tempfile -import shutil -import subprocess -import yaml - -# Import Salt Testing libs -from tests.integration import AdaptedConfigurationTestCaseMixin -from tests.support.mixins import LoaderModuleMockMixin -from tests.support.paths import TMP -from tests.support.unit import TestCase, skipIf -from tests.support.mock import NO_MOCK, NO_MOCK_REASON, patch - -COMMIT_USER_NAME = 'test_user' -COMMIT_USER_EMAIL = 'someone@git.test' -# file contents -PILLAR_CONTENT = {'gna': 'hello'} -FILE_DATA = { - 'top.sls': {'base': {'*': ['user']}}, - 'user.sls': PILLAR_CONTENT - } - -# Import Salt Libs -import salt.utils -from salt.pillar import Pillar -import salt.pillar.git_pillar as git_pillar - - -@skipIf(NO_MOCK, NO_MOCK_REASON) -@skipIf(not git_pillar.HAS_GITPYTHON, 'no GitPython') -class GitPillarTestCase(TestCase, AdaptedConfigurationTestCaseMixin, LoaderModuleMockMixin): - 'test git_pillar pillar' - maxDiff = None - - def setup_loader_modules(self): - self.tmpdir = tempfile.mkdtemp(dir=TMP) - cachedir = os.path.join(self.tmpdir, 'cachedir') - os.makedirs(os.path.join(cachedir, 'pillar_gitfs')) - self.repo_path = self._create_repo() - return { - git_pillar: { - '__opts__': { - 'cachedir': cachedir, - 'pillar_roots': {}, - 'hash_type': 'sha256', - 'file_ignore_regex': [], - 'file_ignore_glob': [], - 'file_roots': {}, - 'state_top': 'top.sls', - 'extension_modules': '', - 'renderer': 'yaml_jinja', - 'renderer_blacklist': [], - 'renderer_whitelist': [], - 'pillar_opts': False - } - } - } - - def setUp(self): - super(GitPillarTestCase, self).setUp() - git_pillar._update('master', 'file://{0}'.format(self.repo_path)) - - def tearDown(self): - shutil.rmtree(self.tmpdir) - super(GitPillarTestCase, self).tearDown() - - def _create_repo(self): - 'create source Git repo in temp directory' - repo = os.path.join(self.tmpdir, 'repo_pillar') - os.makedirs(repo) - subprocess.check_call(['git', 'init', repo]) - for filename in FILE_DATA: - with salt.utils.fopen(os.path.join(repo, filename), 'w') as data_file: - yaml.dump(FILE_DATA[filename], data_file) - - subprocess.check_call(['git', 'add', '.'], cwd=repo) - subprocess.call(['git', 'config', 'user.email', COMMIT_USER_EMAIL], - cwd=repo) - subprocess.call(['git', 'config', 'user.name', COMMIT_USER_NAME], - cwd=repo) - subprocess.check_call(['git', 'commit', '-m', 'first commit'], - cwd=repo) - return repo - - @property - def conf_line(self): - return 'master file://{0}'.format(self.repo_path) - - def test_base(self): - 'check direct call ``ext_pillar()`` interface' - with patch.dict(git_pillar.__opts__, {'environment': None}): - mypillar = git_pillar.ext_pillar('myminion', - self.conf_line, - {}) - self.assertEqual(PILLAR_CONTENT, mypillar) - - def test_from_upper(self): - '''Check whole calling stack from parent Pillar instance - - This test is closer to what happens in real life, and demonstrates - how ``compile_pillar()`` is called twice. - - This kind of test should/would become non-necessary, once git_pillar, - all these pillar are called exactly in the same way (git is an - exception for now), and don't recurse. - ''' - with patch.dict(git_pillar.__opts__, {'ext_pillar': [dict(git=self.conf_line)]}): - pil = Pillar(git_pillar.__opts__, - git_pillar.__grains__, - 'myminion', None) - self.assertEqual(PILLAR_CONTENT, pil.compile_pillar(pillar_dirs={})) - - def test_no_loop(self): - '''Check that the reinstantiation of a pillar object does recurse. - - This test goes in great details of patching that the dedicated - utilities might do in a simpler way. - Namely, we replace the main ``ext_pillar`` entry function by one - that keeps count of its calls. - - Otherwise, the fact that the :class:`MaximumRecursion` error is caught - can go in the way on the testing. - - On the current code base, this test fails if the two first lines of - :func:``git_pillar.ext_pillar`:: - - if pillar_dirs is None: - return - - are replaced by:: - - if pillar_dirs is None: - pillar_dirs = {} - - .. note:: the explicit anti-recursion protection does not prevent - looping between two different Git pillars. - - This test will help subsequent refactors, and also as a base for other - external pillars of the same kind. - ''' - repo2 = os.path.join(self.tmpdir, 'repo_pillar2') - conf_line2 = 'master file://{0}'.format(repo2) - subprocess.check_call(['git', 'clone', self.repo_path, repo2]) - with patch.dict(git_pillar.__opts__, {'ext_pillar': [dict(git=self.conf_line), - dict(git=conf_line2)]}): - git_pillar._update(*conf_line2.split(None, 1)) - - pil = Pillar(git_pillar.__opts__, - git_pillar.__grains__, - 'myminion', 'base') - - orig_ext_pillar = pil.ext_pillars['git'] - orig_ext_pillar.count = 0 - - def ext_pillar_count_calls(minion_id, repo_string, pillar_dirs): - orig_ext_pillar.count += 1 - if orig_ext_pillar.count > 6: - # going all the way to an infinite loop is harsh on the - # test machine - raise RuntimeError('Infinite loop detected') - return orig_ext_pillar(minion_id, repo_string, pillar_dirs) - - from salt.loader import LazyLoader - orig_getitem = LazyLoader.__getitem__ - - def __getitem__(self, key): - if key == 'git.ext_pillar': - return ext_pillar_count_calls - return orig_getitem(self, key) - - with patch.object(LazyLoader, '__getitem__', __getitem__): - self.assertEqual(PILLAR_CONTENT, pil.compile_pillar(pillar_dirs={})) - self.assertTrue(orig_ext_pillar.count < 7) diff --git a/tests/unit/pillar/test_hg.py b/tests/unit/pillar/test_hg.py index 90a63da71c2..66da2465fc8 100644 --- a/tests/unit/pillar/test_hg.py +++ b/tests/unit/pillar/test_hg.py @@ -27,7 +27,7 @@ FILE_DATA = { } # Import Salt Libs -import salt.utils +import salt.utils.files import salt.pillar.hg_pillar as hg_pillar HGLIB = hg_pillar.hglib @@ -66,7 +66,7 @@ class HgPillarTestCase(TestCase, AdaptedConfigurationTestCaseMixin, LoaderModule os.makedirs(hg_repo) subprocess.check_call(['hg', 'init', hg_repo]) for filename in FILE_DATA: - with salt.utils.fopen(os.path.join(hg_repo, filename), 'w') as data_file: + with salt.utils.files.fopen(os.path.join(hg_repo, filename), 'w') as data_file: yaml.dump(FILE_DATA[filename], data_file) subprocess.check_call(['hg', 'ci', '-A', '-R', hg_repo, '-m', 'first commit', '-u', COMMIT_USER_NAME]) return hg_repo diff --git a/tests/unit/pillar/test_mysql.py b/tests/unit/pillar/test_mysql.py index c036e959468..7cae2edfd03 100644 --- a/tests/unit/pillar/test_mysql.py +++ b/tests/unit/pillar/test_mysql.py @@ -9,11 +9,10 @@ from tests.support.mock import NO_MOCK, NO_MOCK_REASON # Import Salt Libs import salt.pillar.mysql as mysql -from salt.ext.six import PY3 @skipIf(NO_MOCK, NO_MOCK_REASON) -@skipIf(PY3, 'MySQL-python is not compatible with python3') +@skipIf(not mysql.HAS_MYSQL, 'Install MySQL bindings before running MySQL unit tests.') class MysqlPillarTestCase(TestCase): maxDiff = None diff --git a/tests/unit/pillar/test_nodegroups.py b/tests/unit/pillar/test_nodegroups.py index 03b4730d95a..4b872d79cb6 100644 --- a/tests/unit/pillar/test_nodegroups.py +++ b/tests/unit/pillar/test_nodegroups.py @@ -27,8 +27,10 @@ fake_pillar_name = 'fake_pillar_name' def side_effect(group_sel, t): if group_sel.find(fake_minion_id) != -1: - return [fake_minion_id, ] - return ['another_minion_id', ] + return {'minions': [fake_minion_id, ], + 'missing': []} + return {'minions': ['another_minion_id', ], + 'missing': []} class NodegroupsPillarTestCase(TestCase, LoaderModuleMockMixin): diff --git a/tests/unit/proxy/__init__.py b/tests/unit/proxy/__init__.py new file mode 100644 index 00000000000..40a96afc6ff --- /dev/null +++ b/tests/unit/proxy/__init__.py @@ -0,0 +1 @@ +# -*- coding: utf-8 -*- diff --git a/tests/unit/proxy/test_esxcluster.py b/tests/unit/proxy/test_esxcluster.py new file mode 100644 index 00000000000..e13faab69ed --- /dev/null +++ b/tests/unit/proxy/test_esxcluster.py @@ -0,0 +1,185 @@ +# -*- coding: utf-8 -*- +''' + :codeauthor: :email:`Alexandru Bleotu ` + + Tests for esxcluster proxy +''' + +# Import Python Libs +from __future__ import absolute_import + +# Import external libs +try: + import jsonschema + HAS_JSONSCHEMA = True +except ImportError: + HAS_JSONSCHEMA = False + +# Import Salt Libs +import salt.proxy.esxcluster as esxcluster +import salt.exceptions +from salt.config.schemas.esxcluster import EsxclusterProxySchema + +# Import Salt Testing Libs +from tests.support.mixins import LoaderModuleMockMixin +from tests.support.unit import TestCase, skipIf +from tests.support.mock import ( + MagicMock, + patch, + NO_MOCK, + NO_MOCK_REASON +) + + +@skipIf(NO_MOCK, NO_MOCK_REASON) +@skipIf(not HAS_JSONSCHEMA, 'jsonschema is required') +class InitTestCase(TestCase, LoaderModuleMockMixin): + '''Tests for salt.proxy.esxcluster.init''' + def setup_loader_modules(self): + return {esxcluster: {'__virtual__': + MagicMock(return_value='esxcluster'), + 'DETAILS': {}, '__pillar__': {}}} + + def setUp(self): + self.opts_userpass = {'proxy': {'proxytype': 'esxcluster', + 'vcenter': 'fake_vcenter', + 'datacenter': 'fake_dc', + 'cluster': 'fake_cluster', + 'mechanism': 'userpass', + 'username': 'fake_username', + 'passwords': ['fake_password'], + 'protocol': 'fake_protocol', + 'port': 100}} + self.opts_sspi = {'proxy': {'proxytype': 'esxcluster', + 'vcenter': 'fake_vcenter', + 'datacenter': 'fake_dc', + 'cluster': 'fake_cluster', + 'mechanism': 'sspi', + 'domain': 'fake_domain', + 'principal': 'fake_principal', + 'protocol': 'fake_protocol', + 'port': 100}} + patches = (('salt.proxy.esxcluster.merge', + MagicMock(return_value=self.opts_sspi['proxy'])),) + for mod, mock in patches: + patcher = patch(mod, mock) + patcher.start() + self.addCleanup(patcher.stop) + + def test_merge(self): + mock_pillar_proxy = MagicMock() + mock_opts_proxy = MagicMock() + mock_merge = MagicMock(return_value=self.opts_sspi['proxy']) + with patch.dict(esxcluster.__pillar__, + {'proxy': mock_pillar_proxy}): + with patch('salt.proxy.esxcluster.merge', mock_merge): + esxcluster.init(opts={'proxy': mock_opts_proxy}) + mock_merge.assert_called_once_with(mock_opts_proxy, mock_pillar_proxy) + + def test_esxcluster_schema(self): + mock_json_validate = MagicMock() + serialized_schema = EsxclusterProxySchema().serialize() + with patch('salt.proxy.esxcluster.jsonschema.validate', + mock_json_validate): + esxcluster.init(self.opts_sspi) + mock_json_validate.assert_called_once_with( + self.opts_sspi['proxy'], serialized_schema) + + def test_invalid_proxy_input_error(self): + with patch('salt.proxy.esxcluster.jsonschema.validate', + MagicMock(side_effect=jsonschema.exceptions.ValidationError( + 'Validation Error'))): + with self.assertRaises(salt.exceptions.InvalidConfigError) as \ + excinfo: + esxcluster.init(self.opts_userpass) + self.assertEqual(excinfo.exception.strerror.message, + 'Validation Error') + + def test_no_username(self): + opts = self.opts_userpass.copy() + del opts['proxy']['username'] + with patch('salt.proxy.esxcluster.merge', + MagicMock(return_value=opts['proxy'])): + with self.assertRaises(salt.exceptions.InvalidConfigError) as \ + excinfo: + esxcluster.init(opts) + self.assertEqual(excinfo.exception.strerror, + 'Mechanism is set to \'userpass\', but no ' + '\'username\' key found in proxy config.') + + def test_no_passwords(self): + opts = self.opts_userpass.copy() + del opts['proxy']['passwords'] + with patch('salt.proxy.esxcluster.merge', + MagicMock(return_value=opts['proxy'])): + with self.assertRaises(salt.exceptions.InvalidConfigError) as \ + excinfo: + esxcluster.init(opts) + self.assertEqual(excinfo.exception.strerror, + 'Mechanism is set to \'userpass\', but no ' + '\'passwords\' key found in proxy config.') + + def test_no_domain(self): + opts = self.opts_sspi.copy() + del opts['proxy']['domain'] + with patch('salt.proxy.esxcluster.merge', + MagicMock(return_value=opts['proxy'])): + with self.assertRaises(salt.exceptions.InvalidConfigError) as \ + excinfo: + esxcluster.init(opts) + self.assertEqual(excinfo.exception.strerror, + 'Mechanism is set to \'sspi\', but no ' + '\'domain\' key found in proxy config.') + + def test_no_principal(self): + opts = self.opts_sspi.copy() + del opts['proxy']['principal'] + with patch('salt.proxy.esxcluster.merge', + MagicMock(return_value=opts['proxy'])): + with self.assertRaises(salt.exceptions.InvalidConfigError) as \ + excinfo: + esxcluster.init(opts) + self.assertEqual(excinfo.exception.strerror, + 'Mechanism is set to \'sspi\', but no ' + '\'principal\' key found in proxy config.') + + def test_find_credentials(self): + mock_find_credentials = MagicMock(return_value=('fake_username', + 'fake_password')) + with patch('salt.proxy.esxcluster.merge', + MagicMock(return_value=self.opts_userpass['proxy'])): + with patch('salt.proxy.esxcluster.find_credentials', + mock_find_credentials): + esxcluster.init(self.opts_userpass) + mock_find_credentials.assert_called_once_with() + + def test_details_userpass(self): + mock_find_credentials = MagicMock(return_value=('fake_username', + 'fake_password')) + with patch('salt.proxy.esxcluster.merge', + MagicMock(return_value=self.opts_userpass['proxy'])): + with patch('salt.proxy.esxcluster.find_credentials', + mock_find_credentials): + esxcluster.init(self.opts_userpass) + self.assertDictEqual(esxcluster.DETAILS, + {'vcenter': 'fake_vcenter', + 'datacenter': 'fake_dc', + 'cluster': 'fake_cluster', + 'mechanism': 'userpass', + 'username': 'fake_username', + 'password': 'fake_password', + 'passwords': ['fake_password'], + 'protocol': 'fake_protocol', + 'port': 100}) + + def test_details_sspi(self): + esxcluster.init(self.opts_sspi) + self.assertDictEqual(esxcluster.DETAILS, + {'vcenter': 'fake_vcenter', + 'datacenter': 'fake_dc', + 'cluster': 'fake_cluster', + 'mechanism': 'sspi', + 'domain': 'fake_domain', + 'principal': 'fake_principal', + 'protocol': 'fake_protocol', + 'port': 100}) diff --git a/tests/unit/proxy/test_esxdatacenter.py b/tests/unit/proxy/test_esxdatacenter.py new file mode 100644 index 00000000000..fb44851a9f3 --- /dev/null +++ b/tests/unit/proxy/test_esxdatacenter.py @@ -0,0 +1,153 @@ +# -*- coding: utf-8 -*- +''' + :codeauthor: :email:`Alexandru Bleotu ` + + Tests for esxdatacenter proxy +''' + +# Import Python Libs +from __future__ import absolute_import + +# Import external libs +try: + import jsonschema + HAS_JSONSCHEMA = True +except ImportError: + HAS_JSONSCHEMA = False + +# Import Salt Libs +import salt.proxy.esxdatacenter as esxdatacenter +import salt.exceptions +from salt.config.schemas.esxdatacenter import EsxdatacenterProxySchema + +# Import Salt Testing Libs +from tests.support.mixins import LoaderModuleMockMixin +from tests.support.unit import TestCase, skipIf +from tests.support.mock import ( + MagicMock, + patch, + NO_MOCK, + NO_MOCK_REASON +) + + +@skipIf(NO_MOCK, NO_MOCK_REASON) +@skipIf(not HAS_JSONSCHEMA, 'jsonschema is required') +class InitTestCase(TestCase, LoaderModuleMockMixin): + '''Tests for salt.proxy.esxdatacenter.init''' + def setup_loader_modules(self): + return {esxdatacenter: {'__virtual__': + MagicMock(return_value='esxdatacenter'), + 'DETAILS': {}}} + + def setUp(self): + self.opts_userpass = {'proxy': {'proxytype': 'esxdatacenter', + 'vcenter': 'fake_vcenter', + 'datacenter': 'fake_dc', + 'mechanism': 'userpass', + 'username': 'fake_username', + 'passwords': ['fake_password'], + 'protocol': 'fake_protocol', + 'port': 100}} + self.opts_sspi = {'proxy': {'proxytype': 'esxdatacenter', + 'vcenter': 'fake_vcenter', + 'datacenter': 'fake_dc', + 'mechanism': 'sspi', + 'domain': 'fake_domain', + 'principal': 'fake_principal', + 'protocol': 'fake_protocol', + 'port': 100}} + + def test_esxdatacenter_schema(self): + mock_json_validate = MagicMock() + serialized_schema = EsxdatacenterProxySchema().serialize() + with patch('salt.proxy.esxdatacenter.jsonschema.validate', + mock_json_validate): + esxdatacenter.init(self.opts_sspi) + mock_json_validate.assert_called_once_with( + self.opts_sspi['proxy'], serialized_schema) + + def test_invalid_proxy_input_error(self): + with patch('salt.proxy.esxdatacenter.jsonschema.validate', + MagicMock(side_effect=jsonschema.exceptions.ValidationError( + 'Validation Error'))): + with self.assertRaises(salt.exceptions.InvalidConfigError) as \ + excinfo: + esxdatacenter.init(self.opts_userpass) + self.assertEqual(excinfo.exception.strerror.message, + 'Validation Error') + + def test_no_username(self): + opts = self.opts_userpass.copy() + del opts['proxy']['username'] + with self.assertRaises(salt.exceptions.InvalidConfigError) as \ + excinfo: + esxdatacenter.init(opts) + self.assertEqual(excinfo.exception.strerror, + 'Mechanism is set to \'userpass\', but no ' + '\'username\' key found in proxy config.') + + def test_no_passwords(self): + opts = self.opts_userpass.copy() + del opts['proxy']['passwords'] + with self.assertRaises(salt.exceptions.InvalidConfigError) as \ + excinfo: + esxdatacenter.init(opts) + self.assertEqual(excinfo.exception.strerror, + 'Mechanism is set to \'userpass\', but no ' + '\'passwords\' key found in proxy config.') + + def test_no_domain(self): + opts = self.opts_sspi.copy() + del opts['proxy']['domain'] + with self.assertRaises(salt.exceptions.InvalidConfigError) as \ + excinfo: + esxdatacenter.init(opts) + self.assertEqual(excinfo.exception.strerror, + 'Mechanism is set to \'sspi\', but no ' + '\'domain\' key found in proxy config.') + + def test_no_principal(self): + opts = self.opts_sspi.copy() + del opts['proxy']['principal'] + with self.assertRaises(salt.exceptions.InvalidConfigError) as \ + excinfo: + esxdatacenter.init(opts) + self.assertEqual(excinfo.exception.strerror, + 'Mechanism is set to \'sspi\', but no ' + '\'principal\' key found in proxy config.') + + def test_find_credentials(self): + mock_find_credentials = MagicMock(return_value=('fake_username', + 'fake_password')) + with patch('salt.proxy.esxdatacenter.find_credentials', + mock_find_credentials): + esxdatacenter.init(self.opts_userpass) + mock_find_credentials.assert_called_once_with() + + def test_details_userpass(self): + mock_find_credentials = MagicMock(return_value=('fake_username', + 'fake_password')) + with patch('salt.proxy.esxdatacenter.find_credentials', + mock_find_credentials): + esxdatacenter.init(self.opts_userpass) + self.assertDictEqual(esxdatacenter.DETAILS, + {'vcenter': 'fake_vcenter', + 'datacenter': 'fake_dc', + 'mechanism': 'userpass', + 'username': 'fake_username', + 'password': 'fake_password', + 'passwords': ['fake_password'], + 'protocol': 'fake_protocol', + 'port': 100}) + + def test_details_sspi(self): + esxdatacenter.init(self.opts_sspi) + self.assertDictEqual(esxdatacenter.DETAILS, + {'vcenter': 'fake_vcenter', + 'datacenter': 'fake_dc', + 'mechanism': 'sspi', + 'domain': 'fake_domain', + 'principal': 'fake_principal', + 'protocol': 'fake_protocol', + 'port': 100}) diff --git a/tests/unit/renderers/test_gpg.py b/tests/unit/renderers/test_gpg.py index 5890e6d8d83..b4b26e38281 100644 --- a/tests/unit/renderers/test_gpg.py +++ b/tests/unit/renderers/test_gpg.py @@ -32,10 +32,10 @@ class GPGTestCase(TestCase, LoaderModuleMockMixin): ''' gpg_exec = '/bin/gpg' - with patch('salt.utils.which', MagicMock(return_value=gpg_exec)): + with patch('salt.utils.path.which', MagicMock(return_value=gpg_exec)): self.assertEqual(gpg._get_gpg_exec(), gpg_exec) - with patch('salt.utils.which', MagicMock(return_value=False)): + with patch('salt.utils.path.which', MagicMock(return_value=False)): self.assertRaises(SaltRenderError, gpg._get_gpg_exec) def test__decrypt_ciphertext(self): @@ -55,7 +55,7 @@ class GPGTestCase(TestCase, LoaderModuleMockMixin): return [None, 'decrypt error'] with patch('salt.renderers.gpg._get_key_dir', MagicMock(return_value=key_dir)), \ - patch('salt.utils.which', MagicMock()): + patch('salt.utils.path.which', MagicMock()): with patch('salt.renderers.gpg.Popen', MagicMock(return_value=GPGDecrypt())): self.assertEqual(gpg._decrypt_ciphertext(crypted), secret) with patch('salt.renderers.gpg.Popen', MagicMock(return_value=GPGNotDecrypt())): diff --git a/tests/unit/renderers/test_nacl.py b/tests/unit/renderers/test_nacl.py new file mode 100644 index 00000000000..04aa31dd883 --- /dev/null +++ b/tests/unit/renderers/test_nacl.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- + +# Import Python Libs +from __future__ import absolute_import + +# Import Salt Testing libs +from tests.support.mixins import LoaderModuleMockMixin +from tests.support.unit import skipIf, TestCase +from tests.support.mock import ( + NO_MOCK, + NO_MOCK_REASON, + MagicMock, + patch +) + +# Import Salt libs +import salt.renderers.nacl as nacl + + +@skipIf(NO_MOCK, NO_MOCK_REASON) +class NaclTestCase(TestCase, LoaderModuleMockMixin): + ''' + unit test NaCl renderer + ''' + def setup_loader_modules(self): + return {nacl: {}} + + def test__decrypt_object(self): + ''' + test _decrypt_object + ''' + secret = 'Use more salt.' + crypted = 'NACL[MRN3cc+fmdxyQbz6WMF+jq1hKdU5X5BBI7OjK+atvHo1ll+w1gZ7XyWtZVfq9gK9rQaMfkDxmidJKwE0Mw==]' + + secret_map = {'secret': secret} + crypted_map = {'secret': crypted} + + secret_list = [secret] + crypted_list = [crypted] + + with patch.dict(nacl.__salt__, {'nacl.dec': MagicMock(return_value=secret)}): + self.assertEqual(nacl._decrypt_object(secret), secret) + self.assertEqual(nacl._decrypt_object(crypted), secret) + self.assertEqual(nacl._decrypt_object(crypted_map), secret_map) + self.assertEqual(nacl._decrypt_object(crypted_list), secret_list) + self.assertEqual(nacl._decrypt_object(None), None) + + def test_render(self): + ''' + test render + ''' + secret = 'Use more salt.' + crypted = 'NACL[MRN3cc+fmdxyQbz6WMF+jq1hKdU5X5BBI7OjK+atvHo1ll+w1gZ7XyWtZVfq9gK9rQaMfkDxmidJKwE0Mw==]' + with patch.dict(nacl.__salt__, {'nacl.dec': MagicMock(return_value=secret)}): + self.assertEqual(nacl.render(crypted), secret) diff --git a/tests/unit/returners/test_local_cache.py b/tests/unit/returners/test_local_cache.py index 20d5ad1720e..741957ffd87 100644 --- a/tests/unit/returners/test_local_cache.py +++ b/tests/unit/returners/test_local_cache.py @@ -26,10 +26,12 @@ from tests.support.mock import ( ) # Import Salt libs -import salt.utils +import salt.utils.files import salt.utils.jid +import salt.utils.job +import salt.utils.platform import salt.returners.local_cache as local_cache -import salt.ext.six as six +from salt.ext import six log = logging.getLogger(__name__) @@ -95,7 +97,10 @@ class LocalCacheCleanOldJobsTestCase(TestCase, LoaderModuleMockMixin): local_cache.clean_old_jobs() # Get the name of the JID directory that was created to test against - jid_dir_name = jid_dir.rpartition('/')[2] + if salt.utils.is_windows(): + jid_dir_name = jid_dir.rpartition('\\')[2] + else: + jid_dir_name = jid_dir.rpartition('/')[2] # Assert the JID directory is still present to be cleaned after keep_jobs interval self.assertEqual([jid_dir_name], os.listdir(TMP_JID_DIR)) @@ -160,7 +165,7 @@ class LocalCacheCleanOldJobsTestCase(TestCase, LoaderModuleMockMixin): dir_name = '/'.join([temp_dir, 'jid']) os.mkdir(dir_name) jid_file_path = '/'.join([dir_name, 'jid']) - with salt.utils.fopen(jid_file_path, 'w') as jid_file: + with salt.utils.files.fopen(jid_file_path, 'w') as jid_file: jid_file.write('this is a jid file') return temp_dir, jid_file_path @@ -278,7 +283,7 @@ class Local_CacheTest(TestCase, AdaptedConfigurationTestCaseMixin, LoaderModuleM # This needed due to a race condition in Windows # `os.makedirs` hasn't released the handle before # `local_cache.clean_old_jobs` tries to delete the new_jid_dir - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): import time lock_dir = new_jid_dir + '.lckchk' tries = 0 diff --git a/tests/unit/runners/test_vault.py b/tests/unit/runners/test_vault.py index 1231caafcea..e9f751fa99f 100644 --- a/tests/unit/runners/test_vault.py +++ b/tests/unit/runners/test_vault.py @@ -18,7 +18,7 @@ from tests.support.mock import ( ) # Import salt libs -import salt.ext.six as six +from salt.ext import six import salt.runners.vault as vault log = logging.getLogger(__name__) diff --git a/tests/unit/runners/test_winrepo.py b/tests/unit/runners/test_winrepo.py index a28726b8b97..2642de959b3 100644 --- a/tests/unit/runners/test_winrepo.py +++ b/tests/unit/runners/test_winrepo.py @@ -13,7 +13,7 @@ from tests.support.unit import skipIf, TestCase from tests.support.mock import NO_MOCK, NO_MOCK_REASON # Import salt libs -import salt.utils +import salt.utils.files import salt.runners.winrepo as winrepo _WINREPO_SLS = r''' @@ -102,6 +102,6 @@ class WinrepoTest(TestCase, LoaderModuleMockMixin): ''' sls_file = os.path.join(self.winrepo_sls_dir, 'wireshark.sls') # Add a winrepo SLS file - with salt.utils.fopen(sls_file, 'w') as fp_: + with salt.utils.files.fopen(sls_file, 'w') as fp_: fp_.write(_WINREPO_SLS) self.assertEqual(winrepo.genrepo(), _WINREPO_GENREPO_DATA) diff --git a/tests/unit/serializers/test_serializers.py b/tests/unit/serializers/test_serializers.py index 2520a462d86..4f4890e06e0 100644 --- a/tests/unit/serializers/test_serializers.py +++ b/tests/unit/serializers/test_serializers.py @@ -9,7 +9,7 @@ from tests.support.unit import skipIf, TestCase # Import 3rd party libs import jinja2 -import salt.ext.six as six +from salt.ext import six # Import salt libs import salt.serializers.configparser as configparser diff --git a/tests/unit/ssh/test_ssh_single.py b/tests/unit/ssh/test_ssh_single.py index 25d89f6eca5..68970ac0933 100644 --- a/tests/unit/ssh/test_ssh_single.py +++ b/tests/unit/ssh/test_ssh_single.py @@ -14,8 +14,8 @@ from tests.support.mock import NO_MOCK, NO_MOCK_REASON # Import Salt libs import tests.integration as integration +import salt.utils.thin as thin from salt.client import ssh -from salt.utils import thin @skipIf(NO_MOCK, NO_MOCK_REASON) diff --git a/tests/unit/states/test_apache.py b/tests/unit/states/test_apache.py index bb10c3728fc..267266dbc5f 100644 --- a/tests/unit/states/test_apache.py +++ b/tests/unit/states/test_apache.py @@ -17,7 +17,7 @@ from tests.support.mock import ( # Import Salt Libs import salt.states.apache as apache -import salt.utils +import salt.utils.files @skipIf(NO_MOCK, NO_MOCK_REASON) @@ -45,13 +45,13 @@ class ApacheTestCase(TestCase, LoaderModuleMockMixin): 'changes': {}, 'comment': ''} - with patch.object(salt.utils, 'fopen', mock_open(read_data=config)): + with patch.object(salt.utils.files, 'fopen', mock_open(read_data=config)): mock_config = MagicMock(return_value=config) with patch.dict(apache.__salt__, {'apache.config': mock_config}): ret.update({'comment': 'Configuration is up to date.'}) self.assertDictEqual(apache.configfile(name, config), ret) - with patch.object(salt.utils, 'fopen', mock_open(read_data=config)): + with patch.object(salt.utils.files, 'fopen', mock_open(read_data=config)): mock_config = MagicMock(return_value=new_config) with patch.dict(apache.__salt__, {'apache.config': mock_config}): ret.update({'comment': 'Configuration will update.', @@ -61,7 +61,7 @@ class ApacheTestCase(TestCase, LoaderModuleMockMixin): with patch.dict(apache.__opts__, {'test': True}): self.assertDictEqual(apache.configfile(name, new_config), ret) - with patch.object(salt.utils, 'fopen', mock_open(read_data=config)): + with patch.object(salt.utils.files, 'fopen', mock_open(read_data=config)): mock_config = MagicMock(return_value=new_config) with patch.dict(apache.__salt__, {'apache.config': mock_config}): ret.update({'comment': 'Successfully created configuration.', diff --git a/tests/unit/states/test_archive.py b/tests/unit/states/test_archive.py index 2823711fe75..30d256773d6 100644 --- a/tests/unit/states/test_archive.py +++ b/tests/unit/states/test_archive.py @@ -102,16 +102,17 @@ class ArchiveTestCase(TestCase, LoaderModuleMockMixin): 'cmd.run_all': mock_run, 'archive.list': list_mock, 'file.source_list': mock_source_list}): - with patch.object(os.path, 'isfile', isfile_mock): - for test_opts, ret_opts in zip(test_tar_opts, ret_tar_opts): - ret = archive.extracted(tmp_dir, - source, - options=test_opts, - enforce_toplevel=False) - ret_opts.append(source) - mock_run.assert_called_with(ret_opts, - cwd=tmp_dir + os.sep, - python_shell=False) + with patch.dict(archive.__states__, {'file.directory': mock_true}): + with patch.object(os.path, 'isfile', isfile_mock): + for test_opts, ret_opts in zip(test_tar_opts, ret_tar_opts): + ret = archive.extracted(tmp_dir, + source, + options=test_opts, + enforce_toplevel=False) + ret_opts.append(source) + mock_run.assert_called_with(ret_opts, + cwd=tmp_dir + os.sep, + python_shell=False) def test_tar_gnutar(self): ''' @@ -142,13 +143,14 @@ class ArchiveTestCase(TestCase, LoaderModuleMockMixin): 'cmd.run_all': run_all, 'archive.list': list_mock, 'file.source_list': mock_source_list}): - with patch.object(os.path, 'isfile', isfile_mock): - ret = archive.extracted('/tmp/out', - source, - options='xvzf', - enforce_toplevel=False, - keep=True) - self.assertEqual(ret['changes']['extracted_files'], 'stdout') + with patch.dict(archive.__states__, {'file.directory': mock_true}): + with patch.object(os.path, 'isfile', isfile_mock): + ret = archive.extracted('/tmp/out', + source, + options='xvzf', + enforce_toplevel=False, + keep=True) + self.assertEqual(ret['changes']['extracted_files'], 'stdout') def test_tar_bsdtar(self): ''' @@ -179,10 +181,11 @@ class ArchiveTestCase(TestCase, LoaderModuleMockMixin): 'cmd.run_all': run_all, 'archive.list': list_mock, 'file.source_list': mock_source_list}): - with patch.object(os.path, 'isfile', isfile_mock): - ret = archive.extracted('/tmp/out', - source, - options='xvzf', - enforce_toplevel=False, - keep=True) - self.assertEqual(ret['changes']['extracted_files'], 'stderr') + with patch.dict(archive.__states__, {'file.directory': mock_true}): + with patch.object(os.path, 'isfile', isfile_mock): + ret = archive.extracted('/tmp/out', + source, + options='xvzf', + enforce_toplevel=False, + keep=True) + self.assertEqual(ret['changes']['extracted_files'], 'stderr') diff --git a/tests/unit/states/test_augeas.py b/tests/unit/states/test_augeas.py index dd316ea7fbb..c6b17e270d9 100644 --- a/tests/unit/states/test_augeas.py +++ b/tests/unit/states/test_augeas.py @@ -141,7 +141,7 @@ class AugeasTestCase(TestCase, LoaderModuleMockMixin): with patch.dict(augeas.__salt__, mock_dict_): mock_filename = MagicMock(return_value='/etc/services') with patch.object(augeas, '_workout_filename', mock_filename): - with patch('salt.utils.fopen', MagicMock(mock_open)): + with patch('salt.utils.files.fopen', MagicMock(mock_open)): mock_diff = MagicMock(return_value=['+ zabbix-agent']) with patch('difflib.unified_diff', mock_diff): self.assertDictEqual(augeas.change(self.name, @@ -218,7 +218,7 @@ class AugeasTestCase(TestCase, LoaderModuleMockMixin): mock_dict_ = {'augeas.execute': mock_execute, 'augeas.method_map': self.mock_method_map} with patch.dict(augeas.__salt__, mock_dict_): - with patch('salt.utils.fopen', MagicMock(mock_open)): + with patch('salt.utils.files.fopen', MagicMock(mock_open)): self.assertDictEqual(augeas.change(self.name, context=self.context, changes=self.changes), diff --git a/tests/unit/states/test_beacon.py b/tests/unit/states/test_beacon.py new file mode 100644 index 00000000000..f3b716b2c16 --- /dev/null +++ b/tests/unit/states/test_beacon.py @@ -0,0 +1,92 @@ +# -*- coding: utf-8 -*- +''' + :codeauthor: :email:`Jayesh Kariya ` +''' +# Import Python libs +from __future__ import absolute_import +import os + +# Import Salt Testing Libs +from tests.support.mixins import LoaderModuleMockMixin +from tests.support.paths import TMP +from tests.support.unit import skipIf, TestCase +from tests.support.mock import ( + NO_MOCK, + NO_MOCK_REASON, + MagicMock, + patch +) + +# Import Salt Libs +import salt.states.beacon as beacon + +SOCK_DIR = os.path.join(TMP, 'test-socks') + + +@skipIf(NO_MOCK, NO_MOCK_REASON) +class BeaconTestCase(TestCase, LoaderModuleMockMixin): + ''' + Test cases for salt.states.beacon + ''' + def setup_loader_modules(self): + return {beacon: {}} + + # 'present' function tests: 1 + + def test_present(self): + ''' + Test to ensure a job is present in the beacon. + ''' + beacon_name = 'ps' + + ret = {'name': beacon_name, + 'changes': {}, + 'result': False, + 'comment': ''} + + mock_dict = MagicMock(side_effect=[ret, []]) + mock_mod = MagicMock(return_value=ret) + mock_lst = MagicMock(side_effect=[{beacon_name: {}}, + {beacon_name: {}}, + {}, + {}]) + with patch.dict(beacon.__salt__, + {'beacons.list': mock_lst, + 'beacons.modify': mock_mod, + 'beacons.add': mock_mod}): + self.assertDictEqual(beacon.present(beacon_name), ret) + + with patch.dict(beacon.__opts__, {'test': False}): + self.assertDictEqual(beacon.present(beacon_name), ret) + + self.assertDictEqual(beacon.present(beacon_name), ret) + + with patch.dict(beacon.__opts__, {'test': True}): + ret.update({'result': True}) + self.assertDictEqual(beacon.present(beacon_name), ret) + + # 'absent' function tests: 1 + + def test_absent(self): + ''' + Test to ensure a job is absent from the schedule. + ''' + beacon_name = 'ps' + + ret = {'name': beacon_name, + 'changes': {}, + 'result': False, + 'comment': ''} + + mock_mod = MagicMock(return_value=ret) + mock_lst = MagicMock(side_effect=[{beacon_name: {}}, {}]) + with patch.dict(beacon.__salt__, + {'beacons.list': mock_lst, + 'beacons.delete': mock_mod}): + with patch.dict(beacon.__opts__, {'test': False}): + self.assertDictEqual(beacon.absent(beacon_name), ret) + + with patch.dict(beacon.__opts__, {'test': True}): + comt = ('ps not configured in beacons') + ret.update({'comment': comt, 'result': True}) + self.assertDictEqual(beacon.absent(beacon_name), ret) diff --git a/tests/unit/states/test_blockdev.py b/tests/unit/states/test_blockdev.py index e5899f1c70a..9cf8b1db27c 100644 --- a/tests/unit/states/test_blockdev.py +++ b/tests/unit/states/test_blockdev.py @@ -17,7 +17,7 @@ from tests.support.mock import ( # Import Salt Libs import salt.states.blockdev as blockdev -import salt.utils +import salt.utils.path @skipIf(NO_MOCK, NO_MOCK_REASON) @@ -85,7 +85,7 @@ class BlockdevTestCase(TestCase, LoaderModuleMockMixin): with patch.dict(blockdev.__salt__, {'cmd.run': MagicMock(return_value='')}): ret.update({'comment': 'Invalid fs_type: foo-bar', 'result': False}) - with patch.object(salt.utils, 'which', + with patch.object(salt.utils.path, 'which', MagicMock(return_value=False)): self.assertDictEqual(blockdev.formatted(name, fs_type='foo-bar'), ret) @@ -93,7 +93,7 @@ class BlockdevTestCase(TestCase, LoaderModuleMockMixin): with patch.dict(blockdev.__salt__, {'cmd.run': MagicMock(return_value='new-thing')}): comt = ('Changes to {0} will be applied '.format(name)) ret.update({'comment': comt, 'result': None}) - with patch.object(salt.utils, 'which', + with patch.object(salt.utils.path, 'which', MagicMock(return_value=True)): with patch.dict(blockdev.__opts__, {'test': True}): self.assertDictEqual(blockdev.formatted(name), ret) @@ -103,7 +103,7 @@ class BlockdevTestCase(TestCase, LoaderModuleMockMixin): 'disk.format_': MagicMock(return_value=True)}): comt = ('Failed to format {0}'.format(name)) ret.update({'comment': comt, 'result': False}) - with patch.object(salt.utils, 'which', + with patch.object(salt.utils.path, 'which', MagicMock(return_value=True)): with patch.dict(blockdev.__opts__, {'test': False}): self.assertDictEqual(blockdev.formatted(name), ret) diff --git a/tests/unit/states/test_boto_apigateway.py b/tests/unit/states/test_boto_apigateway.py index b1ff876d30f..7614ed5556c 100644 --- a/tests/unit/states/test_boto_apigateway.py +++ b/tests/unit/states/test_boto_apigateway.py @@ -16,6 +16,7 @@ from tests.support.mock import NO_MOCK, NO_MOCK_REASON, MagicMock, patch # Import Salt libs import salt.config import salt.loader +import salt.utils.files from salt.utils.versions import LooseVersion # pylint: disable=import-error,no-name-in-module @@ -369,7 +370,7 @@ class TempSwaggerFile(object): def __enter__(self): self.swaggerfile = 'temp-swagger-sample.yaml' - with salt.utils.fopen(self.swaggerfile, 'w') as f: + with salt.utils.files.fopen(self.swaggerfile, 'w') as f: f.write(yaml.dump(self.swaggerdict)) return self.swaggerfile diff --git a/tests/unit/states/test_boto_cloudfront.py b/tests/unit/states/test_boto_cloudfront.py new file mode 100644 index 00000000000..dddb78d384b --- /dev/null +++ b/tests/unit/states/test_boto_cloudfront.py @@ -0,0 +1,223 @@ +# -*- coding: utf-8 -*- +''' +Unit tests for the boto_cloudfront state module. +''' +# Import Python libs +from __future__ import absolute_import +import copy +import textwrap + +# Import Salt Testing Libs +from tests.support.mixins import LoaderModuleMockMixin +from tests.support.unit import skipIf, TestCase +from tests.support.mock import NO_MOCK, NO_MOCK_REASON, MagicMock, patch + +# Import Salt Libs +import salt.config +import salt.loader +import salt.states.boto_cloudfront as boto_cloudfront + + +@skipIf(NO_MOCK, NO_MOCK_REASON) +class BotoCloudfrontTestCase(TestCase, LoaderModuleMockMixin): + ''' + Test cases for salt.states.boto_cloudfront + ''' + def setup_loader_modules(self): + utils = salt.loader.utils( + self.opts, + whitelist=['boto3', 'dictdiffer', 'yamldumper'], + context={}, + ) + return { + boto_cloudfront: { + '__utils__': utils, + } + } + + @classmethod + def setUpClass(cls): + cls.opts = salt.config.DEFAULT_MINION_OPTS + + cls.name = 'my_distribution' + cls.base_ret = {'name': cls.name, 'changes': {}} + + # Most attributes elided since there are so many required ones + cls.config = {'Enabled': True, 'HttpVersion': 'http2'} + cls.tags = {'test_tag1': 'value1'} + + @classmethod + def tearDownClass(cls): + del cls.opts + + del cls.name + del cls.base_ret + + del cls.config + del cls.tags + + def base_ret_with(self, extra_ret): + new_ret = copy.deepcopy(self.base_ret) + new_ret.update(extra_ret) + return new_ret + + def test_present_distribution_retrieval_error(self): + ''' + Test for boto_cloudfront.present when we cannot get the distribution. + ''' + mock_get = MagicMock(return_value={'error': 'get_distribution error'}) + with patch.multiple(boto_cloudfront, + __salt__={'boto_cloudfront.get_distribution': mock_get}, + __opts__={'test': False}, + ): + comment = 'Error checking distribution {0}: get_distribution error' + self.assertDictEqual( + boto_cloudfront.present(self.name, self.config, self.tags), + self.base_ret_with({ + 'result': False, + 'comment': comment.format(self.name), + }), + ) + + def test_present_from_scratch(self): + mock_get = MagicMock(return_value={'result': None}) + + with patch.multiple(boto_cloudfront, + __salt__={'boto_cloudfront.get_distribution': mock_get}, + __opts__={'test': True}, + ): + comment = 'Distribution {0} set for creation.'.format(self.name) + self.assertDictEqual( + boto_cloudfront.present(self.name, self.config, self.tags), + self.base_ret_with({ + 'result': None, + 'comment': comment, + 'pchanges': {'old': None, 'new': self.name}, + }), + ) + + mock_create_failure = MagicMock(return_value={'error': 'create error'}) + with patch.multiple(boto_cloudfront, + __salt__={ + 'boto_cloudfront.get_distribution': mock_get, + 'boto_cloudfront.create_distribution': mock_create_failure, + }, + __opts__={'test': False}, + ): + comment = 'Error creating distribution {0}: create error' + self.assertDictEqual( + boto_cloudfront.present(self.name, self.config, self.tags), + self.base_ret_with({ + 'result': False, + 'comment': comment.format(self.name), + }), + ) + + mock_create_success = MagicMock(return_value={'result': True}) + with patch.multiple(boto_cloudfront, + __salt__={ + 'boto_cloudfront.get_distribution': mock_get, + 'boto_cloudfront.create_distribution': mock_create_success, + }, + __opts__={'test': False}, + ): + comment = 'Created distribution {0}.' + self.assertDictEqual( + boto_cloudfront.present(self.name, self.config, self.tags), + self.base_ret_with({ + 'result': True, + 'comment': comment.format(self.name), + 'changes': {'old': None, 'new': self.name}, + }), + ) + + def test_present_correct_state(self): + mock_get = MagicMock(return_value={'result': { + 'distribution': {'DistributionConfig': self.config}, + 'tags': self.tags, + 'etag': 'test etag', + }}) + with patch.multiple(boto_cloudfront, + __salt__={'boto_cloudfront.get_distribution': mock_get}, + __opts__={'test': False}, + ): + comment = 'Distribution {0} has correct config.' + self.assertDictEqual( + boto_cloudfront.present(self.name, self.config, self.tags), + self.base_ret_with({ + 'result': True, + 'comment': comment.format(self.name), + }), + ) + + def test_present_update_config_and_tags(self): + mock_get = MagicMock(return_value={'result': { + 'distribution': {'DistributionConfig': { + 'Enabled': False, + 'Comment': 'to be removed', + }}, + 'tags': {'bad existing tag': 'also to be removed'}, + 'etag': 'test etag', + }}) + + diff = textwrap.dedent('''\ + --- + +++ + @@ -1,5 +1,5 @@ + config: + - Comment: to be removed + - Enabled: false + + Enabled: true + + HttpVersion: http2 + tags: + - bad existing tag: also to be removed + + test_tag1: value1 + ''') + + with patch.multiple(boto_cloudfront, + __salt__={'boto_cloudfront.get_distribution': mock_get}, + __opts__={'test': True}, + ): + header = 'Distribution {0} set for new config:'.format(self.name) + self.assertDictEqual( + boto_cloudfront.present(self.name, self.config, self.tags), + self.base_ret_with({ + 'result': None, + 'comment': '\n'.join([header, diff]), + 'pchanges': {'diff': diff}, + }), + ) + + mock_update_failure = MagicMock(return_value={'error': 'update error'}) + with patch.multiple(boto_cloudfront, + __salt__={ + 'boto_cloudfront.get_distribution': mock_get, + 'boto_cloudfront.update_distribution': mock_update_failure, + }, + __opts__={'test': False}, + ): + comment = 'Error updating distribution {0}: update error' + self.assertDictEqual( + boto_cloudfront.present(self.name, self.config, self.tags), + self.base_ret_with({ + 'result': False, + 'comment': comment.format(self.name), + }), + ) + + mock_update_success = MagicMock(return_value={'result': True}) + with patch.multiple(boto_cloudfront, + __salt__={ + 'boto_cloudfront.get_distribution': mock_get, + 'boto_cloudfront.update_distribution': mock_update_success, + }, + __opts__={'test': False}, + ): + self.assertDictEqual( + boto_cloudfront.present(self.name, self.config, self.tags), + self.base_ret_with({ + 'result': True, + 'comment': 'Updated distribution {0}.'.format(self.name), + 'changes': {'diff': diff}, + }), + ) diff --git a/tests/unit/states/test_boto_elasticsearch_domain.py b/tests/unit/states/test_boto_elasticsearch_domain.py index 2b9f1ee56a4..a8266c06aab 100644 --- a/tests/unit/states/test_boto_elasticsearch_domain.py +++ b/tests/unit/states/test_boto_elasticsearch_domain.py @@ -12,7 +12,7 @@ from tests.support.unit import skipIf, TestCase from tests.support.mock import MagicMock, NO_MOCK, NO_MOCK_REASON, patch # Import Salt libs -import salt.ext.six as six +from salt.ext import six import salt.loader from salt.utils.versions import LooseVersion import salt.states.boto_elasticsearch_domain as boto_elasticsearch_domain diff --git a/tests/unit/states/test_boto_s3_bucket.py b/tests/unit/states/test_boto_s3_bucket.py index e98166d7bff..5bebb36039b 100644 --- a/tests/unit/states/test_boto_s3_bucket.py +++ b/tests/unit/states/test_boto_s3_bucket.py @@ -13,7 +13,7 @@ from tests.support.unit import skipIf, TestCase from tests.support.mock import MagicMock, NO_MOCK, NO_MOCK_REASON, patch # Import Salt libs -import salt.ext.six as six +from salt.ext import six import salt.loader from salt.utils.versions import LooseVersion import salt.states.boto_s3_bucket as boto_s3_bucket diff --git a/tests/unit/states/test_boto_sqs.py b/tests/unit/states/test_boto_sqs.py index 1ee4c9820e0..80672e87f4b 100644 --- a/tests/unit/states/test_boto_sqs.py +++ b/tests/unit/states/test_boto_sqs.py @@ -4,6 +4,7 @@ ''' # Import Python libs from __future__ import absolute_import +import textwrap # Import Salt Testing Libs from tests.support.mixins import LoaderModuleMockMixin @@ -11,6 +12,8 @@ from tests.support.unit import skipIf, TestCase from tests.support.mock import NO_MOCK, NO_MOCK_REASON, MagicMock, patch # Import Salt Libs +import salt.config +import salt.loader import salt.states.boto_sqs as boto_sqs @@ -20,7 +23,24 @@ class BotoSqsTestCase(TestCase, LoaderModuleMockMixin): Test cases for salt.states.boto_sqs ''' def setup_loader_modules(self): - return {boto_sqs: {}} + utils = salt.loader.utils( + self.opts, + whitelist=['boto3', 'yamldumper'], + context={} + ) + return { + boto_sqs: { + '__utils__': utils, + } + } + + @classmethod + def setUpClass(cls): + cls.opts = salt.config.DEFAULT_MINION_OPTS + + @classmethod + def tearDownClass(cls): + del cls.opts # 'present' function tests: 1 @@ -29,37 +49,57 @@ class BotoSqsTestCase(TestCase, LoaderModuleMockMixin): Test to ensure the SQS queue exists. ''' name = 'mysqs' - attributes = {'ReceiveMessageWaitTimeSeconds': 20} + attributes = {'DelaySeconds': 20} + base_ret = {'name': name, 'changes': {}} - ret = {'name': name, - 'result': False, - 'changes': {}, - 'comment': ''} - - mock = MagicMock(side_effect=[False, False, True, True]) - mock_bool = MagicMock(return_value=False) - mock_attr = MagicMock(return_value={}) + mock = MagicMock( + side_effect=[{'result': b} for b in [False, False, True, True]], + ) + mock_bool = MagicMock(return_value={'error': 'create error'}) + mock_attr = MagicMock(return_value={'result': {}}) with patch.dict(boto_sqs.__salt__, {'boto_sqs.exists': mock, 'boto_sqs.create': mock_bool, 'boto_sqs.get_attributes': mock_attr}): with patch.dict(boto_sqs.__opts__, {'test': False}): - comt = ('Failed to create {0} AWS queue'.format(name)) - ret.update({'comment': comt}) + comt = ['Failed to create SQS queue {0}: create error'.format( + name, + )] + ret = base_ret.copy() + ret.update({'result': False, 'comment': comt}) self.assertDictEqual(boto_sqs.present(name), ret) with patch.dict(boto_sqs.__opts__, {'test': True}): - comt = ('AWS SQS queue {0} is set to be created.'.format(name)) - ret.update({'comment': comt, 'result': None}) + comt = ['SQS queue {0} is set to be created.'.format(name)] + ret = base_ret.copy() + ret.update({ + 'result': None, + 'comment': comt, + 'pchanges': {'old': None, 'new': 'mysqs'}, + }) self.assertDictEqual(boto_sqs.present(name), ret) - - comt = ('Attribute(s) ReceiveMessageWaitTimeSeconds' - ' to be set on mysqs.') - ret.update({'comment': comt}) + diff = textwrap.dedent('''\ + --- + +++ + @@ -1 +1 @@ + -{} + +DelaySeconds: 20 + ''') + comt = [ + 'SQS queue mysqs present.', + 'Attribute(s) DelaySeconds set to be updated:\n{0}'.format( + diff, + ), + ] + ret.update({ + 'comment': comt, + 'pchanges': {'attributes': {'diff': diff}}, + }) self.assertDictEqual(boto_sqs.present(name, attributes), ret) - comt = ('mysqs present. Attributes set.') - ret.update({'comment': comt, 'result': True}) + comt = ['SQS queue mysqs present.'] + ret = base_ret.copy() + ret.update({'result': True, 'comment': comt}) self.assertDictEqual(boto_sqs.present(name), ret) # 'absent' function tests: 1 @@ -69,20 +109,22 @@ class BotoSqsTestCase(TestCase, LoaderModuleMockMixin): Test to ensure the named sqs queue is deleted. ''' name = 'test.example.com.' + base_ret = {'name': name, 'changes': {}} - ret = {'name': name, - 'result': True, - 'changes': {}, - 'comment': ''} - - mock = MagicMock(side_effect=[False, True]) + mock = MagicMock(side_effect=[{'result': False}, {'result': True}]) with patch.dict(boto_sqs.__salt__, {'boto_sqs.exists': mock}): - comt = ('{0} does not exist in None.'.format(name)) - ret.update({'comment': comt}) + comt = ('SQS queue {0} does not exist in None.'.format(name)) + ret = base_ret.copy() + ret.update({'result': True, 'comment': comt}) self.assertDictEqual(boto_sqs.absent(name), ret) with patch.dict(boto_sqs.__opts__, {'test': True}): - comt = ('AWS SQS queue {0} is set to be removed.'.format(name)) - ret.update({'comment': comt, 'result': None}) + comt = ('SQS queue {0} is set to be removed.'.format(name)) + ret = base_ret.copy() + ret.update({ + 'result': None, + 'comment': comt, + 'pchanges': {'old': name, 'new': None}, + }) self.assertDictEqual(boto_sqs.absent(name), ret) diff --git a/tests/unit/states/test_boto_vpc.py b/tests/unit/states/test_boto_vpc.py index e632b1ec2eb..1d5e334a2fb 100644 --- a/tests/unit/states/test_boto_vpc.py +++ b/tests/unit/states/test_boto_vpc.py @@ -11,14 +11,11 @@ from tests.support.unit import skipIf, TestCase from tests.support.mock import NO_MOCK, NO_MOCK_REASON, patch # Import Salt libs -import salt.config -import salt.loader import salt.utils.boto +from salt.ext import six from salt.utils.versions import LooseVersion import salt.states.boto_vpc as boto_vpc -# Import test suite libs - # pylint: disable=import-error,unused-import from tests.unit.modules.test_boto_vpc import BotoVpcTestCaseMixin @@ -34,18 +31,18 @@ except ImportError: HAS_BOTO = False try: - from moto import mock_ec2 + from moto import mock_ec2_deprecated HAS_MOTO = True except ImportError: HAS_MOTO = False - def mock_ec2(self): + def mock_ec2_deprecated(self): ''' - if the mock_ec2 function is not available due to import failure + if the mock_ec2_deprecated function is not available due to import failure this replaces the decorated function with stub_function. - Allows boto_vpc unit tests to use the @mock_ec2 decorator - without a "NameError: name 'mock_ec2' is not defined" error. + Allows boto_vpc unit tests to use the @mock_ec2_deprecated decorator + without a "NameError: name 'mock_ec2_deprecated' is not defined" error. ''' def stub_function(self): @@ -131,7 +128,7 @@ class BotoVpcTestCase(BotoVpcStateTestCaseBase, BotoVpcTestCaseMixin): TestCase for salt.states.boto_vpc state.module ''' - @mock_ec2 + @mock_ec2_deprecated def test_present_when_vpc_does_not_exist(self): ''' Tests present on a VPC that does not exist. @@ -142,14 +139,14 @@ class BotoVpcTestCase(BotoVpcStateTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(vpc_present_result['result']) self.assertEqual(vpc_present_result['changes']['new']['vpc']['state'], 'available') - @mock_ec2 + @mock_ec2_deprecated def test_present_when_vpc_exists(self): vpc = self._create_vpc(name='test') vpc_present_result = self.salt_states['boto_vpc.present']('test', cidr_block) self.assertTrue(vpc_present_result['result']) self.assertEqual(vpc_present_result['changes'], {}) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_present_with_failure(self): with patch('moto.ec2.models.VPCBackend.create_vpc', side_effect=BotoServerError(400, 'Mocked error')): @@ -157,7 +154,7 @@ class BotoVpcTestCase(BotoVpcStateTestCaseBase, BotoVpcTestCaseMixin): self.assertFalse(vpc_present_result['result']) self.assertTrue('Mocked error' in vpc_present_result['comment']) - @mock_ec2 + @mock_ec2_deprecated def test_absent_when_vpc_does_not_exist(self): ''' Tests absent on a VPC that does not exist. @@ -167,7 +164,7 @@ class BotoVpcTestCase(BotoVpcStateTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(vpc_absent_result['result']) self.assertEqual(vpc_absent_result['changes'], {}) - @mock_ec2 + @mock_ec2_deprecated def test_absent_when_vpc_exists(self): vpc = self._create_vpc(name='test') with patch.dict(salt.utils.boto.__salt__, self.funcs): @@ -175,7 +172,7 @@ class BotoVpcTestCase(BotoVpcStateTestCaseBase, BotoVpcTestCaseMixin): self.assertTrue(vpc_absent_result['result']) self.assertEqual(vpc_absent_result['changes']['new']['vpc'], None) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_absent_with_failure(self): vpc = self._create_vpc(name='test') @@ -195,7 +192,7 @@ class BotoVpcResourceTestCaseMixin(BotoVpcTestCaseMixin): _create = getattr(self, '_create_' + self.resource_type) _create(vpc_id=vpc_id, name=name, **self.extra_kwargs) - @mock_ec2 + @mock_ec2_deprecated def test_present_when_resource_does_not_exist(self): ''' Tests present on a resource that does not exist. @@ -210,7 +207,7 @@ class BotoVpcResourceTestCaseMixin(BotoVpcTestCaseMixin): exists = self.funcs['boto_vpc.resource_exists'](self.resource_type, 'test').get('exists') self.assertTrue(exists) - @mock_ec2 + @mock_ec2_deprecated def test_present_when_resource_exists(self): vpc = self._create_vpc(name='test') resource = self._create_resource(vpc_id=vpc.id, name='test') @@ -220,7 +217,7 @@ class BotoVpcResourceTestCaseMixin(BotoVpcTestCaseMixin): self.assertTrue(resource_present_result['result']) self.assertEqual(resource_present_result['changes'], {}) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_present_with_failure(self): vpc = self._create_vpc(name='test') @@ -231,7 +228,7 @@ class BotoVpcResourceTestCaseMixin(BotoVpcTestCaseMixin): self.assertFalse(resource_present_result['result']) self.assertTrue('Mocked error' in resource_present_result['comment']) - @mock_ec2 + @mock_ec2_deprecated def test_absent_when_resource_does_not_exist(self): ''' Tests absent on a resource that does not exist. @@ -241,7 +238,7 @@ class BotoVpcResourceTestCaseMixin(BotoVpcTestCaseMixin): self.assertTrue(resource_absent_result['result']) self.assertEqual(resource_absent_result['changes'], {}) - @mock_ec2 + @mock_ec2_deprecated def test_absent_when_resource_exists(self): vpc = self._create_vpc(name='test') self._create_resource(vpc_id=vpc.id, name='test') @@ -253,7 +250,7 @@ class BotoVpcResourceTestCaseMixin(BotoVpcTestCaseMixin): exists = self.funcs['boto_vpc.resource_exists'](self.resource_type, 'test').get('exists') self.assertFalse(exists) - @mock_ec2 + @mock_ec2_deprecated @skipIf(True, 'Disabled pending https://github.com/spulec/moto/issues/493') def test_absent_with_failure(self): vpc = self._create_vpc(name='test') @@ -291,6 +288,9 @@ class BotoVpcInternetGatewayTestCase(BotoVpcStateTestCaseBase, BotoVpcResourceTe @skipIf(NO_MOCK, NO_MOCK_REASON) +@skipIf(six.PY3, 'Disabled for Python 3 due to upstream bugs: ' + 'https://github.com/spulec/moto/issues/548 and ' + 'https://github.com/gabrielfalcao/HTTPretty/issues/325') @skipIf(HAS_BOTO is False, 'The boto module must be installed.') @skipIf(HAS_MOTO is False, 'The moto module must be installed.') @skipIf(_has_required_boto() is False, 'The boto module must be greater than' @@ -301,7 +301,7 @@ class BotoVpcRouteTableTestCase(BotoVpcStateTestCaseBase, BotoVpcResourceTestCas backend_create = 'RouteTableBackend.create_route_table' backend_delete = 'RouteTableBackend.delete_route_table' - @mock_ec2 + @mock_ec2_deprecated def test_present_with_subnets(self): vpc = self._create_vpc(name='test') subnet1 = self._create_subnet(vpc_id=vpc.id, name='test1') @@ -326,7 +326,7 @@ class BotoVpcRouteTableTestCase(BotoVpcStateTestCaseBase, BotoVpcResourceTestCas new_subnets = changes['new']['subnets_associations'] self.assertEqual(new_subnets[0]['subnet_id'], subnet2.id) - @mock_ec2 + @mock_ec2_deprecated def test_present_with_routes(self): vpc = self._create_vpc(name='test') igw = self._create_internet_gateway(name='test', vpc_id=vpc.id) diff --git a/tests/unit/states/test_docker_image.py b/tests/unit/states/test_docker_image.py index 4d94c2e239c..e96dbc9f9d0 100644 --- a/tests/unit/states/test_docker_image.py +++ b/tests/unit/states/test_docker_image.py @@ -10,14 +10,13 @@ from __future__ import absolute_import from tests.support.mixins import LoaderModuleMockMixin from tests.support.unit import skipIf, TestCase from tests.support.mock import ( - Mock, + MagicMock, NO_MOCK, NO_MOCK_REASON, patch ) # Import Salt Libs -from salt.exceptions import CommandExecutionError import salt.modules.dockermod as docker_mod import salt.states.docker_image as docker_state @@ -50,21 +49,19 @@ class DockerImageTestCase(TestCase, LoaderModuleMockMixin): if ``image:latest`` is already downloaded locally the state should not report changes. ''' - docker_inspect_image = Mock( - return_value={'Id': 'abcdefghijk'}) - docker_pull = Mock( + docker_inspect_image = MagicMock(return_value={'Id': 'abcdefghijkl'}) + docker_pull = MagicMock( return_value={'Layers': - {'Already_Pulled': ['abcdefghijk'], + {'Already_Pulled': ['abcdefghijkl'], 'Pulled': []}, 'Status': 'Image is up to date for image:latest', 'Time_Elapsed': 1.1}) - docker_list_tags = Mock( - return_value=['image:latest'] - ) + docker_list_tags = MagicMock(return_value=['image:latest']) + docker_resolve_tag = MagicMock(return_value='image:latest') __salt__ = {'docker.list_tags': docker_list_tags, 'docker.pull': docker_pull, 'docker.inspect_image': docker_inspect_image, - } + 'docker.resolve_tag': docker_resolve_tag} with patch.dict(docker_state.__dict__, {'__salt__': __salt__}): ret = docker_state.present('image:latest', force=True) @@ -89,29 +86,24 @@ class DockerImageTestCase(TestCase, LoaderModuleMockMixin): if ``image:latest`` is not downloaded and force is true should pull a new image successfuly. ''' - docker_inspect_image = Mock( - side_effect=CommandExecutionError( - 'Error 404: No such image/container: image:latest')) - docker_pull = Mock( + docker_inspect_image = MagicMock(return_value={'Id': '1234567890ab'}) + docker_pull = MagicMock( return_value={'Layers': - {'Already_Pulled': ['abcdefghijk'], - 'Pulled': ['abcdefghijk']}, - 'Status': "Image 'image:latest' was pulled", - 'Time_Elapsed': 1.1}) - docker_list_tags = Mock( - side_effect=[[], ['image:latest']] - ) + {'Pulled': ['abcdefghijkl']}, + 'Status': "Image 'image:latest' was pulled", + 'Time_Elapsed': 1.1}) + docker_list_tags = MagicMock(side_effect=[[], ['image:latest']]) + docker_resolve_tag = MagicMock(return_value='image:latest') __salt__ = {'docker.list_tags': docker_list_tags, 'docker.pull': docker_pull, 'docker.inspect_image': docker_inspect_image, - } + 'docker.resolve_tag': docker_resolve_tag} with patch.dict(docker_state.__dict__, {'__salt__': __salt__}): ret = docker_state.present('image:latest', force=True) self.assertEqual(ret, {'changes': { - 'Layers': {'Already_Pulled': ['abcdefghijk'], - 'Pulled': ['abcdefghijk']}, + 'Layers': {'Pulled': ['abcdefghijkl']}, 'Status': "Image 'image:latest' was pulled", 'Time_Elapsed': 1.1}, 'result': True, diff --git a/tests/unit/states/test_docker_network.py b/tests/unit/states/test_docker_network.py index 172deae7ac3..178fa4d9c6c 100644 --- a/tests/unit/states/test_docker_network.py +++ b/tests/unit/states/test_docker_network.py @@ -42,40 +42,137 @@ class DockerNetworkTestCase(TestCase, LoaderModuleMockMixin): ''' docker_create_network = Mock(return_value='created') docker_connect_container_to_network = Mock(return_value='connected') - docker_inspect_container = Mock(return_value={'Id': 'abcd'}) + docker_inspect_container = Mock(return_value={'Id': 'abcd', 'Name': 'container_bar'}) + # Get docker.networks to return a network with a name which is a superset of the name of + # the network which is to be created, despite this network existing we should still expect + # that the new network will be created. + # Regression test for #41982. + docker_networks = Mock(return_value=[{ + 'Name': 'network_foobar', + 'Containers': {'container': {}} + }]) __salt__ = {'docker.create_network': docker_create_network, 'docker.inspect_container': docker_inspect_container, 'docker.connect_container_to_network': docker_connect_container_to_network, - 'docker.networks': Mock(return_value=[]), + 'docker.networks': docker_networks, } with patch.dict(docker_state.__dict__, {'__salt__': __salt__}): ret = docker_state.present( 'network_foo', containers=['container'], + gateway='192.168.0.1', + ip_range='192.168.0.128/25', + subnet='192.168.0.0/24' ) docker_create_network.assert_called_with('network_foo', driver=None, driver_opts=None, - check_duplicate=True) + gateway='192.168.0.1', + ip_range='192.168.0.128/25', + subnet='192.168.0.0/24') docker_connect_container_to_network.assert_called_with('abcd', 'network_foo') self.assertEqual(ret, {'name': 'network_foo', 'comment': '', - 'changes': {'connected': 'connected', + 'changes': {'connected': ['container_bar'], 'created': 'created'}, 'result': True}) + def test_present_with_change(self): + ''' + Test docker_network.present when the specified network has properties differing from the already present network + ''' + network_details = { + 'Id': 'abcd', + 'Name': 'network_foo', + 'Driver': 'macvlan', + 'Containers': { + 'abcd': {} + }, + 'Options': { + 'parent': 'eth0' + }, + 'IPAM': { + 'Config': [ + { + 'Subnet': '192.168.0.0/24', + 'Gateway': '192.168.0.1' + } + ] + } + } + docker_networks = Mock(return_value=[network_details]) + network_details['Containers'] = {'abcd': {'Id': 'abcd', 'Name': 'container_bar'}} + docker_inspect_network = Mock(return_value=network_details) + docker_inspect_container = Mock(return_value={'Id': 'abcd', 'Name': 'container_bar'}) + docker_disconnect_container_from_network = Mock(return_value='disconnected') + docker_remove_network = Mock(return_value='removed') + docker_create_network = Mock(return_value='created') + docker_connect_container_to_network = Mock(return_value='connected') + + __salt__ = {'docker.networks': docker_networks, + 'docker.inspect_network': docker_inspect_network, + 'docker.inspect_container': docker_inspect_container, + 'docker.disconnect_container_from_network': docker_disconnect_container_from_network, + 'docker.remove_network': docker_remove_network, + 'docker.create_network': docker_create_network, + 'docker.connect_container_to_network': docker_connect_container_to_network, + } + with patch.dict(docker_state.__dict__, + {'__salt__': __salt__}): + ret = docker_state.present( + 'network_foo', + driver='macvlan', + gateway='192.168.1.1', + subnet='192.168.1.0/24', + driver_opts={'parent': 'eth1'}, + containers=['abcd'] + ) + + docker_disconnect_container_from_network.assert_called_with('abcd', 'network_foo') + docker_remove_network.assert_called_with('network_foo') + docker_create_network.assert_called_with('network_foo', + driver='macvlan', + driver_opts={'parent': 'eth1'}, + gateway='192.168.1.1', + ip_range=None, + subnet='192.168.1.0/24') + docker_connect_container_to_network.assert_called_with('abcd', 'network_foo') + + self.assertEqual(ret, {'name': 'network_foo', + 'comment': 'Network \'network_foo\' was replaced with updated config', + 'changes': { + 'updated': {'network_foo': { + 'old': { + 'driver_opts': {'parent': 'eth0'}, + 'gateway': '192.168.0.1', + 'subnet': '192.168.0.0/24' + }, + 'new': { + 'driver_opts': {'parent': 'eth1'}, + 'gateway': '192.168.1.1', + 'subnet': '192.168.1.0/24' + } + }}, + 'reconnected': ['container_bar'] + }, + 'result': True}) + def test_absent(self): ''' Test docker_network.absent ''' docker_remove_network = Mock(return_value='removed') docker_disconnect_container_from_network = Mock(return_value='disconnected') + docker_networks = Mock(return_value=[{ + 'Name': 'network_foo', + 'Containers': {'container': {}} + }]) __salt__ = { 'docker.remove_network': docker_remove_network, 'docker.disconnect_container_from_network': docker_disconnect_container_from_network, - 'docker.networks': Mock(return_value=[{'Containers': {'container': {}}}]), + 'docker.networks': docker_networks, } with patch.dict(docker_state.__dict__, {'__salt__': __salt__}): @@ -88,3 +185,32 @@ class DockerNetworkTestCase(TestCase, LoaderModuleMockMixin): 'changes': {'disconnected': 'disconnected', 'removed': 'removed'}, 'result': True}) + + def test_absent_with_matching_network(self): + ''' + Test docker_network.absent when the specified network does not exist, + but another network with a name which is a superset of the specified + name does exist. In this case we expect there to be no attempt to remove + any network. + Regression test for #41982. + ''' + docker_remove_network = Mock(return_value='removed') + docker_disconnect_container_from_network = Mock(return_value='disconnected') + docker_networks = Mock(return_value=[{ + 'Name': 'network_foobar', + 'Containers': {'container': {}} + }]) + __salt__ = { + 'docker.remove_network': docker_remove_network, + 'docker.disconnect_container_from_network': docker_disconnect_container_from_network, + 'docker.networks': docker_networks, + } + with patch.dict(docker_state.__dict__, + {'__salt__': __salt__}): + ret = docker_state.absent('network_foo') + docker_disconnect_container_from_network.assert_not_called() + docker_remove_network.assert_not_called() + self.assertEqual(ret, {'name': 'network_foo', + 'comment': 'Network \'network_foo\' already absent', + 'changes': {}, + 'result': True}) diff --git a/tests/unit/states/test_elasticsearch.py b/tests/unit/states/test_elasticsearch.py index 112ee869bba..0c2a5da0228 100644 --- a/tests/unit/states/test_elasticsearch.py +++ b/tests/unit/states/test_elasticsearch.py @@ -17,8 +17,8 @@ from tests.support.mock import ( from salt.exceptions import CommandExecutionError # Import Salt Libs +import salt.utils.dictdiffer as dictdiffer from salt.states import elasticsearch -from salt.utils import dictdiffer @skipIf(NO_MOCK, NO_MOCK_REASON) diff --git a/tests/unit/states/test_environ.py b/tests/unit/states/test_environ.py index 28c35c525f3..290015a5033 100644 --- a/tests/unit/states/test_environ.py +++ b/tests/unit/states/test_environ.py @@ -49,7 +49,7 @@ class TestEnvironState(TestCase, LoaderModuleMockMixin): def test_setenv_permanent(self): with patch.dict(envmodule.__salt__, {'reg.set_value': MagicMock(), 'reg.delete_value': MagicMock()}), \ - patch('salt.utils.is_windows', MagicMock(return_value=True)): + patch('salt.utils.platform.is_windows', MagicMock(return_value=True)): ret = envstate.setenv('test', 'value', permanent=True) self.assertEqual(ret['changes'], {'test': 'value'}) envmodule.__salt__['reg.set_value'].assert_called_with("HKCU", "Environment", 'test', 'value') @@ -96,11 +96,14 @@ class TestEnvironState(TestCase, LoaderModuleMockMixin): ret = envstate.setenv('notimportant', {'foo': 'bar'}) self.assertEqual(ret['changes'], {'foo': 'bar'}) - ret = envstate.setenv('notimportant', {'test': False, 'foo': 'baz'}, false_unsets=True) + with patch.dict(envstate.__salt__, {'reg.read_value': MagicMock()}): + ret = envstate.setenv( + 'notimportant', {'test': False, 'foo': 'baz'}, false_unsets=True) self.assertEqual(ret['changes'], {'test': None, 'foo': 'baz'}) self.assertEqual(envstate.os.environ, {'INITIAL': 'initial', 'foo': 'baz'}) - ret = envstate.setenv('notimportant', {'test': False, 'foo': 'bax'}) + with patch.dict(envstate.__salt__, {'reg.read_value': MagicMock()}): + ret = envstate.setenv('notimportant', {'test': False, 'foo': 'bax'}) self.assertEqual(ret['changes'], {'test': '', 'foo': 'bax'}) self.assertEqual(envstate.os.environ, {'INITIAL': 'initial', 'foo': 'bax', 'test': ''}) diff --git a/tests/unit/states/test_esxdatacenter.py b/tests/unit/states/test_esxdatacenter.py new file mode 100644 index 00000000000..60693f83359 --- /dev/null +++ b/tests/unit/states/test_esxdatacenter.py @@ -0,0 +1,187 @@ +# -*- coding: utf-8 -*- +''' + :codeauthor: :email:`Alexandru Bleotu ` + + Tests for functions in salt.states.esxdatacenter +''' + +# Import Python Libs +from __future__ import absolute_import + +# Import Salt Libs +import salt.states.esxdatacenter as esxdatacenter +from salt.exceptions import CommandExecutionError + +# Import Salt Testing Libs +from tests.support.mixins import LoaderModuleMockMixin +from tests.support.unit import TestCase, skipIf +from tests.support.mock import ( + MagicMock, + patch, + NO_MOCK, + NO_MOCK_REASON +) + + +@skipIf(NO_MOCK, NO_MOCK_REASON) +class DatacenterConfiguredTestCase(TestCase, LoaderModuleMockMixin): + '''Tests for salt.modules.esxdatacenter.datacenter_configured''' + + def setup_loader_modules(self): + return { + esxdatacenter: { + '__virtual__': MagicMock(return_value='esxdatacenter')}} + + def setUp(self): + self.mock_si = MagicMock() + self.mock_dc = MagicMock() + + patcher = patch.dict( + esxdatacenter.__salt__, + {'vsphere.get_proxy_type': MagicMock(), + 'vsphere.get_service_instance_via_proxy': + MagicMock(return_value=self.mock_si), + 'vsphere.list_datacenters_via_proxy': + MagicMock(return_value=[self.mock_dc]), + 'vsphere.disconnect': MagicMock()}) + patcher.start() + self.addCleanup(patcher.stop) + patcher = patch.dict(esxdatacenter.__opts__, {'test': False}) + patcher.start() + self.addCleanup(patcher.stop) + + def tearDown(self): + for attrname in ('mock_si',): + try: + delattr(self, attrname) + except AttributeError: + continue + + def test_dc_name_different_proxy(self): + with patch.dict(esxdatacenter.__salt__, + {'vsphere.get_proxy_type': + MagicMock(return_value='different_proxy')}): + res = esxdatacenter.datacenter_configured('fake_dc') + self.assertDictEqual(res, {'name': 'fake_dc', + 'changes': {}, + 'pchanges': {}, + 'result': True, + 'comment': 'Datacenter \'fake_dc\' already ' + 'exists. Nothing to be done.'}) + + def test_dc_name_esxdatacenter_proxy(self): + with patch.dict(esxdatacenter.__salt__, + {'vsphere.get_proxy_type': + MagicMock(return_value='esxdatacenter'), + 'esxdatacenter.get_details': + MagicMock(return_value={'datacenter': 'proxy_dc'})}): + res = esxdatacenter.datacenter_configured('fake_dc') + self.assertDictEqual(res, {'name': 'fake_dc', + 'changes': {}, + 'pchanges': {}, + 'result': True, + 'comment': 'Datacenter \'proxy_dc\' ' + 'already exists. Nothing to be done.'}) + + def test_get_service_instance(self): + mock_get_service_instance = MagicMock() + with patch.dict(esxdatacenter.__salt__, + {'vsphere.get_service_instance_via_proxy': + mock_get_service_instance}): + esxdatacenter.datacenter_configured('fake_dc') + mock_get_service_instance.assert_called_once_with() + + def test_list_datacenters(self): + mock_list_datacenters = MagicMock() + with patch.dict(esxdatacenter.__salt__, + {'vsphere.list_datacenters_via_proxy': + mock_list_datacenters}): + esxdatacenter.datacenter_configured('fake_dc') + mock_list_datacenters.assert_called_once_with( + datacenter_names=['fake_dc'], service_instance=self.mock_si) + + def test_create_datacenter(self): + mock_create_datacenter = MagicMock() + with patch.dict(esxdatacenter.__salt__, + {'vsphere.list_datacenters_via_proxy': + MagicMock(return_value=[]), + 'vsphere.create_datacenter': + mock_create_datacenter}): + res = esxdatacenter.datacenter_configured('fake_dc') + mock_create_datacenter.assert_called_once_with('fake_dc', self.mock_si) + self.assertDictEqual(res, + {'name': 'fake_dc', + 'changes': {'new': {'name': 'fake_dc'}}, + 'pchanges': {}, + 'result': True, + 'comment': 'Created datacenter \'fake_dc\'.'}) + + def test_create_datacenter_test_mode(self): + with patch.dict(esxdatacenter.__opts__, {'test': True}): + with patch.dict(esxdatacenter.__salt__, + {'vsphere.list_datacenters_via_proxy': + MagicMock(return_value=[])}): + res = esxdatacenter.datacenter_configured('fake_dc') + self.assertDictEqual(res, + {'name': 'fake_dc', + 'changes': {}, + 'pchanges': {'new': {'name': 'fake_dc'}}, + 'result': None, + 'comment': 'State will create ' + 'datacenter \'fake_dc\'.'}) + + def test_nothing_to_be_done_test_mode(self): + with patch.dict(esxdatacenter.__opts__, {'test': True}): + with patch.dict(esxdatacenter.__salt__, + {'vsphere.get_proxy_type': + MagicMock(return_value='different_proxy')}): + res = esxdatacenter.datacenter_configured('fake_dc') + self.assertDictEqual(res, {'name': 'fake_dc', + 'changes': {}, + 'pchanges': {}, + 'result': True, + 'comment': 'Datacenter \'fake_dc\' already ' + 'exists. Nothing to be done.'}) + + def test_state_get_service_instance_raise_command_execution_error(self): + mock_disconnect = MagicMock() + with patch.dict(esxdatacenter.__salt__, + {'vsphere.disconnect': mock_disconnect, + 'vsphere.get_service_instance_via_proxy': + MagicMock( + side_effect=CommandExecutionError('Error'))}): + res = esxdatacenter.datacenter_configured('fake_dc') + self.assertEqual(mock_disconnect.call_count, 0) + self.assertDictEqual(res, {'name': 'fake_dc', + 'changes': {}, + 'pchanges': {}, + 'result': False, + 'comment': 'Error'}) + + def test_state_raise_command_execution_error_after_si(self): + mock_disconnect = MagicMock() + with patch.dict(esxdatacenter.__salt__, + {'vsphere.disconnect': mock_disconnect, + 'vsphere.list_datacenters_via_proxy': + MagicMock( + side_effect=CommandExecutionError('Error'))}): + res = esxdatacenter.datacenter_configured('fake_dc') + mock_disconnect.assert_called_once_with(self.mock_si) + self.assertDictEqual(res, {'name': 'fake_dc', + 'changes': {}, + 'pchanges': {}, + 'result': False, + 'comment': 'Error'}) + + def test_state_raise_command_execution_error_test_mode(self): + with patch.dict(esxdatacenter.__opts__, {'test': True}): + with patch.dict(esxdatacenter.__salt__, + {'vsphere.list_datacenters_via_proxy': + MagicMock( + side_effect=CommandExecutionError('Error'))}): + res = esxdatacenter.datacenter_configured('fake_dc') + self.assertDictEqual(res, {'name': 'fake_dc', + 'changes': {}, + 'pchanges': {}, + 'result': None, + 'comment': 'Error'}) diff --git a/tests/unit/states/test_file.py b/tests/unit/states/test_file.py index dc58ca25004..7aa2c0d0f7a 100644 --- a/tests/unit/states/test_file.py +++ b/tests/unit/states/test_file.py @@ -31,9 +31,9 @@ from tests.support.mock import ( import yaml # Import salt libs -import salt import salt.utils import salt.utils.files +import salt.utils.platform import salt.states.file as filestate import salt.serializers.yaml as yamlserializer import salt.serializers.json as jsonserializer @@ -56,7 +56,8 @@ class TestFileState(TestCase, LoaderModuleMockMixin): }, '__opts__': {'test': False, 'cachedir': ''}, '__instance_id__': '', - '__low__': {} + '__low__': {}, + '__utils__': {}, } } @@ -100,7 +101,7 @@ class TestFileState(TestCase, LoaderModuleMockMixin): def test_contents_pillar_doesnt_add_more_newlines(self): # make sure the newline - pillar_value = 'i am the pillar value\n' + pillar_value = 'i am the pillar value{0}'.format(os.linesep) self.run_contents_pillar(pillar_value, expected=pillar_value) @@ -140,7 +141,7 @@ class TestFileState(TestCase, LoaderModuleMockMixin): test_dir = '/tmp' user = 'salt' - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): group = 'salt' else: group = 'saltstack' @@ -167,7 +168,9 @@ class TestFileState(TestCase, LoaderModuleMockMixin): with patch.dict(filestate.__salt__, {'config.manage_mode': mock_t, 'file.user_to_uid': mock_empty, - 'file.group_to_gid': mock_empty}): + 'file.group_to_gid': mock_empty, + 'user.info': mock_empty, + 'user.current': mock_user}): comt = ('User {0} does not exist. Group {1} does not exist.'.format(user, group)) ret.update({'comment': comt, 'name': name}) self.assertDictEqual(filestate.symlink(name, target, user=user, @@ -176,7 +179,9 @@ class TestFileState(TestCase, LoaderModuleMockMixin): with patch.dict(filestate.__salt__, {'config.manage_mode': mock_t, 'file.user_to_uid': mock_uid, 'file.group_to_gid': mock_gid, - 'file.is_link': mock_f}): + 'file.is_link': mock_f, + 'user.info': mock_empty, + 'user.current': mock_user}): with patch.dict(filestate.__opts__, {'test': True}): with patch.object(os.path, 'exists', mock_f): comt = ('Symlink {0} to {1}' @@ -191,7 +196,9 @@ class TestFileState(TestCase, LoaderModuleMockMixin): with patch.dict(filestate.__salt__, {'config.manage_mode': mock_t, 'file.user_to_uid': mock_uid, 'file.group_to_gid': mock_gid, - 'file.is_link': mock_f}): + 'file.is_link': mock_f, + 'user.info': mock_empty, + 'user.current': mock_user}): with patch.dict(filestate.__opts__, {'test': False}): with patch.object(os.path, 'isdir', mock_f): with patch.object(os.path, 'exists', mock_f): @@ -207,7 +214,9 @@ class TestFileState(TestCase, LoaderModuleMockMixin): 'file.user_to_uid': mock_uid, 'file.group_to_gid': mock_gid, 'file.is_link': mock_t, - 'file.readlink': mock_target}): + 'file.readlink': mock_target, + 'user.info': mock_empty, + 'user.current': mock_user}): with patch.dict(filestate.__opts__, {'test': False}): with patch.object(os.path, 'isdir', mock_t): with patch.object(salt.states.file, '_check_symlink_ownership', mock_t): @@ -224,7 +233,9 @@ class TestFileState(TestCase, LoaderModuleMockMixin): 'file.user_to_uid': mock_uid, 'file.group_to_gid': mock_gid, 'file.is_link': mock_f, - 'file.readlink': mock_target}): + 'file.readlink': mock_target, + 'user.info': mock_empty, + 'user.current': mock_user}): with patch.dict(filestate.__opts__, {'test': False}): with patch.object(os.path, 'isdir', mock_t): with patch.object(os.path, 'exists', mock_f): @@ -243,7 +254,9 @@ class TestFileState(TestCase, LoaderModuleMockMixin): 'file.user_to_uid': mock_uid, 'file.group_to_gid': mock_gid, 'file.is_link': mock_f, - 'file.readlink': mock_target}): + 'file.readlink': mock_target, + 'user.info': mock_empty, + 'user.current': mock_user}): with patch.dict(filestate.__opts__, {'test': False}): with patch.object(os.path, 'isdir', mock_t): with patch.object(os.path, 'exists', mock_f): @@ -538,7 +551,10 @@ class TestFileState(TestCase, LoaderModuleMockMixin): 'G12', 'G12', 'G12', 'G12', 'G12']) mock_if = MagicMock(side_effect=[True, False, False, False, False, False, False, False]) - mock_ret = MagicMock(return_value=(ret, None)) + if salt.utils.platform.is_windows(): + mock_ret = MagicMock(return_value=ret) + else: + mock_ret = MagicMock(return_value=(ret, None)) mock_dict = MagicMock(return_value={}) mock_cp = MagicMock(side_effect=[Exception, True]) mock_ex = MagicMock(side_effect=[Exception, {'changes': {name: name}}, @@ -573,8 +589,14 @@ class TestFileState(TestCase, LoaderModuleMockMixin): self.assertDictEqual(filestate.managed(name, create=False), ret) - comt = ('User salt is not available Group saltstack' - ' is not available') + # Group argument is ignored on Windows systems. Group is set to + # user + if salt.utils.platform.is_windows(): + comt = ('User salt is not available Group salt' + ' is not available') + else: + comt = ('User salt is not available Group saltstack' + ' is not available') ret.update({'comment': comt, 'result': False}) self.assertDictEqual(filestate.managed(name, user=user, group=group), ret) @@ -711,26 +733,37 @@ class TestFileState(TestCase, LoaderModuleMockMixin): mock_t = MagicMock(return_value=True) mock_f = MagicMock(return_value=False) - mock_perms = MagicMock(return_value=(ret, '')) + if salt.utils.platform.is_windows(): + mock_perms = MagicMock(return_value=ret) + else: + mock_perms = MagicMock(return_value=(ret, '')) mock_uid = MagicMock(side_effect=['', 'U12', 'U12', 'U12', 'U12', 'U12', 'U12', 'U12', 'U12', 'U12', 'U12']) mock_gid = MagicMock(side_effect=['', 'G12', 'G12', 'G12', 'G12', 'G12', 'G12', 'G12', 'G12', 'G12', 'G12']) + mock_check = MagicMock(return_value=( + None, + 'The directory "{0}" will be changed'.format(name), + {'directory': 'new'})) + mock_error = CommandExecutionError with patch.dict(filestate.__salt__, {'config.manage_mode': mock_t, 'file.user_to_uid': mock_uid, 'file.group_to_gid': mock_gid, 'file.stats': mock_f, 'file.check_perms': mock_perms, - 'file.mkdir': mock_t}): - if salt.utils.is_windows(): + 'file.mkdir': mock_t}), \ + patch('salt.utils.win_dacl.get_sid', mock_error), \ + patch('os.path.isdir', mock_t), \ + patch('salt.states.file._check_directory_win', mock_check): + if salt.utils.platform.is_windows(): comt = ('User salt is not available Group salt' ' is not available') else: comt = ('User salt is not available Group saltstack' ' is not available') ret.update({'comment': comt, 'name': name}) - self.assertDictEqual(filestate.directory(name, user=user, - group=group), ret) + self.assertDictEqual( + filestate.directory(name, user=user, group=group), ret) with patch.object(os.path, 'isabs', mock_f): comt = ('Specified file {0} is not an absolute path' @@ -771,9 +804,19 @@ class TestFileState(TestCase, LoaderModuleMockMixin): with patch.object(os.path, 'isfile', mock_f): with patch.dict(filestate.__opts__, {'test': True}): - comt = ('The following files will be changed:\n{0}:' - ' directory - new\n'.format(name)) - ret.update({'comment': comt, 'result': None, 'pchanges': {'/etc/grub.conf': {'directory': 'new'}}}) + if salt.utils.platform.is_windows(): + comt = 'The directory "{0}" will be changed' \ + ''.format(name) + p_chg = {'directory': 'new'} + else: + comt = ('The following files will be changed:\n{0}:' + ' directory - new\n'.format(name)) + p_chg = {'/etc/grub.conf': {'directory': 'new'}} + ret.update({ + 'comment': comt, + 'result': None, + 'pchanges': p_chg + }) self.assertDictEqual(filestate.directory(name, user=user, group=group), @@ -845,8 +888,14 @@ class TestFileState(TestCase, LoaderModuleMockMixin): 'file.source_list': mock_lst, 'cp.list_master_dirs': mock_emt, 'cp.list_master': mock_l}): - comt = ('User salt is not available Group saltstack' - ' is not available') + + # Group argument is ignored on Windows systems. Group is set to user + if salt.utils.platform.is_windows(): + comt = ('User salt is not available Group salt' + ' is not available') + else: + comt = ('User salt is not available Group saltstack' + ' is not available') ret.update({'comment': comt}) self.assertDictEqual(filestate.recurse(name, source, user=user, group=group), ret) @@ -954,7 +1003,8 @@ class TestFileState(TestCase, LoaderModuleMockMixin): ret.update({'comment': comt, 'name': name}) self.assertDictEqual(filestate.blockreplace(name), ret) - with patch.object(os.path, 'isabs', mock_t): + with patch.object(os.path, 'isabs', mock_t), \ + patch.object(os.path, 'exists', mock_t): with patch.dict(filestate.__salt__, {'file.blockreplace': mock_t}): with patch.dict(filestate.__opts__, {'test': True}): comt = ('Changes would be made') @@ -970,7 +1020,7 @@ class TestFileState(TestCase, LoaderModuleMockMixin): Test to comment out specified lines in a file. ''' with patch.object(os.path, 'exists', MagicMock(return_value=True)): - name = '/etc/aliases' if salt.utils.is_darwin() else '/etc/fstab' + name = '/etc/aliases' if salt.utils.platform.is_darwin() else '/etc/fstab' regex = 'bind 127.0.0.1' ret = {'name': name, @@ -1011,7 +1061,7 @@ class TestFileState(TestCase, LoaderModuleMockMixin): self.assertDictEqual(filestate.comment(name, regex), ret) with patch.dict(filestate.__opts__, {'test': False}): - with patch.object(salt.utils, 'fopen', + with patch.object(salt.utils.files, 'fopen', MagicMock(mock_open())): comt = ('Commented lines successfully') ret.update({'comment': comt, 'result': True}) @@ -1025,7 +1075,7 @@ class TestFileState(TestCase, LoaderModuleMockMixin): Test to uncomment specified commented lines in a file ''' with patch.object(os.path, 'exists', MagicMock(return_value=True)): - name = '/etc/aliases' if salt.utils.is_darwin() else '/etc/fstab' + name = '/etc/aliases' if salt.utils.platform.is_darwin() else '/etc/fstab' regex = 'bind 127.0.0.1' ret = {'name': name, @@ -1066,7 +1116,7 @@ class TestFileState(TestCase, LoaderModuleMockMixin): self.assertDictEqual(filestate.uncomment(name, regex), ret) with patch.dict(filestate.__opts__, {'test': False}): - with patch.object(salt.utils, 'fopen', + with patch.object(salt.utils.files, 'fopen', MagicMock(mock_open())): comt = ('Uncommented lines successfully') ret.update({'comment': comt, 'result': True}) @@ -1130,9 +1180,9 @@ class TestFileState(TestCase, LoaderModuleMockMixin): ret.pop('data', None) ret.update({'name': name}) - with patch.object(salt.utils, 'fopen', + with patch.object(salt.utils.files, 'fopen', MagicMock(mock_open(read_data=''))): - with patch.object(salt.utils, 'istextfile', mock_f): + with patch.dict(filestate.__utils__, {'files.is_text_file': mock_f}): with patch.dict(filestate.__opts__, {'test': True}): change = {'diff': 'Replace binary file'} comt = ('File {0} is set to be updated' @@ -1312,8 +1362,15 @@ class TestFileState(TestCase, LoaderModuleMockMixin): 'file.get_group': mock_grp, 'file.get_mode': mock_grp, 'file.check_perms': mock_t}): - comt = ('User salt is not available Group ' - 'saltstack is not available') + + # Group argument is ignored on Windows systems. Group is set + # to user + if salt.utils.platform.is_windows(): + comt = ('User salt is not available Group salt' + ' is not available') + else: + comt = ('User salt is not available Group saltstack' + ' is not available') ret.update({'comment': comt, 'result': False}) self.assertDictEqual(filestate.copy(name, source, user=user, group=group), ret) diff --git a/tests/unit/states/test_grains.py b/tests/unit/states/test_grains.py index b24e2058f41..836aed83268 100644 --- a/tests/unit/states/test_grains.py +++ b/tests/unit/states/test_grains.py @@ -17,7 +17,7 @@ from tests.support.unit import TestCase, skipIf from tests.support.mock import NO_MOCK, NO_MOCK_REASON, MagicMock, patch # Import salt libs -import salt.utils +import salt.utils.files import salt.modules.grains as grainsmod import salt.states.grains as grains @@ -62,7 +62,7 @@ class GrainsTestCase(TestCase, LoaderModuleMockMixin): grains_file = os.path.join( os.path.dirname(grains.__opts__['conf_file']), 'grains') - with salt.utils.fopen(grains_file, "r") as grf: + with salt.utils.files.fopen(grains_file, "r") as grf: grains_data = grf.read() self.assertMultiLineEqual(grains_string, grains_data) @@ -78,7 +78,7 @@ class GrainsTestCase(TestCase, LoaderModuleMockMixin): grains_file = os.path.join( os.path.dirname(grains.__opts__['conf_file']), 'grains') cstr = yaml.safe_dump(grains_data, default_flow_style=False) - with salt.utils.fopen(grains_file, "w+") as grf: + with salt.utils.files.fopen(grains_file, "w+") as grf: grf.write(cstr) yield diff --git a/tests/unit/states/test_kapacitor.py b/tests/unit/states/test_kapacitor.py index e93ff238f27..dcd396251c4 100644 --- a/tests/unit/states/test_kapacitor.py +++ b/tests/unit/states/test_kapacitor.py @@ -47,7 +47,7 @@ def _present(name='testname', 'kapacitor.enable_task': enable_mock, 'kapacitor.disable_task': disable_mock, }): - with patch('salt.utils.fopen', mock_open(read_data=script)) as open_mock: + with patch('salt.utils.files.fopen', mock_open(read_data=script)) as open_mock: retval = kapacitor.task_present(name, tick_script, task_type=task_type, database=database, retention_policy=retention_policy, enable=enable) diff --git a/tests/unit/states/test_kernelpkg.py b/tests/unit/states/test_kernelpkg.py index 24659a1d00e..dec4b5a9d71 100644 --- a/tests/unit/states/test_kernelpkg.py +++ b/tests/unit/states/test_kernelpkg.py @@ -5,6 +5,7 @@ :maturity: develop versionadded:: oxygen ''' +# pylint: disable=invalid-name,no-member # Import Python libs from __future__ import absolute_import @@ -70,7 +71,7 @@ class KernelPkgTestCase(TestCase, LoaderModuleMockMixin): self.assertTrue(ret['result']) self.assertIsInstance(ret['changes'], dict) self.assertIsInstance(ret['comment'], str) - kernelpkg.__salt__['kernelpkg.upgrade'].assert_called_once() + self.assert_called_once(kernelpkg.__salt__['kernelpkg.upgrade']) with patch.dict(kernelpkg.__opts__, {'test': True}): kernelpkg.__salt__['kernelpkg.upgrade'].reset_mock() @@ -118,7 +119,7 @@ class KernelPkgTestCase(TestCase, LoaderModuleMockMixin): self.assertTrue(ret['result']) self.assertIsInstance(ret['changes'], dict) self.assertIsInstance(ret['comment'], str) - kernelpkg.__salt__['system.reboot'].assert_called_once() + self.assert_called_once(kernelpkg.__salt__['system.reboot']) with patch.dict(kernelpkg.__opts__, {'test': True}): kernelpkg.__salt__['system.reboot'].reset_mock() diff --git a/tests/unit/states/test_ldap.py b/tests/unit/states/test_ldap.py index 51d794c91c5..cb7cf372c51 100644 --- a/tests/unit/states/test_ldap.py +++ b/tests/unit/states/test_ldap.py @@ -12,8 +12,9 @@ with something sensible. from __future__ import absolute_import import copy -import salt.ext.six as six +from salt.ext import six import salt.states.ldap +from salt.utils.oset import OrderedSet from tests.support.mixins import LoaderModuleMockMixin from tests.support.unit import skipIf, TestCase @@ -34,20 +35,20 @@ def _init_db(newdb=None): def _complex_db(): return { 'dnfoo': { - 'attrfoo1': set(( + 'attrfoo1': OrderedSet(( 'valfoo1.1', 'valfoo1.2', )), - 'attrfoo2': set(( + 'attrfoo2': OrderedSet(( 'valfoo2.1', )), }, 'dnbar': { - 'attrbar1': set(( + 'attrbar1': OrderedSet(( 'valbar1.1', 'valbar1.2', )), - 'attrbar2': set(( + 'attrbar2': OrderedSet(( 'valbar2.1', )), }, @@ -72,7 +73,7 @@ def _dummy_connect(connect_spec): def _dummy_search(connect_spec, base, scope): if base not in db: return {} - return {base: dict(((attr, sorted(db[base][attr])) + return {base: dict(((attr, list(db[base][attr])) for attr in db[base] if len(db[base][attr])))} @@ -83,7 +84,7 @@ def _dummy_add(connect_spec, dn, attributes): db[dn] = {} for attr, vals in six.iteritems(attributes): assert len(vals) - db[dn][attr] = set(vals) + db[dn][attr] = OrderedSet(vals) return True @@ -100,7 +101,7 @@ def _dummy_change(connect_spec, dn, before, after): assert dn in db e = db[dn] assert e == before - all_attrs = set() + all_attrs = OrderedSet() all_attrs.update(before) all_attrs.update(after) directives = [] @@ -131,7 +132,7 @@ def _dummy_modify(connect_spec, dn, directives): for op, attr, vals in directives: if op == 'add': assert len(vals) - existing_vals = e.setdefault(attr, set()) + existing_vals = e.setdefault(attr, OrderedSet()) for val in vals: assert val not in existing_vals existing_vals.add(val) @@ -149,7 +150,7 @@ def _dummy_modify(connect_spec, dn, directives): del e[attr] elif op == 'replace': e.pop(attr, None) - e[attr] = set(vals) + e[attr] = OrderedSet(vals) else: raise ValueError() return True @@ -158,7 +159,7 @@ def _dummy_modify(connect_spec, dn, directives): def _dump_db(d=None): if d is None: d = db - return dict(((dn, dict(((attr, sorted(d[dn][attr])) + return dict(((dn, dict(((attr, list(d[dn][attr])) for attr in d[dn]))) for dn in d)) @@ -186,8 +187,8 @@ class LDAPTestCase(TestCase, LoaderModuleMockMixin): for dn, attrs in six.iteritems(replace): for attr, vals in six.iteritems(attrs): if len(vals): - new.setdefault(dn, {})[attr] = sorted(set(vals)) - expected_db.setdefault(dn, {})[attr] = set(vals) + new.setdefault(dn, {})[attr] = list(OrderedSet(vals)) + expected_db.setdefault(dn, {})[attr] = OrderedSet(vals) elif dn in expected_db: new[dn].pop(attr, None) expected_db[dn].pop(attr, None) @@ -195,10 +196,10 @@ class LDAPTestCase(TestCase, LoaderModuleMockMixin): new.pop(dn, None) expected_db.pop(dn, None) if delete_others: - dn_to_delete = set() + dn_to_delete = OrderedSet() for dn, attrs in six.iteritems(expected_db): if dn in replace: - to_delete = set() + to_delete = OrderedSet() for attr, vals in six.iteritems(attrs): if attr not in replace[dn]: to_delete.add(attr) @@ -305,7 +306,7 @@ class LDAPTestCase(TestCase, LoaderModuleMockMixin): def test_managed_no_net_change(self): self._test_helper_nochange( _complex_db(), - {'dnfoo': {'attrfoo1': ['valfoo1.2', 'valfoo1.1']}}) + {'dnfoo': {'attrfoo1': ['valfoo1.1', 'valfoo1.2']}}) def test_managed_repeated_values(self): self._test_helper_success( diff --git a/tests/unit/states/test_libvirt.py b/tests/unit/states/test_libvirt.py index 9301ac5332d..7be24eba27f 100644 --- a/tests/unit/states/test_libvirt.py +++ b/tests/unit/states/test_libvirt.py @@ -20,7 +20,7 @@ from tests.support.mock import ( # Import Salt Libs import salt.states.virt as virt -import salt.utils +import salt.utils.files @skipIf(NO_MOCK, NO_MOCK_REASON) @@ -67,7 +67,7 @@ class LibvirtTestCase(TestCase, LoaderModuleMockMixin): self.assertDictEqual(virt.keys(name, basepath=self.pki_dir), ret) with patch.dict(virt.__opts__, {'test': False}): - with patch.object(salt.utils, 'fopen', MagicMock(mock_open())): + with patch.object(salt.utils.files, 'fopen', MagicMock(mock_open())): comt = ('Updated libvirt certs and keys') ret.update({'comment': comt, 'result': True, 'changes': {'servercert': 'new'}}) @@ -102,7 +102,7 @@ class LibvirtTestCase(TestCase, LoaderModuleMockMixin): expiration_days=700), ret) with patch.dict(virt.__opts__, {'test': False}): - with patch.object(salt.utils, 'fopen', MagicMock(mock_open())): + with patch.object(salt.utils.files, 'fopen', MagicMock(mock_open())): comt = ('Updated libvirt certs and keys') ret.update({'comment': comt, 'result': True, 'changes': {'servercert': 'new'}}) @@ -139,7 +139,7 @@ class LibvirtTestCase(TestCase, LoaderModuleMockMixin): st='California'), ret) with patch.dict(virt.__opts__, {'test': False}): - with patch.object(salt.utils, 'fopen', MagicMock(mock_open())): + with patch.object(salt.utils.files, 'fopen', MagicMock(mock_open())): comt = ('Updated libvirt certs and keys') ret.update({'comment': comt, 'result': True, 'changes': {'servercert': 'new'}}) @@ -184,7 +184,7 @@ class LibvirtTestCase(TestCase, LoaderModuleMockMixin): expiration_days=700), ret) with patch.dict(virt.__opts__, {'test': False}): - with patch.object(salt.utils, 'fopen', MagicMock(mock_open())): + with patch.object(salt.utils.files, 'fopen', MagicMock(mock_open())): comt = ('Updated libvirt certs and keys') ret.update({'comment': comt, 'result': True, 'changes': {'servercert': 'new'}}) diff --git a/tests/unit/states/test_linux_acl.py b/tests/unit/states/test_linux_acl.py index 3ade701b82c..f5488d2ae40 100644 --- a/tests/unit/states/test_linux_acl.py +++ b/tests/unit/states/test_linux_acl.py @@ -15,8 +15,10 @@ from tests.support.mock import ( MagicMock, patch) + # Import Salt Libs import salt.states.linux_acl as linux_acl +from salt.exceptions import CommandExecutionError @skipIf(NO_MOCK, NO_MOCK_REASON) @@ -34,34 +36,120 @@ class LinuxAclTestCase(TestCase, LoaderModuleMockMixin): ''' Test to ensure a Linux ACL is present ''' + self.maxDiff = None name = '/root' acl_type = 'users' acl_name = 'damian' perms = 'rwx' - ret = {'name': name, - 'result': None, - 'comment': '', - 'changes': {}} - mock = MagicMock(side_effect=[{name: {acl_type: [{acl_name: - {'octal': 'A'}}]}}, + {'octal': 'A'}}]}}, + {name: {acl_type: [{acl_name: + {'octal': 'A'}}]}}, + {name: {acl_type: [{acl_name: + {'octal': 'A'}}]}}, + {name: {acl_type: [{}]}}, + {name: {acl_type: [{}]}}, {name: {acl_type: [{}]}}, {name: {acl_type: ''}}]) + mock_modfacl = MagicMock(return_value=True) + with patch.dict(linux_acl.__salt__, {'acl.getfacl': mock}): + # Update - test=True with patch.dict(linux_acl.__opts__, {'test': True}): - comt = ('Permissions have been updated') - ret.update({'comment': comt}) + comt = ('Updated permissions will be applied for {0}: A -> {1}' + ''.format(acl_name, perms)) + ret = {'name': name, + 'comment': comt, + 'changes': {}, + 'pchanges': {'new': {'acl_name': acl_name, + 'acl_type': acl_type, + 'perms': perms}, + 'old': {'acl_name': acl_name, + 'acl_type': acl_type, + 'perms': 'A'}}, + 'result': None} + self.assertDictEqual(linux_acl.present(name, acl_type, acl_name, perms), ret) - - comt = ('Permissions will be applied') - ret.update({'comment': comt}) - self.assertDictEqual(linux_acl.present(name, acl_type, acl_name, - perms), ret) - + # Update - test=False + with patch.dict(linux_acl.__salt__, {'acl.modfacl': mock_modfacl}): + with patch.dict(linux_acl.__opts__, {'test': False}): + comt = ('Updated permissions for {0}'.format(acl_name)) + ret = {'name': name, + 'comment': comt, + 'changes': {'new': {'acl_name': acl_name, + 'acl_type': acl_type, + 'perms': perms}, + 'old': {'acl_name': acl_name, + 'acl_type': acl_type, + 'perms': 'A'}}, + 'pchanges': {}, + 'result': True} + self.assertDictEqual(linux_acl.present(name, acl_type, + acl_name, perms), + ret) + # Update - modfacl error + with patch.dict(linux_acl.__salt__, {'acl.modfacl': MagicMock( + side_effect=CommandExecutionError('Custom err'))}): + with patch.dict(linux_acl.__opts__, {'test': False}): + comt = ('Error updating permissions for {0}: Custom err' + ''.format(acl_name)) + ret = {'name': name, + 'comment': comt, + 'changes': {}, + 'pchanges': {}, + 'result': False} + self.assertDictEqual(linux_acl.present(name, acl_type, + acl_name, perms), + ret) + # New - test=True + with patch.dict(linux_acl.__salt__, {'acl.modfacl': mock_modfacl}): + with patch.dict(linux_acl.__opts__, {'test': True}): + comt = ('New permissions will be applied ' + 'for {0}: {1}'.format(acl_name, perms)) + ret = {'name': name, + 'comment': comt, + 'changes': {}, + 'pchanges': {'new': {'acl_name': acl_name, + 'acl_type': acl_type, + 'perms': perms}}, + 'result': None} + self.assertDictEqual(linux_acl.present(name, acl_type, + acl_name, perms), + ret) + # New - test=False + with patch.dict(linux_acl.__salt__, {'acl.modfacl': mock_modfacl}): + with patch.dict(linux_acl.__opts__, {'test': False}): + comt = ('Applied new permissions for {0}'.format(acl_name)) + ret = {'name': name, + 'comment': comt, + 'changes': {'new': {'acl_name': acl_name, + 'acl_type': acl_type, + 'perms': perms}}, + 'pchanges': {}, + 'result': True} + self.assertDictEqual(linux_acl.present(name, acl_type, + acl_name, perms), + ret) + # New - modfacl error + with patch.dict(linux_acl.__salt__, {'acl.modfacl': MagicMock( + side_effect=CommandExecutionError('Custom err'))}): + with patch.dict(linux_acl.__opts__, {'test': False}): + comt = ('Error updating permissions for {0}: Custom err' + ''.format(acl_name)) + ret = {'name': name, + 'comment': comt, + 'changes': {}, + 'pchanges': {}, + 'result': False} + self.assertDictEqual(linux_acl.present(name, acl_type, + acl_name, perms), + ret) + # No acl type comt = ('ACL Type does not exist') - ret.update({'comment': comt, 'result': False}) + ret = {'name': name, 'comment': comt, 'result': False, + 'changes': {}, 'pchanges': {}} self.assertDictEqual(linux_acl.present(name, acl_type, acl_name, perms), ret) diff --git a/tests/unit/states/test_lxc.py b/tests/unit/states/test_lxc.py index 868636d1961..a940839cd5b 100644 --- a/tests/unit/states/test_lxc.py +++ b/tests/unit/states/test_lxc.py @@ -16,7 +16,7 @@ from tests.support.mock import ( # Import Salt Libs import salt.states.lxc as lxc -import salt.utils +import salt.utils.versions @skipIf(NO_MOCK, NO_MOCK_REASON) @@ -246,7 +246,7 @@ class LxcTestCase(TestCase, LoaderModuleMockMixin): 'comment': comment, 'changes': {}} - with patch.object(salt.utils, 'warn_until', MagicMock()): + with patch.object(salt.utils.versions, 'warn_until', MagicMock()): with patch.dict(lxc.__opts__, {'test': True}): self.assertDictEqual(lxc.edited_conf(name), ret) diff --git a/tests/unit/states/test_modjk.py b/tests/unit/states/test_modjk.py index 1dcfe006916..c91180fe182 100644 --- a/tests/unit/states/test_modjk.py +++ b/tests/unit/states/test_modjk.py @@ -13,7 +13,7 @@ from tests.support.mock import ( # Import Salt Libs import salt.states.modjk as modjk -import salt.ext.six as six +from salt.ext import six if six.PY2: diff --git a/tests/unit/states/test_module.py b/tests/unit/states/test_module.py index 9d3d488860a..4588e20ca8f 100644 --- a/tests/unit/states/test_module.py +++ b/tests/unit/states/test_module.py @@ -73,9 +73,15 @@ class ModuleStateTest(TestCase, LoaderModuleMockMixin): keywords=None, defaults=False) + cls.bspec = ArgSpec(args=[], + varargs='names', + keywords='kwargs', + defaults=None) + @classmethod def tearDownClass(cls): del cls.aspec + del cls.bspec def test_run_module_not_available(self): ''' @@ -88,6 +94,16 @@ class ModuleStateTest(TestCase, LoaderModuleMockMixin): assert ret['comment'] == "Unavailable function: {0}.".format(CMD) assert not ret['result'] + def test_module_run_hidden_varargs(self): + ''' + Tests the return of module.run state when hidden varargs are used with + wrong type. + ''' + with patch('salt.utils.args.get_function_argspec', MagicMock(return_value=self.bspec)): + ret = module._run(CMD, m_names='anyname') + comment = "'names' must be a list." + self.assertEqual(ret['comment'], comment) + def test_run_testmode(self): ''' Tests the return of the module.run state when test=True is passed. @@ -106,7 +122,7 @@ class ModuleStateTest(TestCase, LoaderModuleMockMixin): with patch.dict(module.__salt__, {CMD: _mocked_func_named}): with patch.dict(module.__opts__, {'use_superseded': ['module.run']}): ret = module.run(**{CMD: None}) - assert ret['comment'] == "'{0}' failed: Missing arguments: name".format(CMD) + assert ret['comment'] == "'{0}' failed: Function expects 1 parameters, got only 0".format(CMD) def test_run_correct_arg(self): ''' @@ -115,7 +131,7 @@ class ModuleStateTest(TestCase, LoaderModuleMockMixin): ''' with patch.dict(module.__salt__, {CMD: _mocked_func_named}): with patch.dict(module.__opts__, {'use_superseded': ['module.run']}): - ret = module.run(**{CMD: [{'name': 'Fred'}]}) + ret = module.run(**{CMD: ['Fred']}) assert ret['comment'] == '{0}: Success'.format(CMD) assert ret['result'] diff --git a/tests/unit/states/test_mount.py b/tests/unit/states/test_mount.py index 65173ea698e..1e1886001f4 100644 --- a/tests/unit/states/test_mount.py +++ b/tests/unit/states/test_mount.py @@ -62,6 +62,8 @@ class MountTestCase(TestCase, LoaderModuleMockMixin): mock_str = MagicMock(return_value='salt') mock_user = MagicMock(return_value={'uid': 510}) mock_group = MagicMock(return_value={'gid': 100}) + mock_read_cache = MagicMock(return_value={}) + mock_write_cache = MagicMock(return_value=True) umount1 = ("Forced unmount because devices don't match. " "Wanted: /dev/sdb6, current: /dev/sdb5, /dev/sdb5") with patch.dict(mount.__grains__, {'os': 'Darwin'}): @@ -163,6 +165,8 @@ class MountTestCase(TestCase, LoaderModuleMockMixin): with patch.dict(mount.__salt__, {'mount.active': mock_mnt, 'mount.mount': mock_str, 'mount.umount': mock_f, + 'mount.read_mount_cache': mock_read_cache, + 'mount.write_mount_cache': mock_write_cache, 'mount.set_fstab': mock, 'user.info': mock_user, 'group.info': mock_group}): diff --git a/tests/unit/states/test_postgres.py b/tests/unit/states/test_postgres.py index 7950c2fdf69..ca9f8358213 100644 --- a/tests/unit/states/test_postgres.py +++ b/tests/unit/states/test_postgres.py @@ -21,7 +21,7 @@ import salt.states.postgres_schema as postgres_schema class PostgresUserTestCase(TestCase, LoaderModuleMockMixin): def setup_loader_modules(self): - patcher = patch('salt.utils.which', Mock(return_value='/usr/bin/pgsql')) + patcher = patch('salt.utils.path.which', Mock(return_value='/usr/bin/pgsql')) patcher.start() self.addCleanup(patcher.stop) self.salt_stub = { @@ -78,6 +78,7 @@ class PostgresUserTestCase(TestCase, LoaderModuleMockMixin): maintenance_db=None, login=None, password=None, + valid_until=None, createdb=None) def test_present__update(self): @@ -126,6 +127,7 @@ class PostgresUserTestCase(TestCase, LoaderModuleMockMixin): maintenance_db=None, login=True, password=None, + valid_until=None, createdb=None) def test_present__no_update(self): @@ -166,7 +168,7 @@ class PostgresUserTestCase(TestCase, LoaderModuleMockMixin): class PostgresGroupTestCase(TestCase, LoaderModuleMockMixin): def setup_loader_modules(self): - patcher = patch('salt.utils.which', Mock(return_value='/usr/bin/pgsql')) + patcher = patch('salt.utils.path.which', Mock(return_value='/usr/bin/pgsql')) patcher.start() self.addCleanup(patcher.stop) self.salt_stub = { @@ -311,7 +313,7 @@ class PostgresGroupTestCase(TestCase, LoaderModuleMockMixin): class PostgresExtensionTestCase(TestCase, LoaderModuleMockMixin): def setup_loader_modules(self): - patcher = patch('salt.utils.which', Mock(return_value='/usr/bin/pgsql')) + patcher = patch('salt.utils.path.which', Mock(return_value='/usr/bin/pgsql')) patcher.start() self.addCleanup(patcher.stop) self.salt_stub = { @@ -500,7 +502,7 @@ class PostgresExtensionTestCase(TestCase, LoaderModuleMockMixin): class PostgresSchemaTestCase(TestCase, LoaderModuleMockMixin): def setup_loader_modules(self): - patcher = patch('salt.utils.which', Mock(return_value='/usr/bin/pgsql')) + patcher = patch('salt.utils.path.which', Mock(return_value='/usr/bin/pgsql')) patcher.start() self.addCleanup(patcher.stop) self.salt_stub = { diff --git a/tests/unit/states/test_proxy.py b/tests/unit/states/test_proxy.py index 30990f614e3..ceba58e8764 100644 --- a/tests/unit/states/test_proxy.py +++ b/tests/unit/states/test_proxy.py @@ -15,7 +15,7 @@ from tests.support.mock import ( ) # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six # Import Salt Libs import salt.states.proxy as proxy diff --git a/tests/unit/states/test_rbenv.py b/tests/unit/states/test_rbenv.py index d368a54e52d..7ab9c1f1fdb 100644 --- a/tests/unit/states/test_rbenv.py +++ b/tests/unit/states/test_rbenv.py @@ -33,36 +33,71 @@ class RbenvTestCase(TestCase, LoaderModuleMockMixin): ''' Test to verify that the specified ruby is installed with rbenv. ''' - name = 'rbenv-deps' + # rbenv.is_installed is used wherever test is False. + mock_is = MagicMock(side_effect=[False, True, True, True, True]) - ret = {'name': name, - 'changes': {}, - 'result': True, - 'comment': ''} + # rbenv.install is only called when an action is attempted + # (ie. Successfully... or Failed...) + mock_i = MagicMock(side_effect=[False, False, False]) - mock_t = MagicMock(side_effect=[False, True, True]) - mock_f = MagicMock(return_value=False) - mock_def = MagicMock(return_value='2.7') - mock_ver = MagicMock(return_value=['2.7']) + # rbenv.install_ruby is only called when rbenv is successfully + # installed and an attempt to install a version of Ruby is + # made. + mock_ir = MagicMock(side_effect=[True, False]) + mock_def = MagicMock(return_value='2.3.4') + mock_ver = MagicMock(return_value=['2.3.4', '2.4.1']) with patch.dict(rbenv.__salt__, - {'rbenv.is_installed': mock_f, - 'rbenv.install': mock_t, + {'rbenv.is_installed': mock_is, + 'rbenv.install': mock_i, 'rbenv.default': mock_def, 'rbenv.versions': mock_ver, - 'rbenv.install_ruby': mock_t}): + 'rbenv.install_ruby': mock_ir}): with patch.dict(rbenv.__opts__, {'test': True}): - comt = ('Ruby rbenv-deps is set to be installed') - ret.update({'comment': comt, 'result': None}) + name = '1.9.3-p551' + comt = 'Ruby {0} is set to be installed'.format(name) + ret = {'name': name, 'changes': {}, 'comment': comt, + 'result': None} + self.assertDictEqual(rbenv.installed(name), ret) + + name = '2.4.1' + comt = 'Ruby {0} is already installed'.format(name) + ret = {'name': name, 'changes': {}, 'comment': comt, + 'default': False, 'result': True} + self.assertDictEqual(rbenv.installed(name), ret) + + name = '2.3.4' + comt = 'Ruby {0} is already installed'.format(name) + ret = {'name': name, 'changes': {}, 'comment': comt, + 'default': True, 'result': True} self.assertDictEqual(rbenv.installed(name), ret) with patch.dict(rbenv.__opts__, {'test': False}): - comt = ('Rbenv failed to install') - ret.update({'comment': comt, 'result': False}) + name = '2.4.1' + comt = 'Rbenv failed to install' + ret = {'name': name, 'changes': {}, 'comment': comt, + 'result': False} self.assertDictEqual(rbenv.installed(name), ret) - comt = ('Successfully installed ruby') - ret.update({'comment': comt, 'result': True, 'default': False, - 'changes': {name: 'Installed'}}) + comt = 'Requested ruby exists' + ret = {'name': name, 'comment': comt, 'default': False, + 'changes': {}, 'result': True} + self.assertDictEqual(rbenv.installed(name), ret) + + name = '2.3.4' + comt = 'Requested ruby exists' + ret = {'name': name, 'comment': comt, 'default': True, + 'changes': {}, 'result': True} + self.assertDictEqual(rbenv.installed(name), ret) + + name = '1.9.3-p551' + comt = 'Successfully installed ruby' + ret = {'name': name, 'comment': comt, 'default': False, + 'changes': {name: 'Installed'}, 'result': True} + self.assertDictEqual(rbenv.installed(name), ret) + + comt = 'Failed to install ruby' + ret = {'name': name, 'comment': comt, + 'changes': {}, 'result': False} self.assertDictEqual(rbenv.installed(name), ret) # 'absent' function tests: 1 @@ -71,32 +106,76 @@ class RbenvTestCase(TestCase, LoaderModuleMockMixin): ''' Test to verify that the specified ruby is not installed with rbenv. ''' - name = 'myqueue' - - ret = {'name': name, - 'changes': {}, - 'result': True, - 'comment': ''} - - mock = MagicMock(side_effect=[False, True]) - mock_def = MagicMock(return_value='2.7') - mock_ver = MagicMock(return_value=['2.7']) + # rbenv.is_installed is used for all tests here. + mock_is = MagicMock(side_effect=[False, True, True, True, False, + True, True, True, True, True]) + # rbenv.uninstall_ruby is only called when an action is + # attempted (ie. Successfully... or Failed...) + mock_uninstalled = MagicMock(side_effect=[True, False, False, True]) + mock_def = MagicMock(return_value='2.3.4') + mock_ver = MagicMock(return_value=['2.3.4', '2.4.1']) with patch.dict(rbenv.__salt__, - {'rbenv.is_installed': mock, + {'rbenv.is_installed': mock_is, 'rbenv.default': mock_def, - 'rbenv.versions': mock_ver}): + 'rbenv.versions': mock_ver, + 'rbenv.uninstall_ruby': mock_uninstalled}): + with patch.dict(rbenv.__opts__, {'test': True}): - comt = ('Ruby myqueue is set to be uninstalled') - ret.update({'comment': comt, 'result': None}) + name = '1.9.3-p551' + comt = 'Rbenv not installed, {0} not either'.format(name) + ret = {'name': name, 'changes': {}, 'comment': comt, + 'result': True} self.assertDictEqual(rbenv.absent(name), ret) + comt = 'Ruby {0} is already uninstalled'.format(name) + ret = {'name': name, 'changes': {}, 'comment': comt, + 'result': True} + self.assertDictEqual(rbenv.absent(name), ret) + + name = '2.3.4' + comt = 'Ruby {0} is set to be uninstalled'.format(name) + ret = {'name': name, 'changes': {}, 'comment': comt, + 'default': True, 'result': None} + self.assertDictEqual(rbenv.absent('2.3.4'), ret) + + name = '2.4.1' + comt = 'Ruby {0} is set to be uninstalled'.format(name) + ret = {'name': name, 'changes': {}, 'comment': comt, + 'default': False, 'result': None} + self.assertDictEqual(rbenv.absent('2.4.1'), ret) + with patch.dict(rbenv.__opts__, {'test': False}): - comt = ('Rbenv not installed, myqueue not either') - ret.update({'comment': comt, 'result': True}) + name = '1.9.3-p551' + comt = 'Rbenv not installed, {0} not either'.format(name) + ret = {'name': name, 'changes': {}, 'comment': comt, + 'result': True} self.assertDictEqual(rbenv.absent(name), ret) - comt = ('Ruby myqueue is already absent') - ret.update({'comment': comt, 'result': True}) + comt = 'Ruby {0} is already absent'.format(name) + ret = {'name': name, 'changes': {}, 'comment': comt, + 'result': True} + self.assertDictEqual(rbenv.absent(name), ret) + + name = '2.3.4' + comt = 'Successfully removed ruby' + ret = {'name': name, 'changes': {name: 'Uninstalled'}, + 'comment': comt, 'default': True, 'result': True} + self.assertDictEqual(rbenv.absent(name), ret) + + comt = 'Failed to uninstall ruby' + ret = {'name': name, 'changes': {}, 'comment': comt, + 'default': True, 'result': False} + self.assertDictEqual(rbenv.absent(name), ret) + + name = '2.4.1' + comt = 'Failed to uninstall ruby' + ret = {'name': name, 'changes': {}, 'comment': comt, + 'default': False, 'result': False} + self.assertDictEqual(rbenv.absent(name), ret) + + comt = 'Successfully removed ruby' + ret = {'name': name, 'changes': {name: 'Uninstalled'}, + 'comment': comt, 'default': False, 'result': True} self.assertDictEqual(rbenv.absent(name), ret) # 'install_rbenv' function tests: 1 @@ -112,16 +191,30 @@ class RbenvTestCase(TestCase, LoaderModuleMockMixin): 'result': True, 'comment': ''} - with patch.dict(rbenv.__opts__, {'test': True}): - comt = ('Rbenv is set to be installed') - ret.update({'comment': comt, 'result': None}) - self.assertDictEqual(rbenv.install_rbenv(name), ret) + mock_is = MagicMock(side_effect=[False, True, True, False, False]) + mock_i = MagicMock(side_effect=[False, True]) + with patch.dict(rbenv.__salt__, + {'rbenv.is_installed': mock_is, + 'rbenv.install': mock_i}): - with patch.dict(rbenv.__opts__, {'test': False}): - mock = MagicMock(side_effect=[False, True]) - with patch.dict(rbenv.__salt__, - {'rbenv.is_installed': mock, - 'rbenv.install': mock}): - comt = ('Rbenv installed') + with patch.dict(rbenv.__opts__, {'test': True}): + comt = 'Rbenv is set to be installed' + ret.update({'comment': comt, 'result': None}) + self.assertDictEqual(rbenv.install_rbenv(name), ret) + + comt = 'Rbenv is already installed' + ret.update({'comment': comt, 'result': True}) + self.assertDictEqual(rbenv.install_rbenv(name), ret) + + with patch.dict(rbenv.__opts__, {'test': False}): + comt = 'Rbenv is already installed' + ret.update({'comment': comt, 'result': True}) + self.assertDictEqual(rbenv.install_rbenv(name), ret) + + comt = 'Rbenv failed to install' + ret.update({'comment': comt, 'result': False}) + self.assertDictEqual(rbenv.install_rbenv(name), ret) + + comt = 'Rbenv installed' ret.update({'comment': comt, 'result': True}) self.assertDictEqual(rbenv.install_rbenv(name), ret) diff --git a/tests/unit/states/test_rvm.py b/tests/unit/states/test_rvm.py index a57f8ce565c..0ccd62062ce 100644 --- a/tests/unit/states/test_rvm.py +++ b/tests/unit/states/test_rvm.py @@ -12,7 +12,7 @@ from tests.support.mock import NO_MOCK, NO_MOCK_REASON, MagicMock, patch import salt.states.rvm as rvm # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six @skipIf(NO_MOCK, NO_MOCK_REASON) diff --git a/tests/unit/states/test_saltmod.py b/tests/unit/states/test_saltmod.py index 363c687abee..a884e331d08 100644 --- a/tests/unit/states/test_saltmod.py +++ b/tests/unit/states/test_saltmod.py @@ -20,6 +20,8 @@ from tests.support.mock import ( ) # Import Salt Libs +import salt.config +import salt.loader import salt.utils.jid import salt.utils.event import salt.states.saltmod as saltmod @@ -31,6 +33,10 @@ class SaltmodTestCase(TestCase, LoaderModuleMockMixin): Test cases for salt.states.saltmod ''' def setup_loader_modules(self): + utils = salt.loader.utils( + salt.config.DEFAULT_MINION_OPTS, + whitelist=['state'] + ) return { saltmod: { '__env__': 'base', @@ -41,7 +47,8 @@ class SaltmodTestCase(TestCase, LoaderModuleMockMixin): 'transport': 'tcp' }, '__salt__': {'saltutil.cmd': MagicMock()}, - '__orchestration_jid__': salt.utils.jid.gen_jid() + '__orchestration_jid__': salt.utils.jid.gen_jid({}), + '__utils__': utils, } } @@ -146,7 +153,17 @@ class SaltmodTestCase(TestCase, LoaderModuleMockMixin): del ret['__jid__'] with patch.dict(saltmod.__opts__, {'test': False}): with patch.dict(saltmod.__salt__, {'saltutil.cmd': MagicMock(return_value=test_batch_return)}): - self.assertDictEqual(saltmod.state(name, tgt, highstate=True), ret) + state_run = saltmod.state(name, tgt, highstate=True) + + # Test return without checking the comment contents. Comments are tested later. + comment = state_run.pop('comment') + ret.pop('comment') + self.assertDictEqual(state_run, ret) + + # Check the comment contents in a non-order specific way (ordering fails sometimes on PY3) + self.assertIn('States ran successfully. No changes made to', comment) + for minion in ['minion1', 'minion2', 'minion3']: + self.assertIn(minion, comment) # 'function' function tests: 1 @@ -241,8 +258,8 @@ class SaltmodTestCase(TestCase, LoaderModuleMockMixin): ''' name = 'state' - ret = {'changes': True, 'name': 'state', 'result': True, - 'comment': 'Runner function \'state\' executed.', + ret = {'changes': {}, 'name': 'state', 'result': True, + 'comment': 'Runner function \'state\' executed with return True.', '__orchestration__': True} runner_mock = MagicMock(return_value={'return': True}) @@ -257,8 +274,8 @@ class SaltmodTestCase(TestCase, LoaderModuleMockMixin): ''' name = 'state' - ret = {'changes': True, 'name': 'state', 'result': True, - 'comment': 'Wheel function \'state\' executed.', + ret = {'changes': {}, 'name': 'state', 'result': True, + 'comment': 'Wheel function \'state\' executed with return True.', '__orchestration__': True} wheel_mock = MagicMock(return_value={'return': True}) @@ -281,7 +298,7 @@ class StatemodTests(TestCase, LoaderModuleMockMixin): 'extension_modules': os.path.join(self.tmp_cachedir, 'extmods'), }, '__salt__': {'saltutil.cmd': MagicMock()}, - '__orchestration_jid__': salt.utils.jid.gen_jid() + '__orchestration_jid__': salt.utils.jid.gen_jid({}) } } diff --git a/tests/unit/states/test_selinux.py b/tests/unit/states/test_selinux.py index 36474b95c4e..f6ce3bfb8f1 100644 --- a/tests/unit/states/test_selinux.py +++ b/tests/unit/states/test_selinux.py @@ -118,9 +118,11 @@ class SelinuxTestCase(TestCase, LoaderModuleMockMixin): with patch.dict(selinux.__opts__, {'test': False}): comt = ('Boolean samba_create_home_dirs has been set to on') ret.update({'comment': comt, 'result': True}) + ret.update({'changes': {'State': {'old': 'off', 'new': 'on'}}}) self.assertDictEqual(selinux.boolean(name, value), ret) comt = ('Failed to set the boolean ' 'samba_create_home_dirs to on') - ret.update({'comment': comt, 'result': True}) + ret.update({'comment': comt, 'result': False}) + ret.update({'changes': {}}) self.assertDictEqual(selinux.boolean(name, value), ret) diff --git a/tests/unit/states/test_syslog_ng.py b/tests/unit/states/test_syslog_ng.py index 8a2924779fe..a2984dfec57 100644 --- a/tests/unit/states/test_syslog_ng.py +++ b/tests/unit/states/test_syslog_ng.py @@ -14,7 +14,7 @@ from tests.support.mixins import LoaderModuleMockMixin from tests.support.unit import skipIf, TestCase from tests.support.mock import NO_MOCK, NO_MOCK_REASON, MagicMock, patch -import salt.utils +import salt.utils.files import salt.states.syslog_ng as syslog_ng import salt.modules.syslog_ng as syslog_ng_module @@ -362,7 +362,7 @@ class SyslogNGTestCase(TestCase, LoaderModuleMockMixin): got = syslog_ng.config(id, config=parsed_yaml_config, write=True) written_config = "" - with salt.utils.fopen(config_file_name, "r") as f: + with salt.utils.files.fopen(config_file_name, "r") as f: written_config = f.read() config_without_whitespaces = remove_whitespaces(written_config) diff --git a/tests/unit/states/test_win_servermanager.py b/tests/unit/states/test_win_servermanager.py index 8f55809bcb3..398f2ef2328 100644 --- a/tests/unit/states/test_win_servermanager.py +++ b/tests/unit/states/test_win_servermanager.py @@ -40,15 +40,26 @@ class WinServermanagerTestCase(TestCase, LoaderModuleMockMixin): 'squidward': 'patrick'}]) mock_install = MagicMock( return_value={'Success': True, + 'Restarted': False, 'RestartNeeded': False, - 'ExitCode': 1234}) + 'ExitCode': 1234, + 'Features': { + 'squidward': { + 'DisplayName': 'Squidward', + 'Message': '', + 'RestartNeeded': True, + 'SkipReason': 0, + 'Success': True + } + }}) with patch.dict(win_servermanager.__salt__, {"win_servermanager.list_installed": mock_list, "win_servermanager.install": mock_install}): ret = {'name': 'spongebob', 'changes': {}, 'result': True, - 'comment': 'The feature spongebob is already installed'} + 'comment': 'The following features are already installed:\n' + '- spongebob'} self.assertDictEqual(win_servermanager.installed('spongebob'), ret) with patch.dict(win_servermanager.__opts__, {"test": True}): @@ -56,21 +67,16 @@ class WinServermanagerTestCase(TestCase, LoaderModuleMockMixin): 'result': None, 'comment': '', 'changes': { - 'feature': 'spongebob will be installed ' - 'recurse=False'}} + 'spongebob': 'Will be installed recurse=False'}} self.assertDictEqual( win_servermanager.installed('spongebob'), ret) with patch.dict(win_servermanager.__opts__, {"test": False}): ret = {'name': 'squidward', 'result': True, - 'comment': 'Installed squidward', + 'comment': 'Installed the following:\n- squidward', 'changes': { - 'Success': True, - 'RestartNeeded': False, - 'ExitCode': 1234, - 'feature': {'squidward': {'new': 'patrick', - 'old': ''}}}} + 'squidward': {'new': 'patrick', 'old': ''}}} self.assertDictEqual( win_servermanager.installed('squidward'), ret) @@ -87,14 +93,25 @@ class WinServermanagerTestCase(TestCase, LoaderModuleMockMixin): mock_remove = MagicMock( return_value={'Success': True, 'RestartNeeded': False, - 'ExitCode': 1234}) + 'Restarted': False, + 'ExitCode': 1234, + 'Features': { + 'squidward': { + 'DisplayName': 'Squidward', + 'Message': '', + 'RestartNeeded': True, + 'SkipReason': 0, + 'Success': True + } + }}) with patch.dict(win_servermanager.__salt__, {"win_servermanager.list_installed": mock_list, "win_servermanager.remove": mock_remove}): ret = {'name': 'squidward', 'changes': {}, 'result': True, - 'comment': 'The feature squidward is not installed'} + 'comment': 'The following features are not installed:\n' + '- squidward'} self.assertDictEqual( win_servermanager.removed('squidward'), ret) @@ -102,19 +119,15 @@ class WinServermanagerTestCase(TestCase, LoaderModuleMockMixin): ret = {'name': 'squidward', 'result': None, 'comment': '', - 'changes': {'feature': 'squidward will be removed'}} + 'changes': {'squidward': 'Will be removed'}} self.assertDictEqual( win_servermanager.removed('squidward'), ret) with patch.dict(win_servermanager.__opts__, {"test": False}): ret = {'name': 'squidward', 'result': True, - 'comment': 'Removed squidward', + 'comment': 'Removed the following:\n- squidward', 'changes': { - 'Success': True, - 'RestartNeeded': False, - 'ExitCode': 1234, - 'feature': {'squidward': {'new': '', - 'old': 'patrick'}}}} + 'squidward': {'new': '', 'old': 'patrick'}}} self.assertDictEqual( win_servermanager.removed('squidward'), ret) diff --git a/tests/unit/states/test_win_snmp.py b/tests/unit/states/test_win_snmp.py index 3618d01ab0e..69ffad1c6a8 100644 --- a/tests/unit/states/test_win_snmp.py +++ b/tests/unit/states/test_win_snmp.py @@ -11,7 +11,7 @@ from __future__ import absolute_import # Import Salt Libs import salt.states.win_snmp as win_snmp -import salt.ext.six as six +from salt.ext import six # Import Salt Testing Libs from tests.support.mixins import LoaderModuleMockMixin diff --git a/tests/unit/states/test_winrepo.py b/tests/unit/states/test_winrepo.py index a702bbd2de0..d5e4321620d 100644 --- a/tests/unit/states/test_winrepo.py +++ b/tests/unit/states/test_winrepo.py @@ -67,9 +67,8 @@ class WinrepoTestCase(TestCase, LoaderModuleMockMixin): 'changes': {}, 'result': False, 'comment': ''} - ret.update({'comment': - '{file_roots}/win/repo is ' - 'missing'.format(file_roots=BASE_FILE_ROOTS_DIR)}) + ret.update({'comment': '{0} is missing'.format( + os.sep.join([BASE_FILE_ROOTS_DIR, 'win', 'repo']))}) self.assertDictEqual(winrepo.genrepo('salt'), ret) mock = MagicMock(return_value={'winrepo_dir': 'salt', diff --git a/tests/unit/states/test_zcbuildout.py b/tests/unit/states/test_zcbuildout.py index d4e6ec30269..a1e0a2bdef7 100644 --- a/tests/unit/states/test_zcbuildout.py +++ b/tests/unit/states/test_zcbuildout.py @@ -10,7 +10,7 @@ from tests.support.unit import skipIf from tests.support.helpers import requires_network # Import Salt libs -import salt.utils +import salt.utils.path from tests.unit.modules.test_zcbuildout import Base, KNOWN_VIRTUALENV_BINARY_NAMES import salt.modules.zcbuildout as modbuildout import salt.states.zcbuildout as buildout @@ -19,7 +19,7 @@ import salt.modules.cmdmod as cmd ROOT = os.path.join(FILES, 'file/base/buildout') -@skipIf(salt.utils.which_bin(KNOWN_VIRTUALENV_BINARY_NAMES) is None, +@skipIf(salt.utils.path.which_bin(KNOWN_VIRTUALENV_BINARY_NAMES) is None, 'The \'virtualenv\' packaged needs to be installed') class BuildoutTestCase(Base): diff --git a/tests/unit/templates/test_jinja.py b/tests/unit/templates/test_jinja.py index ab99f53d2fc..146e2307470 100644 --- a/tests/unit/templates/test_jinja.py +++ b/tests/unit/templates/test_jinja.py @@ -19,13 +19,13 @@ from tests.support.paths import TMP_CONF_DIR # Import salt libs import salt.config -import salt.ext.six as six +from salt.ext import six import salt.loader -import salt.utils +import salt.utils.files +from salt.utils import get_context from salt.exceptions import SaltRenderError from salt.ext.six.moves import builtins -from salt.utils import get_context -from salt.utils.decorators import JinjaFilter +from salt.utils.decorators.jinja import JinjaFilter from salt.utils.jinja import ( SaltCacheLoader, SerializerExtension, @@ -173,7 +173,7 @@ class TestGetTemplate(TestCase): if the file is not contained in the searchpath ''' fn_ = os.path.join(TEMPLATES_DIR, 'files', 'test', 'hello_simple') - with salt.utils.fopen(fn_) as fp_: + with salt.utils.files.fopen(fn_) as fp_: out = render_jinja_tmpl( fp_.read(), dict( @@ -189,7 +189,7 @@ class TestGetTemplate(TestCase): if the file is not contained in the searchpath ''' filename = os.path.join(TEMPLATES_DIR, 'files', 'test', 'hello_import') - with salt.utils.fopen(filename) as fp_: + with salt.utils.files.fopen(filename) as fp_: out = render_jinja_tmpl( fp_.read(), dict( @@ -209,7 +209,7 @@ class TestGetTemplate(TestCase): fc = MockFileClient() with patch.object(SaltCacheLoader, 'file_client', MagicMock(return_value=fc)): filename = os.path.join(TEMPLATES_DIR, 'files', 'test', 'hello_import') - with salt.utils.fopen(filename) as fp_: + with salt.utils.files.fopen(filename) as fp_: out = render_jinja_tmpl( fp_.read(), dict(opts={'cachedir': TEMPLATES_DIR, 'file_client': 'remote', @@ -235,7 +235,7 @@ class TestGetTemplate(TestCase): 'files', 'test', 'hello_import_generalerror') fc = MockFileClient() with patch.object(SaltCacheLoader, 'file_client', MagicMock(return_value=fc)): - with salt.utils.fopen(filename) as fp_: + with salt.utils.files.fopen(filename) as fp_: self.assertRaisesRegex( SaltRenderError, expected, @@ -259,7 +259,7 @@ class TestGetTemplate(TestCase): 'files', 'test', 'hello_import_undefined') fc = MockFileClient() with patch.object(SaltCacheLoader, 'file_client', MagicMock(return_value=fc)): - with salt.utils.fopen(filename) as fp_: + with salt.utils.files.fopen(filename) as fp_: self.assertRaisesRegex( SaltRenderError, expected, @@ -283,7 +283,7 @@ class TestGetTemplate(TestCase): 'files', 'test', 'hello_import_error') fc = MockFileClient() with patch.object(SaltCacheLoader, 'file_client', MagicMock(return_value=fc)): - with salt.utils.fopen(filename) as fp_: + with salt.utils.files.fopen(filename) as fp_: self.assertRaisesRegex( SaltRenderError, expected, @@ -295,7 +295,7 @@ class TestGetTemplate(TestCase): fc = MockFileClient() with patch.object(SaltCacheLoader, 'file_client', MagicMock(return_value=fc)): filename = os.path.join(TEMPLATES_DIR, 'files', 'test', 'hello_import') - with salt.utils.fopen(filename) as fp_: + with salt.utils.files.fopen(filename) as fp_: out = render_jinja_tmpl( fp_.read(), dict(opts={'cachedir': TEMPLATES_DIR, 'file_client': 'remote', @@ -306,7 +306,7 @@ class TestGetTemplate(TestCase): self.assertEqual(fc.requests[0]['path'], 'salt://macro') filename = os.path.join(TEMPLATES_DIR, 'files', 'test', 'non_ascii') - with salt.utils.fopen(filename) as fp_: + with salt.utils.files.fopen(filename) as fp_: out = render_jinja_tmpl( fp_.read(), dict(opts={'cachedir': TEMPLATES_DIR, 'file_client': 'remote', @@ -373,7 +373,7 @@ class TestGetTemplate(TestCase): saltenv='test', salt=self.local_salt ) - with salt.utils.fopen(out['data']) as fp: + with salt.utils.files.fopen(out['data']) as fp: result = fp.read() if six.PY2: result = result.decode('utf-8') @@ -524,7 +524,7 @@ class TestCustomExtensions(TestCase): env.filters.update(JinjaFilter.salt_jinja_filters) if six.PY3: rendered = env.from_string('{{ dataset|unique }}').render(dataset=dataset).strip("'{}").split("', '") - self.assertEqual(rendered, list(unique)) + self.assertEqual(sorted(rendered), sorted(list(unique))) else: rendered = env.from_string('{{ dataset|unique }}').render(dataset=dataset) self.assertEqual(rendered, u"{0}".format(unique)) @@ -536,7 +536,7 @@ class TestCustomExtensions(TestCase): env.filters.update(JinjaFilter.salt_jinja_filters) if six.PY3: rendered = env.from_string('{{ dataset|unique }}').render(dataset=dataset).strip("'{}").split("', '") - self.assertEqual(rendered, list(unique)) + self.assertEqual(sorted(rendered), sorted(list(unique))) else: rendered = env.from_string('{{ dataset|unique }}').render(dataset=dataset) self.assertEqual(rendered, u"{0}".format(unique)) diff --git a/tests/unit/test_auth.py b/tests/unit/test_auth.py index e8d868e7fff..d08f9e71b19 100644 --- a/tests/unit/test_auth.py +++ b/tests/unit/test_auth.py @@ -22,7 +22,10 @@ class LoadAuthTestCase(TestCase): def setUp(self): # pylint: disable=W0221 patches = ( ('salt.payload.Serial', None), - ('salt.loader.auth', dict(return_value={'pam.auth': 'fake_func_str', 'pam.groups': 'fake_groups_function_str'})) + ('salt.loader.auth', dict(return_value={'pam.auth': 'fake_func_str', 'pam.groups': 'fake_groups_function_str'})), + ('salt.loader.eauth_tokens', dict(return_value={'localfs.mk_token': 'fake_func_mktok', + 'localfs.get_token': 'fake_func_gettok', + 'localfs.rm_roken': 'fake_func_rmtok'})) ) for mod, mock in patches: if mock: @@ -46,7 +49,8 @@ class LoadAuthTestCase(TestCase): self.assertEqual(ret, '', "Did not bail when the auth loader didn't have the auth type.") # Test a case with valid params - with patch('salt.utils.arg_lookup', MagicMock(return_value={'args': ['username', 'password']})) as format_call_mock: + with patch('salt.utils.args.arg_lookup', + MagicMock(return_value={'args': ['username', 'password']})) as format_call_mock: expected_ret = call('fake_func_str') ret = self.lauth.load_name(valid_eauth_load) format_call_mock.assert_has_calls((expected_ret,), any_order=True) @@ -151,7 +155,8 @@ class MasterACLTestCase(ModuleCase): Test to ensure a simple name can auth against a given function. This tests to ensure test_user can access test.ping but *not* sys.doc ''' - with patch('salt.utils.minions.CkMinions.check_minions', MagicMock(return_value='some_minions')): + _check_minions_return = {'minions': ['some_minions'], 'missing': []} + with patch('salt.utils.minions.CkMinions.check_minions', MagicMock(return_value=_check_minions_return)): # Can we access test.ping? self.clear.publish(self.valid_clear_load) self.assertEqual(self.fire_event_mock.call_args[0][0]['fun'], 'test.ping') @@ -166,7 +171,8 @@ class MasterACLTestCase(ModuleCase): ''' Tests to ensure test_group can access test.echo but *not* sys.doc ''' - with patch('salt.utils.minions.CkMinions.check_minions', MagicMock(return_value='some_minions')): + _check_minions_return = {'minions': ['some_minions'], 'missing': []} + with patch('salt.utils.minions.CkMinions.check_minions', MagicMock(return_value=_check_minions_return)): self.valid_clear_load['kwargs']['user'] = 'new_user' self.valid_clear_load['fun'] = 'test.echo' self.valid_clear_load['arg'] = 'hello' @@ -232,7 +238,8 @@ class MasterACLTestCase(ModuleCase): requested_tgt = 'minion_glob1' self.valid_clear_load['tgt'] = requested_tgt self.valid_clear_load['fun'] = requested_function - with patch('salt.utils.minions.CkMinions.check_minions', MagicMock(return_value=['minion_glob1'])): # Assume that there is a listening minion match + _check_minions_return = {'minions': ['minion_glob1'], 'missing': []} + with patch('salt.utils.minions.CkMinions.check_minions', MagicMock(return_value=_check_minions_return)): # Assume that there is a listening minion match self.clear.publish(self.valid_clear_load) self.assertTrue(self.fire_event_mock.called, 'Did not fire {0} for minion tgt {1}'.format(requested_function, requested_tgt)) self.assertEqual(self.fire_event_mock.call_args[0][0]['fun'], requested_function, 'Did not fire {0} for minion glob'.format(requested_function)) @@ -258,7 +265,8 @@ class MasterACLTestCase(ModuleCase): minion1: - test.empty: ''' - with patch('salt.utils.minions.CkMinions.check_minions', MagicMock(return_value='minion1')): + _check_minions_return = {'minions': ['minion1'], 'missing': []} + with patch('salt.utils.minions.CkMinions.check_minions', MagicMock(return_value=_check_minions_return)): self.valid_clear_load['kwargs'].update({'username': 'test_user_func'}) self.valid_clear_load.update({'user': 'test_user_func', 'tgt': 'minion1', @@ -278,7 +286,8 @@ class MasterACLTestCase(ModuleCase): - 'TEST' - 'TEST.*' ''' - with patch('salt.utils.minions.CkMinions.check_minions', MagicMock(return_value='minion1')): + _check_minions_return = {'minions': ['minion1'], 'missing': []} + with patch('salt.utils.minions.CkMinions.check_minions', MagicMock(return_value=_check_minions_return)): self.valid_clear_load['kwargs'].update({'username': 'test_user_func'}) self.valid_clear_load.update({'user': 'test_user_func', 'tgt': 'minion1', @@ -298,7 +307,8 @@ class MasterACLTestCase(ModuleCase): - 'TEST' - 'TEST.*' ''' - with patch('salt.utils.minions.CkMinions.check_minions', MagicMock(return_value='minion1')): + _check_minions_return = {'minions': ['minion1'], 'missing': []} + with patch('salt.utils.minions.CkMinions.check_minions', MagicMock(return_value=_check_minions_return)): self.valid_clear_load['kwargs'].update({'username': 'test_user_func'}) self.valid_clear_load.update({'user': 'test_user_func', 'tgt': 'minion1', @@ -323,7 +333,8 @@ class MasterACLTestCase(ModuleCase): - 'TEST' - 'TEST.*' ''' - with patch('salt.utils.minions.CkMinions.check_minions', MagicMock(return_value='minion1')): + _check_minions_return = {'minions': ['minion1'], 'missing': []} + with patch('salt.utils.minions.CkMinions.check_minions', MagicMock(return_value=_check_minions_return)): self.valid_clear_load['kwargs'].update({'username': 'test_user_func'}) # Wrong last arg self.valid_clear_load.update({'user': 'test_user_func', @@ -355,7 +366,8 @@ class MasterACLTestCase(ModuleCase): kwargs: text: 'KWMSG:.*' ''' - with patch('salt.utils.minions.CkMinions.check_minions', MagicMock(return_value='some_minions')): + _check_minions_return = {'minions': ['some_minions'], 'missing': []} + with patch('salt.utils.minions.CkMinions.check_minions', MagicMock(return_value=_check_minions_return)): self.valid_clear_load['kwargs'].update({'username': 'test_user_func'}) self.valid_clear_load.update({'user': 'test_user_func', 'tgt': '*', @@ -377,7 +389,8 @@ class MasterACLTestCase(ModuleCase): kwargs: text: 'KWMSG:.*' ''' - with patch('salt.utils.minions.CkMinions.check_minions', MagicMock(return_value='some_minions')): + _check_minions_return = {'minions': ['some_minions'], 'missing': []} + with patch('salt.utils.minions.CkMinions.check_minions', MagicMock(return_value=_check_minions_return)): self.valid_clear_load['kwargs'].update({'username': 'test_user_func'}) self.valid_clear_load.update({'user': 'test_user_func', 'tgt': '*', @@ -430,7 +443,8 @@ class MasterACLTestCase(ModuleCase): 'kwa': 'kwa.*' 'kwb': 'kwb' ''' - with patch('salt.utils.minions.CkMinions.check_minions', MagicMock(return_value='some_minions')): + _check_minions_return = {'minions': ['some_minions'], 'missing': []} + with patch('salt.utils.minions.CkMinions.check_minions', MagicMock(return_value=_check_minions_return)): self.valid_clear_load['kwargs'].update({'username': 'test_user_func'}) self.valid_clear_load.update({'user': 'test_user_func', 'tgt': '*', @@ -460,7 +474,8 @@ class MasterACLTestCase(ModuleCase): 'kwa': 'kwa.*' 'kwb': 'kwb' ''' - with patch('salt.utils.minions.CkMinions.check_minions', MagicMock(return_value='some_minions')): + _check_minions_return = {'minions': ['some_minions'], 'missing': []} + with patch('salt.utils.minions.CkMinions.check_minions', MagicMock(return_value=_check_minions_return)): self.valid_clear_load['kwargs'].update({'username': 'test_user_func'}) self.valid_clear_load.update({'user': 'test_user_func', 'tgt': '*', diff --git a/tests/unit/test_crypt.py b/tests/unit/test_crypt.py index c3be622f965..9e3708402cd 100644 --- a/tests/unit/test_crypt.py +++ b/tests/unit/test_crypt.py @@ -10,14 +10,21 @@ from tests.support.mock import patch, call, mock_open, NO_MOCK, NO_MOCK_REASON, # salt libs import salt.utils +import salt.utils.files from salt import crypt # third-party libs try: - import Crypto.PublicKey.RSA # pylint: disable=unused-import + from Cryptodome.PublicKey import RSA # pylint: disable=unused-import HAS_PYCRYPTO_RSA = True except ImportError: HAS_PYCRYPTO_RSA = False +if not HAS_PYCRYPTO_RSA: + try: + from Crypto.PublicKey import RSA + HAS_PYCRYPTO_RSA = True + except ImportError: + HAS_PYCRYPTO_RSA = False PRIVKEY_DATA = ( @@ -82,25 +89,47 @@ SIG = ( class CryptTestCase(TestCase): def test_gen_keys(self): - with patch.multiple(os, umask=MagicMock(), chmod=MagicMock(), chown=MagicMock, + with patch.multiple(os, umask=MagicMock(), chmod=MagicMock(), access=MagicMock(return_value=True)): - with patch('salt.utils.fopen', mock_open()): - open_priv_wb = call('/keydir/keyname.pem', 'wb+') - open_pub_wb = call('/keydir/keyname.pub', 'wb+') + with patch('salt.utils.files.fopen', mock_open()): + open_priv_wb = call('/keydir{0}keyname.pem'.format(os.sep), 'wb+') + open_pub_wb = call('/keydir{0}keyname.pub'.format(os.sep), 'wb+') with patch('os.path.isfile', return_value=True): - self.assertEqual(crypt.gen_keys('/keydir', 'keyname', 2048), '/keydir/keyname.pem') - self.assertNotIn(open_priv_wb, salt.utils.fopen.mock_calls) - self.assertNotIn(open_pub_wb, salt.utils.fopen.mock_calls) + self.assertEqual(crypt.gen_keys('/keydir', 'keyname', 2048), '/keydir{0}keyname.pem'.format(os.sep)) + self.assertNotIn(open_priv_wb, salt.utils.files.fopen.mock_calls) + self.assertNotIn(open_pub_wb, salt.utils.files.fopen.mock_calls) with patch('os.path.isfile', return_value=False): - with patch('salt.utils.fopen', mock_open()): + with patch('salt.utils.files.fopen', mock_open()): crypt.gen_keys('/keydir', 'keyname', 2048) - salt.utils.fopen.assert_has_calls([open_priv_wb, open_pub_wb], any_order=True) + salt.utils.files.fopen.assert_has_calls([open_priv_wb, open_pub_wb], any_order=True) + + @patch('os.umask', MagicMock()) + @patch('os.chmod', MagicMock()) + @patch('os.chown', MagicMock()) + @patch('os.access', MagicMock(return_value=True)) + def test_gen_keys_with_passphrase(self): + with patch('salt.utils.files.fopen', mock_open()): + open_priv_wb = call('/keydir/keyname.pem', 'wb+') + open_pub_wb = call('/keydir/keyname.pub', 'wb+') + with patch('os.path.isfile', return_value=True): + self.assertEqual(crypt.gen_keys('/keydir', 'keyname', 2048, passphrase='password'), '/keydir/keyname.pem') + self.assertNotIn(open_priv_wb, salt.utils.files.fopen.mock_calls) + self.assertNotIn(open_pub_wb, salt.utils.files.fopen.mock_calls) + with patch('os.path.isfile', return_value=False): + with patch('salt.utils.files.fopen', mock_open()): + crypt.gen_keys('/keydir', 'keyname', 2048) + salt.utils.files.fopen.assert_has_calls([open_priv_wb, open_pub_wb], any_order=True) def test_sign_message(self): - key = Crypto.PublicKey.RSA.importKey(PRIVKEY_DATA) + key = RSA.importKey(PRIVKEY_DATA) with patch('salt.crypt._get_rsa_key', return_value=key): self.assertEqual(SIG, salt.crypt.sign_message('/keydir/keyname.pem', MSG)) + def test_sign_message_with_passphrase(self): + key = RSA.importKey(PRIVKEY_DATA) + with patch('salt.crypt._get_rsa_key', return_value=key): + self.assertEqual(SIG, crypt.sign_message('/keydir/keyname.pem', MSG, passphrase='password')) + def test_verify_signature(self): - with patch('salt.utils.fopen', mock_open(read_data=PUBKEY_DATA)): + with patch('salt.utils.files.fopen', mock_open(read_data=PUBKEY_DATA)): self.assertTrue(crypt.verify_signature('/keydir/keyname.pub', MSG, SIG)) diff --git a/tests/unit/test_doc.py b/tests/unit/test_doc.py index 336833a46ca..c3e7e7db17b 100644 --- a/tests/unit/test_doc.py +++ b/tests/unit/test_doc.py @@ -6,6 +6,7 @@ # Import Python libs from __future__ import absolute_import +import os # Import Salt Testing libs from tests.support.unit import TestCase @@ -13,6 +14,7 @@ from tests.support.unit import TestCase # Import Salt libs import tests.integration as integration import salt.modules.cmdmod +import salt.utils.platform class DocTestCase(TestCase): @@ -32,8 +34,15 @@ class DocTestCase(TestCase): https://github.com/saltstack/salt/issues/12788 ''' salt_dir = integration.CODE_DIR - salt_dir += '/' - cmd = 'grep -r :doc: ' + salt_dir + + if salt.utils.platform.is_windows(): + # No grep in Windows, use findstr + # findstr in windows doesn't prepend 'Binary` to binary files, so + # use the '/P' switch to skip files with unprintable characters + cmd = 'findstr /C:":doc:" /S /P {0}\\*'.format(salt_dir) + else: + salt_dir += '/' + cmd = 'grep -r :doc: ' + salt_dir grep_call = salt.modules.cmdmod.run_stdout(cmd=cmd).split('\n') @@ -43,25 +52,30 @@ class DocTestCase(TestCase): if line.startswith('Binary'): continue - key, val = line.split(':', 1) + if salt.utils.platform.is_windows(): + # Need the space after the colon so it doesn't split the drive + # letter + key, val = line.split(': ', 1) + else: + key, val = line.split(':', 1) # Don't test man pages, this file, # the page that documents to not use ":doc:", or # the doc/conf.py file if 'man' in key \ or key.endswith('test_doc.py') \ - or key.endswith('doc/conf.py') \ - or key.endswith('/conventions/documentation.rst') \ - or key.endswith('doc/topics/releases/2016.11.2.rst') \ - or key.endswith('doc/topics/releases/2016.11.3.rst') \ - or key.endswith('doc/topics/releases/2016.3.5.rst'): + or key.endswith(os.sep.join(['doc', 'conf.py'])) \ + or key.endswith(os.sep.join(['conventions', 'documentation.rst'])) \ + or key.endswith(os.sep.join(['doc', 'topics', 'releases', '2016.11.2.rst'])) \ + or key.endswith(os.sep.join(['doc', 'topics', 'releases', '2016.11.3.rst'])) \ + or key.endswith(os.sep.join(['doc', 'topics', 'releases', '2016.3.5.rst'])): continue # Set up test return dict if test_ret.get(key) is None: - test_ret[key] = [val.lstrip()] + test_ret[key] = [val.strip()] else: - test_ret[key].append(val.lstrip()) + test_ret[key].append(val.strip()) # Allow test results to show files with :doc: ref, rather than truncating self.maxDiff = None diff --git a/tests/unit/test_fileclient.py b/tests/unit/test_fileclient.py index 077630693a2..b6bd2207e02 100644 --- a/tests/unit/test_fileclient.py +++ b/tests/unit/test_fileclient.py @@ -6,6 +6,7 @@ # Import Python libs from __future__ import absolute_import import errno +import os # Import Salt Testing libs from tests.support.mock import patch, Mock @@ -38,7 +39,7 @@ class FileclientTestCase(TestCase): for exists in range(2): with patch('os.makedirs', self._fake_makedir()): with Client(self.opts)._cache_loc('testfile') as c_ref_itr: - assert c_ref_itr == '/__test__/files/base/testfile' + assert c_ref_itr == os.sep + os.sep.join(['__test__', 'files', 'base', 'testfile']) def test_cache_raises_exception_on_non_eexist_ioerror(self): ''' diff --git a/tests/unit/test_files.py b/tests/unit/test_files.py index d3816084d4b..bb425b71502 100644 --- a/tests/unit/test_files.py +++ b/tests/unit/test_files.py @@ -14,11 +14,8 @@ import tempfile from tests.support.unit import TestCase # Import Salt libs -import salt.utils -from salt.utils import files as util_files - -# Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six +import salt.utils.files class FilesTestCase(TestCase): @@ -38,7 +35,7 @@ class FilesTestCase(TestCase): os.makedirs(current_directory) for name, content in six.iteritems(files): path = os.path.join(temp_directory, folder, name) - with salt.utils.fopen(path, 'w+') as fh: + with salt.utils.files.fopen(path, 'w+') as fh: fh.write(content) def _validate_folder_structure_and_contents(self, target_directory, @@ -46,7 +43,7 @@ class FilesTestCase(TestCase): for folder, files in six.iteritems(desired_structure): for name, content in six.iteritems(files): path = os.path.join(target_directory, folder, name) - with salt.utils.fopen(path) as fh: + with salt.utils.files.fopen(path) as fh: assert fh.read().strip() == content def setUp(self): @@ -71,7 +68,7 @@ class FilesTestCase(TestCase): } self._create_temp_structure(test_target_directory, TARGET_STRUCTURE) try: - util_files.recursive_copy(self.temp_dir, test_target_directory) + salt.utils.files.recursive_copy(self.temp_dir, test_target_directory) DESIRED_STRUCTURE = copy.copy(TARGET_STRUCTURE) DESIRED_STRUCTURE.update(self.STRUCTURE) self._validate_folder_structure_and_contents( diff --git a/tests/unit/test_master.py b/tests/unit/test_master.py new file mode 100644 index 00000000000..c663d2c45ca --- /dev/null +++ b/tests/unit/test_master.py @@ -0,0 +1,209 @@ +# -*- coding: utf-8 -*- + +# Import Python libs +from __future__ import absolute_import + +# Import Salt libs +import salt.config +import salt.master + +# Import Salt Testing Libs +from tests.support.unit import TestCase +from tests.support.mock import ( + patch, + MagicMock, +) + + +class ClearFuncsTestCase(TestCase): + ''' + TestCase for salt.master.ClearFuncs class + ''' + + def setUp(self): + opts = salt.config.master_config(None) + self.clear_funcs = salt.master.ClearFuncs(opts, {}) + + def test_runner_token_not_authenticated(self): + ''' + Asserts that a TokenAuthenticationError is returned when the token can't authenticate. + ''' + mock_ret = {u'error': {u'name': u'TokenAuthenticationError', + u'message': u'Authentication failure of type "token" occurred.'}} + ret = self.clear_funcs.runner({u'token': u'asdfasdfasdfasdf'}) + self.assertDictEqual(mock_ret, ret) + + def test_runner_token_authorization_error(self): + ''' + Asserts that a TokenAuthenticationError is returned when the token authenticates, but is + not authorized. + ''' + token = u'asdfasdfasdfasdf' + clear_load = {u'token': token, u'fun': u'test.arg'} + mock_token = {u'token': token, u'eauth': u'foo', u'name': u'test'} + mock_ret = {u'error': {u'name': u'TokenAuthenticationError', + u'message': u'Authentication failure of type "token" occurred ' + u'for user test.'}} + + with patch('salt.auth.LoadAuth.authenticate_token', MagicMock(return_value=mock_token)), \ + patch('salt.auth.LoadAuth.get_auth_list', MagicMock(return_value=[])): + ret = self.clear_funcs.runner(clear_load) + + self.assertDictEqual(mock_ret, ret) + + def test_runner_token_salt_invocation_error(self): + ''' + Asserts that a SaltInvocationError is returned when the token authenticates, but the + command is malformed. + ''' + token = u'asdfasdfasdfasdf' + clear_load = {u'token': token, u'fun': u'badtestarg'} + mock_token = {u'token': token, u'eauth': u'foo', u'name': u'test'} + mock_ret = {u'error': {u'name': u'SaltInvocationError', + u'message': u'A command invocation error occurred: Check syntax.'}} + + with patch('salt.auth.LoadAuth.authenticate_token', MagicMock(return_value=mock_token)), \ + patch('salt.auth.LoadAuth.get_auth_list', MagicMock(return_value=[])): + ret = self.clear_funcs.runner(clear_load) + + self.assertDictEqual(mock_ret, ret) + + def test_runner_eauth_not_authenticated(self): + ''' + Asserts that an EauthAuthenticationError is returned when the user can't authenticate. + ''' + mock_ret = {u'error': {u'name': u'EauthAuthenticationError', + u'message': u'Authentication failure of type "eauth" occurred for ' + u'user UNKNOWN.'}} + ret = self.clear_funcs.runner({u'eauth': u'foo'}) + self.assertDictEqual(mock_ret, ret) + + def test_runner_eauth_authorization_error(self): + ''' + Asserts that an EauthAuthenticationError is returned when the user authenticates, but is + not authorized. + ''' + clear_load = {u'eauth': u'foo', u'username': u'test', u'fun': u'test.arg'} + mock_ret = {u'error': {u'name': u'EauthAuthenticationError', + u'message': u'Authentication failure of type "eauth" occurred for ' + u'user test.'}} + with patch('salt.auth.LoadAuth.authenticate_eauth', MagicMock(return_value=True)), \ + patch('salt.auth.LoadAuth.get_auth_list', MagicMock(return_value=[])): + ret = self.clear_funcs.runner(clear_load) + + self.assertDictEqual(mock_ret, ret) + + def test_runner_eauth_salt_invocation_errpr(self): + ''' + Asserts that an EauthAuthenticationError is returned when the user authenticates, but the + command is malformed. + ''' + clear_load = {u'eauth': u'foo', u'username': u'test', u'fun': u'bad.test.arg.func'} + mock_ret = {u'error': {u'name': u'SaltInvocationError', + u'message': u'A command invocation error occurred: Check syntax.'}} + with patch('salt.auth.LoadAuth.authenticate_eauth', MagicMock(return_value=True)), \ + patch('salt.auth.LoadAuth.get_auth_list', MagicMock(return_value=[])): + ret = self.clear_funcs.runner(clear_load) + + self.assertDictEqual(mock_ret, ret) + + def test_runner_user_not_authenticated(self): + ''' + Asserts that an UserAuthenticationError is returned when the user can't authenticate. + ''' + mock_ret = {u'error': {u'name': u'UserAuthenticationError', + u'message': u'Authentication failure of type "user" occurred'}} + ret = self.clear_funcs.runner({}) + self.assertDictEqual(mock_ret, ret) + + def test_wheel_token_not_authenticated(self): + ''' + Asserts that a TokenAuthenticationError is returned when the token can't authenticate. + ''' + mock_ret = {u'error': {u'name': u'TokenAuthenticationError', + u'message': u'Authentication failure of type "token" occurred.'}} + ret = self.clear_funcs.wheel({u'token': u'asdfasdfasdfasdf'}) + self.assertDictEqual(mock_ret, ret) + + def test_wheel_token_authorization_error(self): + ''' + Asserts that a TokenAuthenticationError is returned when the token authenticates, but is + not authorized. + ''' + token = u'asdfasdfasdfasdf' + clear_load = {u'token': token, u'fun': u'test.arg'} + mock_token = {u'token': token, u'eauth': u'foo', u'name': u'test'} + mock_ret = {u'error': {u'name': u'TokenAuthenticationError', + u'message': u'Authentication failure of type "token" occurred ' + u'for user test.'}} + + with patch('salt.auth.LoadAuth.authenticate_token', MagicMock(return_value=mock_token)), \ + patch('salt.auth.LoadAuth.get_auth_list', MagicMock(return_value=[])): + ret = self.clear_funcs.wheel(clear_load) + + self.assertDictEqual(mock_ret, ret) + + def test_wheel_token_salt_invocation_error(self): + ''' + Asserts that a SaltInvocationError is returned when the token authenticates, but the + command is malformed. + ''' + token = u'asdfasdfasdfasdf' + clear_load = {u'token': token, u'fun': u'badtestarg'} + mock_token = {u'token': token, u'eauth': u'foo', u'name': u'test'} + mock_ret = {u'error': {u'name': u'SaltInvocationError', + u'message': u'A command invocation error occurred: Check syntax.'}} + + with patch('salt.auth.LoadAuth.authenticate_token', MagicMock(return_value=mock_token)), \ + patch('salt.auth.LoadAuth.get_auth_list', MagicMock(return_value=[])): + ret = self.clear_funcs.wheel(clear_load) + + self.assertDictEqual(mock_ret, ret) + + def test_wheel_eauth_not_authenticated(self): + ''' + Asserts that an EauthAuthenticationError is returned when the user can't authenticate. + ''' + mock_ret = {u'error': {u'name': u'EauthAuthenticationError', + u'message': u'Authentication failure of type "eauth" occurred for ' + u'user UNKNOWN.'}} + ret = self.clear_funcs.wheel({u'eauth': u'foo'}) + self.assertDictEqual(mock_ret, ret) + + def test_wheel_eauth_authorization_error(self): + ''' + Asserts that an EauthAuthenticationError is returned when the user authenticates, but is + not authorized. + ''' + clear_load = {u'eauth': u'foo', u'username': u'test', u'fun': u'test.arg'} + mock_ret = {u'error': {u'name': u'EauthAuthenticationError', + u'message': u'Authentication failure of type "eauth" occurred for ' + u'user test.'}} + with patch('salt.auth.LoadAuth.authenticate_eauth', MagicMock(return_value=True)), \ + patch('salt.auth.LoadAuth.get_auth_list', MagicMock(return_value=[])): + ret = self.clear_funcs.wheel(clear_load) + + self.assertDictEqual(mock_ret, ret) + + def test_wheel_eauth_salt_invocation_errpr(self): + ''' + Asserts that an EauthAuthenticationError is returned when the user authenticates, but the + command is malformed. + ''' + clear_load = {u'eauth': u'foo', u'username': u'test', u'fun': u'bad.test.arg.func'} + mock_ret = {u'error': {u'name': u'SaltInvocationError', + u'message': u'A command invocation error occurred: Check syntax.'}} + with patch('salt.auth.LoadAuth.authenticate_eauth', MagicMock(return_value=True)), \ + patch('salt.auth.LoadAuth.get_auth_list', MagicMock(return_value=[])): + ret = self.clear_funcs.wheel(clear_load) + + self.assertDictEqual(mock_ret, ret) + + def test_wheel_user_not_authenticated(self): + ''' + Asserts that an UserAuthenticationError is returned when the user can't authenticate. + ''' + mock_ret = {u'error': {u'name': u'UserAuthenticationError', + u'message': u'Authentication failure of type "user" occurred'}} + ret = self.clear_funcs.wheel({}) + self.assertDictEqual(mock_ret, ret) diff --git a/tests/unit/test_minion.py b/tests/unit/test_minion.py index 535dfeedfcf..e60e08edf30 100644 --- a/tests/unit/test_minion.py +++ b/tests/unit/test_minion.py @@ -13,8 +13,8 @@ from tests.support.unit import TestCase, skipIf from tests.support.mock import NO_MOCK, NO_MOCK_REASON, patch, MagicMock from tests.support.helpers import skip_if_not_root # Import salt libs -from salt import minion -from salt.utils import event +import salt.minion as minion +import salt.utils.event as event from salt.exceptions import SaltSystemExit import salt.syspaths import tornado diff --git a/tests/unit/test_payload.py b/tests/unit/test_payload.py index d41b62695f1..21f79539234 100644 --- a/tests/unit/test_payload.py +++ b/tests/unit/test_payload.py @@ -26,7 +26,7 @@ import salt.exceptions # Import 3rd-party libs import msgpack import zmq -import salt.ext.six as six +from salt.ext import six import logging diff --git a/tests/unit/test_pillar.py b/tests/unit/test_pillar.py index 4cfaa6ef3b5..44791642326 100644 --- a/tests/unit/test_pillar.py +++ b/tests/unit/test_pillar.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- ''' :codeauthor: :email:`Pedro Algarvio (pedro@algarvio.me)` + :codeauthor: :email:`Alexandru Bleotu (alexandru.bleotu@morganstanley.com)` tests.unit.pillar_test @@ -18,6 +19,8 @@ from tests.support.paths import TMP # Import salt libs import salt.pillar +import salt.utils.stringutils +import salt.exceptions @skipIf(NO_MOCK, NO_MOCK_REASON) @@ -55,6 +58,244 @@ class PillarTestCase(TestCase): self.assertEqual(pillar.opts['environment'], 'dev') self.assertEqual(pillar.opts['pillarenv'], 'dev') + def test_ext_pillar_no_extra_minion_data_val_dict(self): + opts = { + 'renderer': 'json', + 'renderer_blacklist': [], + 'renderer_whitelist': [], + 'state_top': '', + 'pillar_roots': { + 'dev': [], + 'base': [] + }, + 'file_roots': { + 'dev': [], + 'base': [] + }, + 'extension_modules': '', + 'pillarenv_from_saltenv': True + } + mock_ext_pillar_func = MagicMock() + with patch('salt.loader.pillars', + MagicMock(return_value={'fake_ext_pillar': + mock_ext_pillar_func})): + pillar = salt.pillar.Pillar(opts, {}, 'mocked-minion', 'dev') + # ext pillar function doesn't have the extra_minion_data arg + with patch('salt.utils.args.get_function_argspec', + MagicMock(return_value=MagicMock(args=[]))): + pillar._external_pillar_data('fake_pillar', {'arg': 'foo'}, + 'fake_ext_pillar') + mock_ext_pillar_func.assert_called_once_with('mocked-minion', + 'fake_pillar', + arg='foo') + # ext pillar function has the extra_minion_data arg + mock_ext_pillar_func.reset_mock() + with patch('salt.utils.args.get_function_argspec', + MagicMock(return_value=MagicMock(args=['extra_minion_data']))): + pillar._external_pillar_data('fake_pillar', {'arg': 'foo'}, + 'fake_ext_pillar') + mock_ext_pillar_func.assert_called_once_with('mocked-minion', + 'fake_pillar', + arg='foo') + + def test_ext_pillar_no_extra_minion_data_val_list(self): + opts = { + 'renderer': 'json', + 'renderer_blacklist': [], + 'renderer_whitelist': [], + 'state_top': '', + 'pillar_roots': { + 'dev': [], + 'base': [] + }, + 'file_roots': { + 'dev': [], + 'base': [] + }, + 'extension_modules': '', + 'pillarenv_from_saltenv': True + } + mock_ext_pillar_func = MagicMock() + with patch('salt.loader.pillars', + MagicMock(return_value={'fake_ext_pillar': + mock_ext_pillar_func})): + pillar = salt.pillar.Pillar(opts, {}, 'mocked-minion', 'dev') + # ext pillar function doesn't have the extra_minion_data arg + with patch('salt.utils.args.get_function_argspec', + MagicMock(return_value=MagicMock(args=[]))): + pillar._external_pillar_data('fake_pillar', ['foo'], + 'fake_ext_pillar') + mock_ext_pillar_func.assert_called_once_with('mocked-minion', + 'fake_pillar', + 'foo') + # ext pillar function has the extra_minion_data arg + mock_ext_pillar_func.reset_mock() + with patch('salt.utils.args.get_function_argspec', + MagicMock(return_value=MagicMock(args=['extra_minion_data']))): + pillar._external_pillar_data('fake_pillar', ['foo'], + 'fake_ext_pillar') + mock_ext_pillar_func.assert_called_once_with('mocked-minion', + 'fake_pillar', + 'foo') + + def test_ext_pillar_no_extra_minion_data_val_elem(self): + opts = { + 'renderer': 'json', + 'renderer_blacklist': [], + 'renderer_whitelist': [], + 'state_top': '', + 'pillar_roots': { + 'dev': [], + 'base': [] + }, + 'file_roots': { + 'dev': [], + 'base': [] + }, + 'extension_modules': '', + 'pillarenv_from_saltenv': True + } + mock_ext_pillar_func = MagicMock() + with patch('salt.loader.pillars', + MagicMock(return_value={'fake_ext_pillar': + mock_ext_pillar_func})): + pillar = salt.pillar.Pillar(opts, {}, 'mocked-minion', 'dev') + # ext pillar function doesn't have the extra_minion_data arg + with patch('salt.utils.args.get_function_argspec', + MagicMock(return_value=MagicMock(args=[]))): + pillar._external_pillar_data('fake_pillar', 'fake_val', + 'fake_ext_pillar') + mock_ext_pillar_func.assert_called_once_with('mocked-minion', + 'fake_pillar', 'fake_val') + # ext pillar function has the extra_minion_data arg + mock_ext_pillar_func.reset_mock() + with patch('salt.utils.args.get_function_argspec', + MagicMock(return_value=MagicMock(args=['extra_minion_data']))): + pillar._external_pillar_data('fake_pillar', 'fake_val', + 'fake_ext_pillar') + mock_ext_pillar_func.assert_called_once_with('mocked-minion', + 'fake_pillar', 'fake_val') + + def test_ext_pillar_with_extra_minion_data_val_dict(self): + opts = { + 'renderer': 'json', + 'renderer_blacklist': [], + 'renderer_whitelist': [], + 'state_top': '', + 'pillar_roots': { + 'dev': [], + 'base': [] + }, + 'file_roots': { + 'dev': [], + 'base': [] + }, + 'extension_modules': '', + 'pillarenv_from_saltenv': True + } + mock_ext_pillar_func = MagicMock() + with patch('salt.loader.pillars', + MagicMock(return_value={'fake_ext_pillar': + mock_ext_pillar_func})): + pillar = salt.pillar.Pillar(opts, {}, 'mocked-minion', 'dev', + extra_minion_data={'fake_key': 'foo'}) + # ext pillar function doesn't have the extra_minion_data arg + with patch('salt.utils.args.get_function_argspec', + MagicMock(return_value=MagicMock(args=[]))): + pillar._external_pillar_data('fake_pillar', {'arg': 'foo'}, + 'fake_ext_pillar') + mock_ext_pillar_func.assert_called_once_with( + 'mocked-minion', 'fake_pillar', arg='foo') + # ext pillar function has the extra_minion_data arg + mock_ext_pillar_func.reset_mock() + with patch('salt.utils.args.get_function_argspec', + MagicMock(return_value=MagicMock(args=['extra_minion_data']))): + pillar._external_pillar_data('fake_pillar', {'arg': 'foo'}, + 'fake_ext_pillar') + mock_ext_pillar_func.assert_called_once_with( + 'mocked-minion', 'fake_pillar', arg='foo', + extra_minion_data={'fake_key': 'foo'}) + + def test_ext_pillar_with_extra_minion_data_val_list(self): + opts = { + 'renderer': 'json', + 'renderer_blacklist': [], + 'renderer_whitelist': [], + 'state_top': '', + 'pillar_roots': { + 'dev': [], + 'base': [] + }, + 'file_roots': { + 'dev': [], + 'base': [] + }, + 'extension_modules': '', + 'pillarenv_from_saltenv': True + } + mock_ext_pillar_func = MagicMock() + with patch('salt.loader.pillars', + MagicMock(return_value={'fake_ext_pillar': + mock_ext_pillar_func})): + pillar = salt.pillar.Pillar(opts, {}, 'mocked-minion', 'dev', + extra_minion_data={'fake_key': 'foo'}) + # ext pillar function doesn't have the extra_minion_data arg + with patch('salt.utils.args.get_function_argspec', + MagicMock(return_value=MagicMock(args=[]))): + pillar._external_pillar_data('fake_pillar', ['bar'], + 'fake_ext_pillar') + mock_ext_pillar_func.assert_called_once_with( + 'mocked-minion', 'fake_pillar', 'bar') + # ext pillar function has the extra_minion_data arg + mock_ext_pillar_func.reset_mock() + with patch('salt.utils.args.get_function_argspec', + MagicMock(return_value=MagicMock(args=['extra_minion_data']))): + pillar._external_pillar_data('fake_pillar', ['bar'], + 'fake_ext_pillar') + mock_ext_pillar_func.assert_called_once_with( + 'mocked-minion', 'fake_pillar', 'bar', + extra_minion_data={'fake_key': 'foo'}) + + def test_ext_pillar_with_extra_minion_data_val_elem(self): + opts = { + 'renderer': 'json', + 'renderer_blacklist': [], + 'renderer_whitelist': [], + 'state_top': '', + 'pillar_roots': { + 'dev': [], + 'base': [] + }, + 'file_roots': { + 'dev': [], + 'base': [] + }, + 'extension_modules': '', + 'pillarenv_from_saltenv': True + } + mock_ext_pillar_func = MagicMock() + with patch('salt.loader.pillars', + MagicMock(return_value={'fake_ext_pillar': + mock_ext_pillar_func})): + pillar = salt.pillar.Pillar(opts, {}, 'mocked-minion', 'dev', + extra_minion_data={'fake_key': 'foo'}) + # ext pillar function doesn't have the extra_minion_data arg + with patch('salt.utils.args.get_function_argspec', + MagicMock(return_value=MagicMock(args=[]))): + pillar._external_pillar_data('fake_pillar', 'bar', + 'fake_ext_pillar') + mock_ext_pillar_func.assert_called_once_with( + 'mocked-minion', 'fake_pillar', 'bar') + # ext pillar function has the extra_minion_data arg + mock_ext_pillar_func.reset_mock() + with patch('salt.utils.args.get_function_argspec', + MagicMock(return_value=MagicMock(args=['extra_minion_data']))): + pillar._external_pillar_data('fake_pillar', 'bar', + 'fake_ext_pillar') + mock_ext_pillar_func.assert_called_once_with( + 'mocked-minion', 'fake_pillar', 'bar', + extra_minion_data={'fake_key': 'foo'}) + def test_malformed_pillar_sls(self): with patch('salt.pillar.compile_template') as compile_template: opts = { @@ -213,7 +454,7 @@ base: - ssh.minion - generic.minion '''.format(nodegroup_order=nodegroup_order, glob_order=glob_order) - self.top_file.write(salt.utils.to_bytes(s)) + self.top_file.write(salt.utils.stringutils.to_bytes(s)) self.top_file.flush() self.ssh_file = tempfile.NamedTemporaryFile(dir=TMP) self.ssh_file.write(b''' @@ -317,3 +558,174 @@ p2: }[sls] client.get_state.side_effect = get_state + + +@skipIf(NO_MOCK, NO_MOCK_REASON) +@patch('salt.transport.Channel.factory', MagicMock()) +class RemotePillarTestCase(TestCase): + ''' + Tests for instantiating a RemotePillar in salt.pillar + ''' + def setUp(self): + self.grains = {} + + def tearDown(self): + for attr in ('grains',): + try: + delattr(self, attr) + except AttributeError: + continue + + def test_get_opts_in_pillar_override_call(self): + mock_get_extra_minion_data = MagicMock(return_value={}) + with patch( + 'salt.pillar.RemotePillarMixin.get_ext_pillar_extra_minion_data', + mock_get_extra_minion_data): + + salt.pillar.RemotePillar({}, self.grains, 'mocked-minion', 'dev') + mock_get_extra_minion_data.assert_called_once_with( + {'environment': 'dev'}) + + def test_multiple_keys_in_opts_added_to_pillar(self): + opts = { + 'renderer': 'json', + 'path_to_add': 'fake_data', + 'path_to_add2': {'fake_data2': ['fake_data3', 'fake_data4']}, + 'pass_to_ext_pillars': ['path_to_add', 'path_to_add2'] + } + pillar = salt.pillar.RemotePillar(opts, self.grains, + 'mocked-minion', 'dev') + self.assertEqual(pillar.extra_minion_data, + {'path_to_add': 'fake_data', + 'path_to_add2': {'fake_data2': ['fake_data3', + 'fake_data4']}}) + + def test_subkey_in_opts_added_to_pillar(self): + opts = { + 'renderer': 'json', + 'path_to_add': 'fake_data', + 'path_to_add2': {'fake_data5': 'fake_data6', + 'fake_data2': ['fake_data3', 'fake_data4']}, + 'pass_to_ext_pillars': ['path_to_add2:fake_data5'] + } + pillar = salt.pillar.RemotePillar(opts, self.grains, + 'mocked-minion', 'dev') + self.assertEqual(pillar.extra_minion_data, + {'path_to_add2': {'fake_data5': 'fake_data6'}}) + + def test_non_existent_leaf_opt_in_add_to_pillar(self): + opts = { + 'renderer': 'json', + 'path_to_add': 'fake_data', + 'path_to_add2': {'fake_data5': 'fake_data6', + 'fake_data2': ['fake_data3', 'fake_data4']}, + 'pass_to_ext_pillars': ['path_to_add2:fake_data_non_exist'] + } + pillar = salt.pillar.RemotePillar(opts, self.grains, + 'mocked-minion', 'dev') + self.assertEqual(pillar.pillar_override, {}) + + def test_non_existent_intermediate_opt_in_add_to_pillar(self): + opts = { + 'renderer': 'json', + 'path_to_add': 'fake_data', + 'path_to_add2': {'fake_data5': 'fake_data6', + 'fake_data2': ['fake_data3', 'fake_data4']}, + 'pass_to_ext_pillars': ['path_to_add_no_exist'] + } + pillar = salt.pillar.RemotePillar(opts, self.grains, + 'mocked-minion', 'dev') + self.assertEqual(pillar.pillar_override, {}) + + def test_malformed_add_to_pillar(self): + opts = { + 'renderer': 'json', + 'path_to_add': 'fake_data', + 'path_to_add2': {'fake_data5': 'fake_data6', + 'fake_data2': ['fake_data3', 'fake_data4']}, + 'pass_to_ext_pillars': MagicMock() + } + with self.assertRaises(salt.exceptions.SaltClientError) as excinfo: + salt.pillar.RemotePillar(opts, self.grains, 'mocked-minion', 'dev') + self.assertEqual(excinfo.exception.strerror, + '\'pass_to_ext_pillars\' config is malformed.') + + def test_pillar_send_extra_minion_data_from_config(self): + opts = { + 'renderer': 'json', + 'pillarenv': 'fake_pillar_env', + 'path_to_add': 'fake_data', + 'path_to_add2': {'fake_data5': 'fake_data6', + 'fake_data2': ['fake_data3', 'fake_data4']}, + 'pass_to_ext_pillars': ['path_to_add']} + mock_channel = MagicMock( + crypted_transfer_decode_dictentry=MagicMock(return_value={})) + with patch('salt.transport.Channel.factory', + MagicMock(return_value=mock_channel)): + pillar = salt.pillar.RemotePillar(opts, self.grains, + 'mocked_minion', 'fake_env') + + ret = pillar.compile_pillar() + self.assertEqual(pillar.channel, mock_channel) + mock_channel.crypted_transfer_decode_dictentry.assert_called_once_with( + {'cmd': '_pillar', 'ver': '2', + 'id': 'mocked_minion', + 'grains': {}, + 'saltenv': 'fake_env', + 'pillarenv': 'fake_pillar_env', + 'pillar_override': {}, + 'extra_minion_data': {'path_to_add': 'fake_data'}}, + dictkey='pillar') + + +@skipIf(NO_MOCK, NO_MOCK_REASON) +@patch('salt.transport.client.AsyncReqChannel.factory', MagicMock()) +class AsyncRemotePillarTestCase(TestCase): + ''' + Tests for instantiating a AsyncRemotePillar in salt.pillar + ''' + def setUp(self): + self.grains = {} + + def tearDown(self): + for attr in ('grains',): + try: + delattr(self, attr) + except AttributeError: + continue + + def test_get_opts_in_pillar_override_call(self): + mock_get_extra_minion_data = MagicMock(return_value={}) + with patch( + 'salt.pillar.RemotePillarMixin.get_ext_pillar_extra_minion_data', + mock_get_extra_minion_data): + + salt.pillar.RemotePillar({}, self.grains, 'mocked-minion', 'dev') + mock_get_extra_minion_data.assert_called_once_with( + {'environment': 'dev'}) + + def test_pillar_send_extra_minion_data_from_config(self): + opts = { + 'renderer': 'json', + 'pillarenv': 'fake_pillar_env', + 'path_to_add': 'fake_data', + 'path_to_add2': {'fake_data5': 'fake_data6', + 'fake_data2': ['fake_data3', 'fake_data4']}, + 'pass_to_ext_pillars': ['path_to_add']} + mock_channel = MagicMock( + crypted_transfer_decode_dictentry=MagicMock(return_value={})) + with patch('salt.transport.client.AsyncReqChannel.factory', + MagicMock(return_value=mock_channel)): + pillar = salt.pillar.RemotePillar(opts, self.grains, + 'mocked_minion', 'fake_env') + + ret = pillar.compile_pillar() + mock_channel.crypted_transfer_decode_dictentry.assert_called_once_with( + {'cmd': '_pillar', 'ver': '2', + 'id': 'mocked_minion', + 'grains': {}, + 'saltenv': 'fake_env', + 'pillarenv': 'fake_pillar_env', + 'pillar_override': {}, + 'extra_minion_data': {'path_to_add': 'fake_data'}}, + dictkey='pillar') diff --git a/tests/unit/test_pydsl.py b/tests/unit/test_pydsl.py index 9926f9110c4..3500beb16b8 100644 --- a/tests/unit/test_pydsl.py +++ b/tests/unit/test_pydsl.py @@ -16,13 +16,14 @@ from tests.support.paths import TMP # Import Salt libs import salt.loader import salt.config -import salt.utils +import salt.utils.files +import salt.utils.versions from salt.state import HighState from salt.utils.pydsl import PyDslError # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import StringIO @@ -90,12 +91,7 @@ class PyDSLRendererTestCase(CommonTestCaseBoilerplate): def render_sls(self, content, sls='', saltenv='base', **kws): if 'env' in kws: - salt.utils.warn_until( - 'Oxygen', - 'Parameter \'env\' has been detected in the argument list. This ' - 'parameter is no longer used and has been replaced by \'saltenv\' ' - 'as of Salt 2016.11.0. This warning will be removed in Salt Oxygen.' - ) + # "env" is not supported; Use "saltenv". kws.pop('env') return self.HIGHSTATE.state.rend['pydsl']( @@ -267,16 +263,18 @@ class PyDSLRendererTestCase(CommonTestCaseBoilerplate): ''')) self.assertEqual(len(result), 3) self.assertEqual(result['A']['cmd'][0], 'run') - self.assertEqual(result['A']['cmd'][1]['name'], 'echo hello') - self.assertEqual(result['A']['cmd'][2]['cwd'], '/') - self.assertEqual(result['A']['cmd'][3]['name'], 'echo hello world') + self.assertIn({'name': 'echo hello'}, result['A']['cmd']) + self.assertIn({'cwd': '/'}, result['A']['cmd']) + self.assertIn({'name': 'echo hello world'}, result['A']['cmd']) + self.assertEqual(len(result['A']['cmd']), 4) self.assertEqual(len(result['B']['pkg']), 1) self.assertEqual(result['B']['pkg'][0], 'installed') self.assertEqual(result['B']['service'][0], 'running') - self.assertEqual(result['B']['service'][1]['require'][0]['pkg'], 'B') - self.assertEqual(result['B']['service'][2]['watch'][0]['cmd'], 'A') + self.assertIn({'require': [{'pkg': 'B'}]}, result['B']['service']) + self.assertIn({'watch': [{'cmd': 'A'}]}, result['B']['service']) + self.assertEqual(len(result['B']['service']), 3) def test_ordered_states(self): result = self.render_sls(textwrap.dedent(''' @@ -346,7 +344,7 @@ class PyDSLRendererTestCase(CommonTestCaseBoilerplate): '''.format(output, output, output))) self.state_highstate({'base': ['aaa']}, dirpath) - with salt.utils.fopen(output, 'r') as f: + with salt.utils.files.fopen(output, 'r') as f: self.assertEqual(''.join(f.read().split()), "XYZABCDEF") finally: @@ -379,9 +377,9 @@ class PyDSLRendererTestCase(CommonTestCaseBoilerplate): A() '''.format(dirpath, dirpath, dirpath, dirpath))) self.state_highstate({'base': ['aaa']}, dirpath) - with salt.utils.fopen(os.path.join(dirpath, 'yyy.txt'), 'rt') as f: + with salt.utils.files.fopen(os.path.join(dirpath, 'yyy.txt'), 'rt') as f: self.assertEqual(f.read(), 'hehe\nhoho\n') - with salt.utils.fopen(os.path.join(dirpath, 'xxx.txt'), 'rt') as f: + with salt.utils.files.fopen(os.path.join(dirpath, 'xxx.txt'), 'rt') as f: self.assertEqual(f.read(), 'hehe\n') finally: shutil.rmtree(dirpath, ignore_errors=True) @@ -454,5 +452,5 @@ class PyDSLRendererTestCase(CommonTestCaseBoilerplate): def write_to(fpath, content): - with salt.utils.fopen(fpath, 'w') as f: + with salt.utils.files.fopen(fpath, 'w') as f: f.write(content) diff --git a/tests/unit/test_pyobjects.py b/tests/unit/test_pyobjects.py index 464b458a465..8908d63f061 100644 --- a/tests/unit/test_pyobjects.py +++ b/tests/unit/test_pyobjects.py @@ -2,9 +2,12 @@ # Import Pytohn libs from __future__ import absolute_import +import jinja2 +import logging import os import shutil import tempfile +import textwrap import uuid # Import Salt Testing libs @@ -14,11 +17,14 @@ from tests.support.unit import TestCase import tests.integration as integration import salt.config import salt.state -import salt.utils +import salt.utils.files from salt.template import compile_template from salt.utils.odict import OrderedDict from salt.utils.pyobjects import (StateFactory, State, Registry, SaltObject, InvalidFunction, DuplicateState) + +log = logging.getLogger(__name__) + File = StateFactory('file') Service = StateFactory('service') @@ -58,33 +64,37 @@ Service = StateFactory('service') Service.running(extend('apache'), watch=[{'file': '/etc/file'}]) ''' -map_template = '''#!pyobjects +map_prefix = '''\ +#!pyobjects from salt.utils.pyobjects import StateFactory Service = StateFactory('service') - +{% macro priority(value) %} + priority = {{ value }} +{% endmacro %} class Samba(Map): - __merge__ = 'samba:lookup' - - class Debian: - server = 'samba' - client = 'samba-client' - service = 'samba' - - class RougeChapeau: - __match__ = 'RedHat' - server = 'samba' - client = 'samba' - service = 'smb' - - class Ubuntu: - __grain__ = 'os' - service = 'smbd' +''' +map_suffix = ''' with Pkg.installed("samba", names=[Samba.server, Samba.client]): Service.running("samba", name=Samba.service) ''' +map_data = { + 'debian': " class Debian:\n" + " server = 'samba'\n" + " client = 'samba-client'\n" + " service = 'samba'\n", + 'centos': " class RougeChapeau:\n" + " __match__ = 'RedHat'\n" + " server = 'samba'\n" + " client = 'samba'\n" + " service = 'smb'\n", + 'ubuntu': " class Ubuntu:\n" + " __grain__ = 'os'\n" + " service = 'smbd'\n" +} + import_template = '''#!pyobjects import salt://map.sls @@ -140,6 +150,24 @@ with Pkg.installed("pkg"): ''' +class MapBuilder(object): + def build_map(self, template=None): + ''' + Build from a specific template or just use a default if no template + is passed to this function. + ''' + if template is None: + template = textwrap.dedent('''\ + {{ ubuntu }} + {{ centos }} + {{ debian }} + ''') + full_template = map_prefix + template + map_suffix + ret = jinja2.Template(full_template).render(**map_data) + log.debug('built map: \n%s', ret) + return ret + + class StateTests(TestCase): def setUp(self): Registry.empty() @@ -269,7 +297,7 @@ class RendererMixin(object): def write_template_file(self, filename, content): full_path = os.path.join(self.state_tree_dir, filename) - with salt.utils.fopen(full_path, 'w') as f: + with salt.utils.files.fopen(full_path, 'w') as f: f.write(content) return full_path @@ -292,7 +320,7 @@ class RendererMixin(object): state.opts['renderer_whitelist']) -class RendererTests(RendererMixin, StateTests): +class RendererTests(RendererMixin, StateTests, MapBuilder): def test_basic(self): ret = self.render(basic_template) self.assertEqual(ret, OrderedDict([ @@ -350,7 +378,7 @@ class RendererTests(RendererMixin, StateTests): }) ])) - self.write_template_file("map.sls", map_template) + self.write_template_file("map.sls", self.build_map()) render_and_assert(import_template) render_and_assert(from_import_template) render_and_assert(import_as_template) @@ -359,7 +387,7 @@ class RendererTests(RendererMixin, StateTests): render_and_assert(recursive_import_template) def test_import_scope(self): - self.write_template_file("map.sls", map_template) + self.write_template_file("map.sls", self.build_map()) self.write_template_file("recursive_map.sls", recursive_map_template) def do_render(): @@ -401,32 +429,97 @@ class RendererTests(RendererMixin, StateTests): ])) -class MapTests(RendererMixin, TestCase): - def test_map(self): - def samba_with_grains(grains): - return self.render(map_template, {'grains': grains}) +class MapTests(RendererMixin, TestCase, MapBuilder): + maxDiff = None - def assert_ret(ret, server, client, service): - self.assertEqual(ret, OrderedDict([ - ('samba', { - 'pkg.installed': [ - {'names': [server, client]} - ], - 'service.running': [ - {'name': service}, - {'require': [{'pkg': 'samba'}]} - ] - }) + debian_grains = {'os_family': 'Debian', 'os': 'Debian'} + ubuntu_grains = {'os_family': 'Debian', 'os': 'Ubuntu'} + centos_grains = {'os_family': 'RedHat', 'os': 'CentOS'} + + debian_attrs = ('samba', 'samba-client', 'samba') + ubuntu_attrs = ('samba', 'samba-client', 'smbd') + centos_attrs = ('samba', 'samba', 'smb') + + def samba_with_grains(self, template, grains): + return self.render(template, {'grains': grains}) + + def assert_equal(self, ret, server, client, service): + self.assertDictEqual(ret, OrderedDict([ + ('samba', OrderedDict([ + ('pkg.installed', [ + {'names': [server, client]} + ]), + ('service.running', [ + {'name': service}, + {'require': [{'pkg': 'samba'}]} + ]) ])) + ])) - ret = samba_with_grains({'os_family': 'Debian', 'os': 'Debian'}) - assert_ret(ret, 'samba', 'samba-client', 'samba') + def assert_not_equal(self, ret, server, client, service): + try: + self.assert_equal(ret, server, client, service) + except AssertionError: + pass + else: + raise AssertionError('both dicts are equal') - ret = samba_with_grains({'os_family': 'Debian', 'os': 'Ubuntu'}) - assert_ret(ret, 'samba', 'samba-client', 'smbd') + def test_map(self): + ''' + Test declarative ordering + ''' + # With declarative ordering, the ubuntu-specfic service name should + # override the one inherited from debian. + template = self.build_map(textwrap.dedent('''\ + {{ debian }} + {{ centos }} + {{ ubuntu }} + ''')) - ret = samba_with_grains({'os_family': 'RedHat', 'os': 'CentOS'}) - assert_ret(ret, 'samba', 'samba', 'smb') + ret = self.samba_with_grains(template, self.debian_grains) + self.assert_equal(ret, *self.debian_attrs) + + ret = self.samba_with_grains(template, self.ubuntu_grains) + self.assert_equal(ret, *self.ubuntu_attrs) + + ret = self.samba_with_grains(template, self.centos_grains) + self.assert_equal(ret, *self.centos_attrs) + + # Switching the order, debian should still work fine but ubuntu should + # no longer match, since the debian service name should override the + # ubuntu one. + template = self.build_map(textwrap.dedent('''\ + {{ ubuntu }} + {{ debian }} + ''')) + + ret = self.samba_with_grains(template, self.debian_grains) + self.assert_equal(ret, *self.debian_attrs) + + ret = self.samba_with_grains(template, self.ubuntu_grains) + self.assert_not_equal(ret, *self.ubuntu_attrs) + + def test_map_with_priority(self): + ''' + With declarative ordering, the debian service name would override the + ubuntu one since debian comes second. This will test overriding this + behavior using the priority attribute. + ''' + template = self.build_map(textwrap.dedent('''\ + {{ priority(('os_family', 'os')) }} + {{ ubuntu }} + {{ centos }} + {{ debian }} + ''')) + + ret = self.samba_with_grains(template, self.debian_grains) + self.assert_equal(ret, *self.debian_attrs) + + ret = self.samba_with_grains(template, self.ubuntu_grains) + self.assert_equal(ret, *self.ubuntu_attrs) + + ret = self.samba_with_grains(template, self.centos_grains) + self.assert_equal(ret, *self.centos_attrs) class SaltObjectTests(TestCase): diff --git a/tests/unit/test_spm.py b/tests/unit/test_spm.py index 42a69c184ea..6f1eb7b78dd 100644 --- a/tests/unit/test_spm.py +++ b/tests/unit/test_spm.py @@ -12,7 +12,7 @@ from tests.support.mock import patch, MagicMock from tests.support.helpers import destructiveTest from tests.support.mixins import AdaptedConfigurationTestCaseMixin import salt.config -import salt.utils +import salt.utils.files import salt.spm _F1 = { @@ -100,7 +100,7 @@ class SPMTest(TestCase, AdaptedConfigurationTestCaseMixin): dirname, _ = os.path.split(path) if not os.path.exists(dirname): os.makedirs(dirname) - with salt.utils.fopen(path, 'w') as f: + with salt.utils.files.fopen(path, 'w') as f: f.write(contents) return fdir @@ -120,7 +120,7 @@ class SPMTest(TestCase, AdaptedConfigurationTestCaseMixin): for path, contents in _F1['contents']: path = os.path.join(self.minion_config['file_roots']['base'][0], _F1['definition']['name'], path) assert os.path.exists(path) - with salt.utils.fopen(path, 'r') as rfh: + with salt.utils.files.fopen(path, 'r') as rfh: assert rfh.read() == contents # Check database with patch('salt.client.Caller', MagicMock(return_value=self.minion_opts)): diff --git a/tests/unit/test_ssh_config_roster.py b/tests/unit/test_ssh_config_roster.py new file mode 100644 index 00000000000..f3479caf89e --- /dev/null +++ b/tests/unit/test_ssh_config_roster.py @@ -0,0 +1,87 @@ +# -*- coding: utf-8 -*- + +# Import Python libs +from __future__ import absolute_import +import collections + +# Import Salt Testing Libs +from tests.support import mock +from tests.support import mixins +from tests.support.unit import skipIf, TestCase + +# Import Salt Libs +import salt.roster.sshconfig as sshconfig + +_SAMPLE_SSH_CONFIG = """ +Host * + User user.mcuserface + +Host abc* + IdentityFile ~/.ssh/id_rsa_abc + +Host def* + IdentityFile ~/.ssh/id_rsa_def + +Host abc.asdfgfdhgjkl.com + HostName 123.123.123.123 + +Host abc123.asdfgfdhgjkl.com + HostName 123.123.123.124 + +Host def.asdfgfdhgjkl.com + HostName 234.234.234.234 +""" + +_TARGET_ABC = collections.OrderedDict([ + ('user', 'user.mcuserface'), + ('priv', '~/.ssh/id_rsa_abc'), + ('host', 'abc.asdfgfdhgjkl.com') +]) + +_TARGET_ABC123 = collections.OrderedDict([ + ('user', 'user.mcuserface'), + ('priv', '~/.ssh/id_rsa_abc'), + ('host', 'abc123.asdfgfdhgjkl.com') +]) + +_TARGET_DEF = collections.OrderedDict([ + ('user', 'user.mcuserface'), + ('priv', '~/.ssh/id_rsa_def'), + ('host', 'def.asdfgfdhgjkl.com') +]) + +_ALL = { + 'abc.asdfgfdhgjkl.com': _TARGET_ABC, + 'abc123.asdfgfdhgjkl.com': _TARGET_ABC123, + 'def.asdfgfdhgjkl.com': _TARGET_DEF +} + +_ABC_GLOB = { + 'abc.asdfgfdhgjkl.com': _TARGET_ABC, + 'abc123.asdfgfdhgjkl.com': _TARGET_ABC123 +} + + +@skipIf(mock.NO_MOCK, mock.NO_MOCK_REASON) +class SSHConfigRosterTestCase(TestCase, mixins.LoaderModuleMockMixin): + + @classmethod + def setUpClass(cls): + cls.mock_fp = mock_fp = mock.mock_open(read_data=_SAMPLE_SSH_CONFIG) + + def setup_loader_modules(self): + return {sshconfig: {}} + + def test_all(self): + with mock.patch('salt.utils.fopen', self.mock_fp): + with mock.patch('salt.roster.sshconfig._get_ssh_config_file'): + self.mock_fp.return_value.__iter__.return_value = _SAMPLE_SSH_CONFIG.splitlines() + targets = sshconfig.targets('*') + self.assertEqual(targets, _ALL) + + def test_abc_glob(self): + with mock.patch('salt.utils.fopen', self.mock_fp): + with mock.patch('salt.roster.sshconfig._get_ssh_config_file'): + self.mock_fp.return_value.__iter__.return_value = _SAMPLE_SSH_CONFIG.splitlines() + targets = sshconfig.targets('abc*') + self.assertEqual(targets, _ABC_GLOB) diff --git a/tests/unit/test_state.py b/tests/unit/test_state.py index 3d36e70d62b..df2cd702599 100644 --- a/tests/unit/test_state.py +++ b/tests/unit/test_state.py @@ -16,8 +16,9 @@ from tests.support.mock import NO_MOCK, NO_MOCK_REASON, patch from tests.support.mixins import AdaptedConfigurationTestCaseMixin # Import Salt libs -import salt.state import salt.exceptions +from salt.ext import six +import salt.state from salt.utils.odict import OrderedDict, DefaultOrderedDict @@ -472,3 +473,28 @@ class TopFileMergeTestCase(TestCase, AdaptedConfigurationTestCaseMixin): expected_merge = DefaultOrderedDict(OrderedDict) self.assertEqual(merged_tops, expected_merge) + + +class StateReturnsTestCase(TestCase): + ''' + TestCase for code handling state returns. + ''' + + def test_comment_lists_are_converted_to_string(self): + ''' + Tests that states returning a list of comments + have that converted to a single string + ''' + ret = { + 'name': 'myresource', + 'result': True, + 'comment': ['comment 1', 'comment 2'], + 'changes': {}, + } + salt.state.State.verify_ret(ret) # sanity check + with self.assertRaises(salt.exceptions.SaltException): + # Not suitable for export as is + salt.state.State.verify_ret_for_export(ret) + salt.state.State.munge_ret_for_export(ret) + self.assertIsInstance(ret[u'comment'], six.string_types) + salt.state.State.verify_ret_for_export(ret) diff --git a/tests/unit/test_stateconf.py b/tests/unit/test_stateconf.py index 5396d0dac00..38798710d0e 100644 --- a/tests/unit/test_stateconf.py +++ b/tests/unit/test_stateconf.py @@ -17,7 +17,7 @@ from salt.exceptions import SaltRenderError from salt.ext.six.moves import StringIO # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six REQUISITES = ['require', 'require_in', 'use', 'use_in', 'watch', 'watch_in'] @@ -102,8 +102,9 @@ test: - name: echo sls_dir={{sls_dir}} - cwd: / ''', sls='path.to.sls') - self.assertEqual(result['test']['cmd.run'][0]['name'], - 'echo sls_dir=path/to') + self.assertEqual( + result['test']['cmd.run'][0]['name'], + 'echo sls_dir=path{0}to'.format(os.sep)) def test_states_declared_with_shorthand_no_args(self): result = self._render_sls(''' diff --git a/tests/unit/test_test_module_names.py b/tests/unit/test_test_module_names.py index 62f824b636e..485f7376859 100644 --- a/tests/unit/test_test_module_names.py +++ b/tests/unit/test_test_module_names.py @@ -13,32 +13,33 @@ from tests.support.unit import TestCase from tests.support.paths import CODE_DIR EXCLUDED_DIRS = [ - 'tests/pkg', - 'tests/perf', - 'tests/support', - 'tests/unit/utils/cache_mods', - 'tests/unit/modules/inspectlib', - 'tests/unit/modules/zypp/', - 'tests/unit/templates/files', - 'tests/integration/files/', - 'tests/integration/cloud/helpers', + os.path.join('tests', 'pkg'), + os.path.join('tests', 'perf'), + os.path.join('tests', 'support'), + os.path.join('tests', 'unit', 'utils', 'cache_mods'), + os.path.join('tests', 'unit', 'modules', 'inspectlib'), + os.path.join('tests', 'unit', 'modules', 'zypp'), + os.path.join('tests', 'unit', 'templates', 'files'), + os.path.join('tests', 'integration', 'files'), + os.path.join('tests', 'integration', 'cloud', 'helpers'), ] EXCLUDED_FILES = [ - 'tests/eventlisten.py', - 'tests/buildpackage.py', - 'tests/saltsh.py', - 'tests/minionswarm.py', - 'tests/wheeltest.py', - 'tests/runtests.py', - 'tests/jenkins.py', - 'tests/salt-tcpdump.py', - 'tests/conftest.py', - 'tests/packdump.py', - 'tests/consist.py', - 'tests/modparser.py', - 'tests/committer_parser.py', - 'tests/unit/transport/mixins.py', - 'tests/integration/utils/testprogram.py', + os.path.join('tests', 'eventlisten.py'), + os.path.join('tests', 'buildpackage.py'), + os.path.join('tests', 'saltsh.py'), + os.path.join('tests', 'minionswarm.py'), + os.path.join('tests', 'wheeltest.py'), + os.path.join('tests', 'runtests.py'), + os.path.join('tests', 'jenkins.py'), + os.path.join('tests', 'salt-tcpdump.py'), + os.path.join('tests', 'conftest.py'), + os.path.join('tests', 'packdump.py'), + os.path.join('tests', 'consist.py'), + os.path.join('tests', 'modparser.py'), + os.path.join('tests', 'committer_parser.py'), + os.path.join('tests', 'zypp_plugin.py'), + os.path.join('tests', 'unit', 'transport', 'mixins.py'), + os.path.join('tests', 'integration', 'utils', 'testprogram.py'), ] diff --git a/tests/unit/test_transport.py b/tests/unit/test_transport.py new file mode 100644 index 00000000000..f24acb019eb --- /dev/null +++ b/tests/unit/test_transport.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- + +# Import python libs +from __future__ import absolute_import +import logging + +from salt.transport import MessageClientPool + +# Import Salt Testing libs +from tests.support.unit import TestCase + +log = logging.getLogger(__name__) + + +class MessageClientPoolTest(TestCase): + + class MockClass(object): + def __init__(self, *args, **kwargs): + self.args = args + self.kwargs = kwargs + + def test_init(self): + opts = {'sock_pool_size': 10} + args = (0,) + kwargs = {'kwarg': 1} + message_client_pool = MessageClientPool(self.MockClass, opts, args=args, kwargs=kwargs) + self.assertEqual(opts['sock_pool_size'], len(message_client_pool.message_clients)) + for message_client in message_client_pool.message_clients: + self.assertEqual(message_client.args, args) + self.assertEqual(message_client.kwargs, kwargs) + + def test_init_without_config(self): + opts = {} + args = (0,) + kwargs = {'kwarg': 1} + message_client_pool = MessageClientPool(self.MockClass, opts, args=args, kwargs=kwargs) + # The size of pool is set as 1 by the MessageClientPool init method. + self.assertEqual(1, len(message_client_pool.message_clients)) + for message_client in message_client_pool.message_clients: + self.assertEqual(message_client.args, args) + self.assertEqual(message_client.kwargs, kwargs) + + def test_init_less_than_one(self): + opts = {'sock_pool_size': -1} + args = (0,) + kwargs = {'kwarg': 1} + message_client_pool = MessageClientPool(self.MockClass, opts, args=args, kwargs=kwargs) + # The size of pool is set as 1 by the MessageClientPool init method. + self.assertEqual(1, len(message_client_pool.message_clients)) + for message_client in message_client_pool.message_clients: + self.assertEqual(message_client.args, args) + self.assertEqual(message_client.kwargs, kwargs) diff --git a/tests/unit/test_zypp_plugins.py b/tests/unit/test_zypp_plugins.py new file mode 100644 index 00000000000..c3afe1c3069 --- /dev/null +++ b/tests/unit/test_zypp_plugins.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +''' + :codeauthor: :email:`Bo Maryniuk ` +''' + +# Import Python Libs +from __future__ import absolute_import + +# Import Salt Testing Libs +from tests.support.unit import TestCase, skipIf +from tests.support.mock import ( + MagicMock, + patch, + NO_MOCK, + NO_MOCK_REASON +) + +import os +import imp +import sys +from zypp_plugin import BogusIO + +if sys.version_info >= (3,): + BUILTINS_OPEN = 'builtins.open' +else: + BUILTINS_OPEN = '__builtin__.open' + +zyppnotify = imp.load_source('zyppnotify', os.path.sep.join(os.path.dirname(__file__).split( + os.path.sep)[:-2] + ['scripts', 'suse', 'zypper', 'plugins', 'commit', 'zyppnotify'])) + + +@skipIf(NO_MOCK, NO_MOCK_REASON) +class ZyppPluginsTestCase(TestCase): + ''' + Test shipped libzypp plugins. + ''' + def test_drift_detector(self): + ''' + Test drift detector for a correct cookie file. + Returns: + + ''' + drift = zyppnotify.DriftDetector() + drift._get_mtime = MagicMock(return_value=123) + drift._get_checksum = MagicMock(return_value='deadbeef') + bogus_io = BogusIO() + with patch(BUILTINS_OPEN, bogus_io): + drift.PLUGINEND(None, None) + self.assertEqual(str(bogus_io), 'deadbeef 123\n') + self.assertEqual(bogus_io.mode, 'w') + self.assertEqual(bogus_io.path, '/var/cache/salt/minion/rpmdb.cookie') diff --git a/tests/unit/transport/mixins.py b/tests/unit/transport/mixins.py index a42c2dabc8e..43333050e73 100644 --- a/tests/unit/transport/mixins.py +++ b/tests/unit/transport/mixins.py @@ -7,7 +7,7 @@ from __future__ import absolute_import import salt.transport.client # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six class ReqChannelMixin(object): diff --git a/tests/unit/transport/test_tcp.py b/tests/unit/transport/test_tcp.py index a4e24e5db02..fb8e6701ad9 100644 --- a/tests/unit/transport/test_tcp.py +++ b/tests/unit/transport/test_tcp.py @@ -9,19 +9,24 @@ import threading import tornado.gen import tornado.ioloop -from tornado.testing import AsyncTestCase +import tornado.concurrent +from tornado.testing import AsyncTestCase, gen_test import salt.config -import salt.ext.six as six -import salt.utils +from salt.ext import six +import salt.utils.platform +import salt.utils.process import salt.transport.server import salt.transport.client import salt.exceptions +from salt.ext.six.moves import range +from salt.transport.tcp import SaltMessageClientPool # Import Salt Testing libs from tests.support.unit import TestCase, skipIf from tests.support.helpers import get_unused_localhost_port, flaky from tests.support.mixins import AdaptedConfigurationTestCaseMixin +from tests.support.mock import MagicMock, patch from tests.unit.transport.mixins import PubChannelMixin, ReqChannelMixin @@ -96,7 +101,7 @@ class BaseTCPReqCase(TestCase, AdaptedConfigurationTestCaseMixin): raise tornado.gen.Return((payload, {'fun': 'send_clear'})) -@skipIf(salt.utils.is_darwin(), 'hanging test suite on MacOS') +@skipIf(salt.utils.platform.is_darwin(), 'hanging test suite on MacOS') class ClearReqTestCases(BaseTCPReqCase, ReqChannelMixin): ''' Test all of the clear msg stuff @@ -116,7 +121,7 @@ class ClearReqTestCases(BaseTCPReqCase, ReqChannelMixin): raise tornado.gen.Return((payload, {'fun': 'send_clear'})) -@skipIf(salt.utils.is_darwin(), 'hanging test suite on MacOS') +@skipIf(salt.utils.platform.is_darwin(), 'hanging test suite on MacOS') class AESReqTestCases(BaseTCPReqCase, ReqChannelMixin): def setUp(self): self.channel = salt.transport.client.ReqChannel.factory(self.minion_config) @@ -234,3 +239,74 @@ class AsyncPubChannelTest(BaseTCPPubCase, PubChannelMixin): ''' Tests around the publish system ''' + + +class SaltMessageClientPoolTest(AsyncTestCase): + def setUp(self): + super(SaltMessageClientPoolTest, self).setUp() + sock_pool_size = 5 + with patch('salt.transport.tcp.SaltMessageClient.__init__', MagicMock(return_value=None)): + self.message_client_pool = SaltMessageClientPool({'sock_pool_size': sock_pool_size}, + args=({}, '', 0)) + self.original_message_clients = self.message_client_pool.message_clients + self.message_client_pool.message_clients = [MagicMock() for _ in range(sock_pool_size)] + + def tearDown(self): + with patch('salt.transport.tcp.SaltMessageClient.close', MagicMock(return_value=None)): + del self.original_message_clients + super(SaltMessageClientPoolTest, self).tearDown() + + def test_send(self): + for message_client_mock in self.message_client_pool.message_clients: + message_client_mock.send_queue = [0, 0, 0] + message_client_mock.send.return_value = [] + self.assertEqual([], self.message_client_pool.send()) + self.message_client_pool.message_clients[2].send_queue = [0] + self.message_client_pool.message_clients[2].send.return_value = [1] + self.assertEqual([1], self.message_client_pool.send()) + + def test_write_to_stream(self): + for message_client_mock in self.message_client_pool.message_clients: + message_client_mock.send_queue = [0, 0, 0] + message_client_mock._stream.write.return_value = [] + self.assertEqual([], self.message_client_pool.write_to_stream('')) + self.message_client_pool.message_clients[2].send_queue = [0] + self.message_client_pool.message_clients[2]._stream.write.return_value = [1] + self.assertEqual([1], self.message_client_pool.write_to_stream('')) + + def test_close(self): + self.message_client_pool.close() + self.assertEqual([], self.message_client_pool.message_clients) + + def test_on_recv(self): + for message_client_mock in self.message_client_pool.message_clients: + message_client_mock.on_recv.return_value = None + self.message_client_pool.on_recv() + for message_client_mock in self.message_client_pool.message_clients: + self.assertTrue(message_client_mock.on_recv.called) + + def test_connect_all(self): + @gen_test + def test_connect(self): + yield self.message_client_pool.connect() + + for message_client_mock in self.message_client_pool.message_clients: + future = tornado.concurrent.Future() + future.set_result('foo') + message_client_mock.connect.return_value = future + + self.assertIsNone(test_connect(self)) + + def test_connect_partial(self): + @gen_test(timeout=0.1) + def test_connect(self): + yield self.message_client_pool.connect() + + for idx, message_client_mock in enumerate(self.message_client_pool.message_clients): + future = tornado.concurrent.Future() + if idx % 2 == 0: + future.set_result('foo') + message_client_mock.connect.return_value = future + + with self.assertRaises(tornado.ioloop.TimeoutError): + test_connect(self) diff --git a/tests/unit/transport/test_zeromq.py b/tests/unit/transport/test_zeromq.py index e6fdf94bb98..fee80023228 100644 --- a/tests/unit/transport/test_zeromq.py +++ b/tests/unit/transport/test_zeromq.py @@ -25,17 +25,20 @@ import tornado.gen # Import Salt libs import salt.config -import salt.ext.six as six +from salt.ext import six import salt.utils import salt.transport.server import salt.transport.client import salt.exceptions +from salt.ext.six.moves import range +from salt.transport.zeromq import AsyncReqMessageClientPool # Import test support libs from tests.support.paths import TMP_CONF_DIR from tests.support.unit import TestCase, skipIf from tests.support.helpers import flaky, get_unused_localhost_port from tests.support.mixins import AdaptedConfigurationTestCaseMixin +from tests.support.mock import MagicMock, patch from tests.unit.transport.mixins import PubChannelMixin, ReqChannelMixin ON_SUSE = False @@ -271,3 +274,34 @@ class AsyncPubChannelTest(BaseZMQPubCase, PubChannelMixin): ''' def get_new_ioloop(self): return zmq.eventloop.ioloop.ZMQIOLoop() + + +class AsyncReqMessageClientPoolTest(TestCase): + def setUp(self): + super(AsyncReqMessageClientPoolTest, self).setUp() + sock_pool_size = 5 + with patch('salt.transport.zeromq.AsyncReqMessageClient.__init__', MagicMock(return_value=None)): + self.message_client_pool = AsyncReqMessageClientPool({'sock_pool_size': sock_pool_size}, + args=({}, '')) + self.original_message_clients = self.message_client_pool.message_clients + self.message_client_pool.message_clients = [MagicMock() for _ in range(sock_pool_size)] + + def tearDown(self): + with patch('salt.transport.zeromq.AsyncReqMessageClient.destroy', MagicMock(return_value=None)): + del self.original_message_clients + super(AsyncReqMessageClientPoolTest, self).tearDown() + + def test_send(self): + for message_client_mock in self.message_client_pool.message_clients: + message_client_mock.send_queue = [0, 0, 0] + message_client_mock.send.return_value = [] + + self.assertEqual([], self.message_client_pool.send()) + + self.message_client_pool.message_clients[2].send_queue = [0] + self.message_client_pool.message_clients[2].send.return_value = [1] + self.assertEqual([1], self.message_client_pool.send()) + + def test_destroy(self): + self.message_client_pool.destroy() + self.assertEqual([], self.message_client_pool.message_clients) diff --git a/tests/unit/utils/cache_mods/cache_mod.py b/tests/unit/utils/cache_mods/cache_mod.py index 40032a58323..91fe9ee6879 100644 --- a/tests/unit/utils/cache_mods/cache_mod.py +++ b/tests/unit/utils/cache_mods/cache_mod.py @@ -1,14 +1,15 @@ +# -*- coding: utf-8 -*- +''' +This is a module used in unit.utils.cache to test the context wrapper functions +''' + import salt.utils.cache -''' -This is a module used in -unit.utils.cache to test -the context wrapper functions. -''' def __virtual__(): return True + @salt.utils.cache.context_cache def test_context_module(): if 'called' in __context__: diff --git a/tests/unit/utils/test_args.py b/tests/unit/utils/test_args.py index 928a9f60cd6..73899782654 100644 --- a/tests/unit/utils/test_args.py +++ b/tests/unit/utils/test_args.py @@ -2,16 +2,23 @@ # Import python libs from __future__ import absolute_import +from collections import namedtuple # Import Salt Libs -from salt.utils import args +from salt.exceptions import SaltInvocationError +import salt.utils.args # Import Salt Testing Libs from tests.support.unit import TestCase, skipIf -from tests.support.mock import NO_MOCK, NO_MOCK_REASON +from tests.support.mock import ( + create_autospec, + DEFAULT, + NO_MOCK, + NO_MOCK_REASON, + patch +) -@skipIf(NO_MOCK, NO_MOCK_REASON) class ArgsTestCase(TestCase): ''' TestCase for salt.utils.args module @@ -21,5 +28,69 @@ class ArgsTestCase(TestCase): ''' Test passing a jid on the command line ''' - cmd = args.condition_input(['*', 'foo.bar', 20141020201325675584], None) + cmd = salt.utils.args.condition_input(['*', 'foo.bar', 20141020201325675584], None) self.assertIsInstance(cmd[2], str) + + def test_clean_kwargs(self): + self.assertDictEqual(salt.utils.args.clean_kwargs(foo='bar'), {'foo': 'bar'}) + self.assertDictEqual(salt.utils.args.clean_kwargs(__pub_foo='bar'), {}) + self.assertDictEqual(salt.utils.args.clean_kwargs(__foo_bar='gwar'), {}) + self.assertDictEqual(salt.utils.args.clean_kwargs(foo_bar='gwar'), {'foo_bar': 'gwar'}) + + def test_get_function_argspec(self): + def dummy_func(first, second, third, fourth='fifth'): + pass + + expected_argspec = namedtuple('ArgSpec', 'args varargs keywords defaults')( + args=['first', 'second', 'third', 'fourth'], varargs=None, keywords=None, defaults=('fifth',)) + ret = salt.utils.args.get_function_argspec(dummy_func) + + self.assertEqual(ret, expected_argspec) + + def test_parse_kwarg(self): + ret = salt.utils.args.parse_kwarg('foo=bar') + self.assertEqual(ret, ('foo', 'bar')) + + ret = salt.utils.args.parse_kwarg('foobar') + self.assertEqual(ret, (None, None)) + + def test_arg_lookup(self): + def dummy_func(first, second, third, fourth='fifth'): + pass + + expected_dict = {'args': ['first', 'second', 'third'], 'kwargs': {'fourth': 'fifth'}} + ret = salt.utils.args.arg_lookup(dummy_func) + self.assertEqual(expected_dict, ret) + + @skipIf(NO_MOCK, NO_MOCK_REASON) + def test_format_call(self): + with patch('salt.utils.args.arg_lookup') as arg_lookup: + def dummy_func(first=None, second=None, third=None): + pass + + arg_lookup.return_value = {'args': ['first', 'second', 'third'], 'kwargs': {}} + get_function_argspec = DEFAULT + get_function_argspec.return_value = namedtuple('ArgSpec', 'args varargs keywords defaults')( + args=['first', 'second', 'third', 'fourth'], varargs=None, keywords=None, defaults=('fifth',)) + + # Make sure we raise an error if we don't pass in the requisite number of arguments + self.assertRaises(SaltInvocationError, salt.utils.format_call, dummy_func, {'1': 2}) + + # Make sure we warn on invalid kwargs + ret = salt.utils.format_call(dummy_func, {'first': 2, 'second': 2, 'third': 3}) + self.assertGreaterEqual(len(ret['warnings']), 1) + + ret = salt.utils.format_call(dummy_func, {'first': 2, 'second': 2, 'third': 3}, + expected_extra_kws=('first', 'second', 'third')) + self.assertDictEqual(ret, {'args': [], 'kwargs': {}}) + + @skipIf(NO_MOCK, NO_MOCK_REASON) + def test_argspec_report(self): + def _test_spec(arg1, arg2, kwarg1=None): + pass + + sys_mock = create_autospec(_test_spec) + test_functions = {'test_module.test_spec': sys_mock} + ret = salt.utils.args.argspec_report(test_functions, 'test_module.test_spec') + self.assertDictEqual(ret, {'test_module.test_spec': + {'kwargs': True, 'args': None, 'defaults': None, 'varargs': True}}) diff --git a/tests/unit/utils/test_async.py b/tests/unit/utils/test_async.py index fc6b2b2106a..bea7a95a74d 100644 --- a/tests/unit/utils/test_async.py +++ b/tests/unit/utils/test_async.py @@ -8,7 +8,7 @@ import tornado.testing import tornado.gen from tornado.testing import AsyncTestCase -from salt.utils import async +import salt.utils.async as async class HelperA(object): diff --git a/tests/unit/utils/test_cache.py b/tests/unit/utils/test_cache.py index 88b9d07ac9b..649bb2abc7b 100644 --- a/tests/unit/utils/test_cache.py +++ b/tests/unit/utils/test_cache.py @@ -19,7 +19,7 @@ from tests.support.unit import TestCase # Import salt libs import salt.config import salt.loader -from salt.utils import cache +import salt.utils.cache as cache class CacheDictTestCase(TestCase): diff --git a/tests/unit/utils/test_cloud.py b/tests/unit/utils/test_cloud.py index f8cb0217f7d..93ae5702532 100644 --- a/tests/unit/utils/test_cloud.py +++ b/tests/unit/utils/test_cloud.py @@ -19,7 +19,7 @@ from tests.support.unit import TestCase, skipIf from tests.support.paths import TMP, CODE_DIR # Import salt libs -from salt.utils import cloud +import salt.utils.cloud as cloud GPG_KEYDIR = os.path.join(TMP, 'gpg-keydir') diff --git a/tests/unit/utils/test_color.py b/tests/unit/utils/test_color.py new file mode 100644 index 00000000000..cc4c835b65c --- /dev/null +++ b/tests/unit/utils/test_color.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +''' +Unit tests for salt.utils.color.py +''' + +# Import python libs +from __future__ import absolute_import + +# Import Salt Testing libs +from tests.support.unit import TestCase + +# Import Salt libs +import salt.utils.color + + +class ColorUtilsTestCase(TestCase): + + def test_get_colors(self): + ret = salt.utils.color.get_colors() + self.assertEqual('\x1b[0;37m', str(ret['LIGHT_GRAY'])) + + ret = salt.utils.color.get_colors(use=False) + self.assertDictContainsSubset({'LIGHT_GRAY': ''}, ret) + + ret = salt.utils.color.get_colors(use='LIGHT_GRAY') + # LIGHT_YELLOW now == LIGHT_GRAY + self.assertEqual(str(ret['LIGHT_YELLOW']), str(ret['LIGHT_GRAY'])) diff --git a/tests/unit/utils/test_configcomparer.py b/tests/unit/utils/test_configcomparer.py index 2cc17e38d2d..ab0e1581941 100644 --- a/tests/unit/utils/test_configcomparer.py +++ b/tests/unit/utils/test_configcomparer.py @@ -8,7 +8,7 @@ import copy from tests.support.unit import TestCase # Import Salt libs -from salt.utils import configcomparer +import salt.utils.configcomparer as configcomparer class UtilConfigcomparerTestCase(TestCase): diff --git a/tests/unit/utils/test_configparser.py b/tests/unit/utils/test_configparser.py new file mode 100644 index 00000000000..8fe98aee110 --- /dev/null +++ b/tests/unit/utils/test_configparser.py @@ -0,0 +1,268 @@ +# -*- coding: utf-8 -*- +''' +tests.unit.utils.test_configparser +================================== + +Test the funcs in the custom parsers in salt.utils.configparser +''' +# Import Python Libs +from __future__ import absolute_import +import copy +import errno +import logging +import os + +log = logging.getLogger(__name__) + +# Import Salt Testing Libs +from tests.support.unit import TestCase +from tests.support.paths import TMP + +# Import salt libs +import salt.utils.files +import salt.utils.stringutils +import salt.utils.configparser + +# The user.name param here is intentionally indented with spaces instead of a +# tab to test that we properly load a file with mixed indentation. +ORIG_CONFIG = u'''[user] + name = Артём Анисимов +\temail = foo@bar.com +[remote "origin"] +\turl = https://github.com/terminalmage/salt.git +\tfetch = +refs/heads/*:refs/remotes/origin/* +\tpushurl = git@github.com:terminalmage/salt.git +[color "diff"] +\told = 196 +\tnew = 39 +[core] +\tpager = less -R +\trepositoryformatversion = 0 +\tfilemode = true +\tbare = false +\tlogallrefupdates = true +[alias] +\tmodified = ! git status --porcelain | awk 'match($1, "M"){print $2}' +\tgraph = log --all --decorate --oneline --graph +\thist = log --pretty=format:\\"%h %ad | %s%d [%an]\\" --graph --date=short +[http] +\tsslverify = false'''.split(u'\n') # future lint: disable=non-unicode-string + + +class TestGitConfigParser(TestCase): + ''' + Tests for salt.utils.configparser.GitConfigParser + ''' + maxDiff = None + orig_config = os.path.join(TMP, u'test_gitconfig.orig') + new_config = os.path.join(TMP, u'test_gitconfig.new') + remote = u'remote "origin"' + + def tearDown(self): + del self.conf + try: + os.remove(self.new_config) + except OSError as exc: + if exc.errno != errno.ENOENT: + raise + + def setUp(self): + if not os.path.exists(self.orig_config): + with salt.utils.files.fopen(self.orig_config, u'wb') as fp_: + fp_.write( + salt.utils.stringutils.to_bytes( + u'\n'.join(ORIG_CONFIG) + ) + ) + self.conf = salt.utils.configparser.GitConfigParser() + self.conf.read(self.orig_config) + + @classmethod + def tearDownClass(cls): + try: + os.remove(cls.orig_config) + except OSError as exc: + if exc.errno != errno.ENOENT: + raise + + @staticmethod + def fix_indent(lines): + ''' + Fixes the space-indented 'user' line, because when we write the config + object to a file space indentation will be replaced by tab indentation. + ''' + ret = copy.copy(lines) + for i, _ in enumerate(ret): + if ret[i].startswith(salt.utils.configparser.GitConfigParser.SPACEINDENT): + ret[i] = ret[i].replace(salt.utils.configparser.GitConfigParser.SPACEINDENT, u'\t') + return ret + + @staticmethod + def get_lines(path): + with salt.utils.files.fopen(path, u'r') as fp_: + return salt.utils.stringutils.to_unicode(fp_.read()).splitlines() + + def _test_write(self, mode): + with salt.utils.files.fopen(self.new_config, mode) as fp_: + self.conf.write(fp_) + self.assertEqual( + self.get_lines(self.new_config), + self.fix_indent(ORIG_CONFIG) + ) + + def test_get(self): + ''' + Test getting an option's value + ''' + # Numeric values should be loaded as strings + self.assertEqual(self.conf.get(u'color "diff"', u'old'), u'196') + # Complex strings should be loaded with their literal quotes and + # slashes intact + self.assertEqual( + self.conf.get(u'alias', u'modified'), + u"""! git status --porcelain | awk 'match($1, "M"){print $2}'""" + ) + # future lint: disable=non-unicode-string + self.assertEqual( + self.conf.get(u'alias', u'hist'), + salt.utils.stringutils.to_unicode( + r"""log --pretty=format:\"%h %ad | %s%d [%an]\" --graph --date=short""" + ) + ) + # future lint: enable=non-unicode-string + + def test_read_space_indent(self): + ''' + Test that user.name was successfully loaded despite being indented + using spaces instead of a tab. Additionally, this tests that the value + was loaded as a unicode type on PY2. + ''' + self.assertEqual(self.conf.get(u'user', u'name'), u'Артём Анисимов') + + def test_set_new_option(self): + ''' + Test setting a new option in an existing section + ''' + self.conf.set(u'http', u'useragent', u'myawesomeagent') + self.assertEqual(self.conf.get(u'http', u'useragent'), u'myawesomeagent') + + def test_add_section(self): + ''' + Test adding a section and adding an item to that section + ''' + self.conf.add_section(u'foo') + self.conf.set(u'foo', u'bar', u'baz') + self.assertEqual(self.conf.get(u'foo', u'bar'), u'baz') + + def test_replace_option(self): + ''' + Test replacing an existing option + ''' + # We're also testing the normalization of key names, here. Setting + # "sslVerify" should actually set an "sslverify" option. + self.conf.set(u'http', u'sslVerify', u'true') + self.assertEqual(self.conf.get(u'http', u'sslverify'), u'true') + + def test_set_multivar(self): + ''' + Test setting a multivar and then writing the resulting file + ''' + orig_refspec = u'+refs/heads/*:refs/remotes/origin/*' + new_refspec = u'+refs/tags/*:refs/tags/*' + # Make sure that the original value is a string + self.assertEqual( + self.conf.get(self.remote, u'fetch'), + orig_refspec + ) + # Add another refspec + self.conf.set_multivar(self.remote, u'fetch', new_refspec) + # The value should now be a list + self.assertEqual( + self.conf.get(self.remote, u'fetch'), + [orig_refspec, new_refspec] + ) + # Write the config object to a file + with salt.utils.files.fopen(self.new_config, u'w') as fp_: + self.conf.write(fp_) + # Confirm that the new file was written correctly + expected = self.fix_indent(ORIG_CONFIG) + expected.insert(6, u'\tfetch = %s' % new_refspec) # pylint: disable=string-substitution-usage-error + self.assertEqual(self.get_lines(self.new_config), expected) + + def test_remove_option(self): + ''' + test removing an option, including all items from a multivar + ''' + for item in (u'fetch', u'pushurl'): + self.conf.remove_option(self.remote, item) + # To confirm that the option is now gone, a get should raise an + # NoOptionError exception. + self.assertRaises( + salt.utils.configparser.NoOptionError, + self.conf.get, + self.remote, + item) + + def test_remove_option_regexp(self): + ''' + test removing an option, including all items from a multivar + ''' + orig_refspec = u'+refs/heads/*:refs/remotes/origin/*' + new_refspec_1 = u'+refs/tags/*:refs/tags/*' + new_refspec_2 = u'+refs/foo/*:refs/foo/*' + # First, add both refspecs + self.conf.set_multivar(self.remote, u'fetch', new_refspec_1) + self.conf.set_multivar(self.remote, u'fetch', new_refspec_2) + # Make sure that all three values are there + self.assertEqual( + self.conf.get(self.remote, u'fetch'), + [orig_refspec, new_refspec_1, new_refspec_2] + ) + # If the regex doesn't match, no items should be removed + self.assertFalse( + self.conf.remove_option_regexp( + self.remote, + u'fetch', + salt.utils.stringutils.to_unicode(r'\d{7,10}') # future lint: disable=non-unicode-string + ) + ) + # Make sure that all three values are still there (since none should + # have been removed) + self.assertEqual( + self.conf.get(self.remote, u'fetch'), + [orig_refspec, new_refspec_1, new_refspec_2] + ) + # Remove one of the values + self.assertTrue( + self.conf.remove_option_regexp(self.remote, u'fetch', u'tags')) + # Confirm that the value is gone + self.assertEqual( + self.conf.get(self.remote, u'fetch'), + [orig_refspec, new_refspec_2] + ) + # Remove the other one we added earlier + self.assertTrue( + self.conf.remove_option_regexp(self.remote, u'fetch', u'foo')) + # Since the option now only has one value, it should be a string + self.assertEqual(self.conf.get(self.remote, u'fetch'), orig_refspec) + # Remove the last remaining option + self.assertTrue( + self.conf.remove_option_regexp(self.remote, u'fetch', u'heads')) + # Trying to do a get now should raise an exception + self.assertRaises( + salt.utils.configparser.NoOptionError, + self.conf.get, + self.remote, + u'fetch') + + def test_write(self): + ''' + Test writing using non-binary filehandle + ''' + self._test_write(mode=u'w') + + def test_write_binary(self): + ''' + Test writing using binary filehandle + ''' + self._test_write(mode=u'wb') diff --git a/tests/unit/utils/test_context.py b/tests/unit/utils/test_context.py index 58bf11ec237..0856bbd3fc7 100644 --- a/tests/unit/utils/test_context.py +++ b/tests/unit/utils/test_context.py @@ -11,11 +11,11 @@ import shutil # Import Salt testing libraries from tests.support.unit import TestCase, skipIf from tests.support.mock import NO_MOCK, NO_MOCK_REASON -from salt.utils.cache import context_cache # Import Salt libraries import salt.payload -import salt.utils +import salt.utils.cache +import salt.utils.files __context__ = {'a': 'b'} __opts__ = {'cachedir': '/tmp'} @@ -38,7 +38,7 @@ class ContextCacheTest(TestCase): ''' Tests to ensure the cache is written correctly ''' - @context_cache + @salt.utils.cache.context_cache def _test_set_cache(): ''' This will inherit globals from the test module itself. @@ -52,7 +52,7 @@ class ContextCacheTest(TestCase): self.assertTrue(os.path.isfile(target_cache_file), 'Context cache did not write cache file') # Test manual de-serialize - with salt.utils.fopen(target_cache_file, 'rb') as fp_: + with salt.utils.files.fopen(target_cache_file, 'rb') as fp_: target_cache_data = salt.payload.Serial(__opts__).load(fp_) self.assertDictEqual(__context__, target_cache_data) @@ -66,13 +66,13 @@ class ContextCacheTest(TestCase): Tests to ensure that the context cache can rehydrate a wrapped function ''' # First populate the cache - @context_cache + @salt.utils.cache.context_cache def _test_set_cache(): pass _test_set_cache() # Then try to rehydate a func - @context_cache + @salt.utils.cache.context_cache def _test_refill_cache(comparison_context): self.assertEqual(__context__, comparison_context) diff --git a/tests/unit/utils/test_decorators.py b/tests/unit/utils/test_decorators.py index a729ec024c5..f2884469d96 100644 --- a/tests/unit/utils/test_decorators.py +++ b/tests/unit/utils/test_decorators.py @@ -7,12 +7,12 @@ # Import Python libs from __future__ import absolute_import -# Import Salt Testing libs -from tests.support.unit import skipIf, TestCase -from tests.support.mock import NO_MOCK, NO_MOCK_REASON, patch -from salt.utils import decorators +# Import Salt libs +import salt.utils.decorators as decorators from salt.version import SaltStackVersion from salt.exceptions import CommandExecutionError, SaltConfigurationError +from tests.support.unit import skipIf, TestCase +from tests.support.mock import NO_MOCK, NO_MOCK_REASON, patch class DummyLogger(object): @@ -59,6 +59,7 @@ class DecoratorsTest(TestCase): self.globs = { '__virtualname__': 'test', '__opts__': {}, + '__pillar__': {}, 'old_function': self.old_function, 'new_function': self.new_function, '_new_function': self._new_function, @@ -149,6 +150,23 @@ class DecoratorsTest(TestCase): ['The function "test.new_function" is using its deprecated ' 'version and will expire in version "Beryllium".']) + def test_with_deprecated_notfound_in_pillar(self): + ''' + Test with_deprecated should raise an exception, if a same name + function with the "_" prefix not implemented. + + :return: + ''' + del self.globs['_new_function'] + self.globs['__pillar__']['use_deprecated'] = ['test.new_function'] + depr = decorators.with_deprecated(self.globs, "Beryllium") + depr._curr_version = self._mk_version("Helium")[1] + with self.assertRaises(CommandExecutionError): + depr(self.new_function)() + self.assertEqual(self.messages, + ['The function "test.new_function" is using its deprecated ' + 'version and will expire in version "Beryllium".']) + def test_with_deprecated_found(self): ''' Test with_deprecated should not raise an exception, if a same name @@ -166,6 +184,23 @@ class DecoratorsTest(TestCase): 'and will expire in version "Beryllium".'] self.assertEqual(self.messages, log_msg) + def test_with_deprecated_found_in_pillar(self): + ''' + Test with_deprecated should not raise an exception, if a same name + function with the "_" prefix is implemented, but should use + an old version instead, if "use_deprecated" is requested. + + :return: + ''' + self.globs['__pillar__']['use_deprecated'] = ['test.new_function'] + self.globs['_new_function'] = self.old_function + depr = decorators.with_deprecated(self.globs, "Beryllium") + depr._curr_version = self._mk_version("Helium")[1] + self.assertEqual(depr(self.new_function)(), self.old_function()) + log_msg = ['The function "test.new_function" is using its deprecated version ' + 'and will expire in version "Beryllium".'] + self.assertEqual(self.messages, log_msg) + def test_with_deprecated_found_eol(self): ''' Test with_deprecated should raise an exception, if a same name @@ -185,6 +220,25 @@ class DecoratorsTest(TestCase): 'is configured as its deprecated version. The lifetime of the function ' '"new_function" expired. Please use its successor "new_function" instead.']) + def test_with_deprecated_found_eol_in_pillar(self): + ''' + Test with_deprecated should raise an exception, if a same name + function with the "_" prefix is implemented, "use_deprecated" is requested + and EOL is reached. + + :return: + ''' + self.globs['__pillar__']['use_deprecated'] = ['test.new_function'] + self.globs['_new_function'] = self.old_function + depr = decorators.with_deprecated(self.globs, "Helium") + depr._curr_version = self._mk_version("Beryllium")[1] + with self.assertRaises(CommandExecutionError): + depr(self.new_function)() + self.assertEqual(self.messages, + ['Although function "new_function" is called, an alias "new_function" ' + 'is configured as its deprecated version. The lifetime of the function ' + '"new_function" expired. Please use its successor "new_function" instead.']) + def test_with_deprecated_no_conf(self): ''' Test with_deprecated should not raise an exception, if a same name @@ -260,6 +314,19 @@ class DecoratorsTest(TestCase): assert depr(self.new_function)() == self.new_function() assert not self.messages + def test_with_deprecated_opt_in_use_superseded_in_pillar(self): + ''' + Test with_deprecated using opt-in policy, + where newer function is used as per configuration. + + :return: + ''' + self.globs['__pillar__']['use_superseded'] = ['test.new_function'] + depr = decorators.with_deprecated(self.globs, "Beryllium", policy=decorators._DeprecationDecorator.OPT_IN) + depr._curr_version = self._mk_version("Helium")[1] + assert depr(self.new_function)() == self.new_function() + assert not self.messages + def test_with_deprecated_opt_in_use_superseded_and_deprecated(self): ''' Test with_deprecated misconfiguration. @@ -272,3 +339,16 @@ class DecoratorsTest(TestCase): depr._curr_version = self._mk_version("Helium")[1] with self.assertRaises(SaltConfigurationError): assert depr(self.new_function)() == self.new_function() + + def test_with_deprecated_opt_in_use_superseded_and_deprecated_in_pillar(self): + ''' + Test with_deprecated misconfiguration. + + :return: + ''' + self.globs['__pillar__']['use_deprecated'] = ['test.new_function'] + self.globs['__pillar__']['use_superseded'] = ['test.new_function'] + depr = decorators.with_deprecated(self.globs, "Beryllium") + depr._curr_version = self._mk_version("Helium")[1] + with self.assertRaises(SaltConfigurationError): + assert depr(self.new_function)() == self.new_function() diff --git a/tests/unit/utils/test_dictupdate.py b/tests/unit/utils/test_dictupdate.py index df482970f76..892c7b03df5 100644 --- a/tests/unit/utils/test_dictupdate.py +++ b/tests/unit/utils/test_dictupdate.py @@ -8,7 +8,7 @@ import copy from tests.support.unit import TestCase # Import Salt libs -from salt.utils import dictupdate +import salt.utils.dictupdate as dictupdate class UtilDictupdateTestCase(TestCase): diff --git a/tests/unit/utils/test_disk_cache.py b/tests/unit/utils/test_disk_cache.py index 2d37afc98b6..1180cf7f522 100644 --- a/tests/unit/utils/test_disk_cache.py +++ b/tests/unit/utils/test_disk_cache.py @@ -17,7 +17,7 @@ import time from tests.support.unit import TestCase # Import salt libs -from salt.utils import cache +import salt.utils.cache as cache class CacheDiskTestCase(TestCase): diff --git a/tests/unit/utils/test_doc.py b/tests/unit/utils/test_doc.py new file mode 100644 index 00000000000..8e74b60f6f4 --- /dev/null +++ b/tests/unit/utils/test_doc.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- +''' +Unit Tests for functions located in salt.utils.doc.py. +''' + +# Import python libs +from __future__ import absolute_import + +# Import Salt libs +import salt.utils.doc + +# Import Salt Testing libs +from tests.support.unit import TestCase + + +class DocUtilsTestCase(TestCase): + ''' + Test case for doc util. + ''' + + def test_parse_docstring(self): + test_keystone_str = '''Management of Keystone users + ============================ + + :depends: - keystoneclient Python module + :configuration: See :py:mod:`salt.modules.keystone` for setup instructions. +''' + + ret = salt.utils.doc.parse_docstring(test_keystone_str) + expected_dict = {'deps': ['keystoneclient'], + 'full': 'Management of Keystone users\n ' + '============================\n\n ' + ':depends: - keystoneclient Python module\n ' + ':configuration: See :py:mod:`salt.modules.keystone` for setup instructions.\n'} + self.assertDictEqual(ret, expected_dict) diff --git a/tests/unit/utils/test_docker.py b/tests/unit/utils/test_docker.py index 5b0587f18c3..baec4476e17 100644 --- a/tests/unit/utils/test_docker.py +++ b/tests/unit/utils/test_docker.py @@ -18,6 +18,7 @@ from tests.support.unit import TestCase # Import salt libs import salt.config import salt.loader +import salt.utils.platform import salt.utils.docker as docker_utils import salt.utils.docker.translate as translate_funcs @@ -29,7 +30,7 @@ def __test_stringlist(testcase, name): alias = salt.utils.docker.ALIASES_REVMAP.get(name) # Using file paths here because "volumes" must be passed through this # set of assertions and it requires absolute paths. - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): data = [r'c:\foo', r'c:\bar', r'c:\baz'] else: data = ['/foo', '/bar', '/baz'] @@ -189,7 +190,7 @@ def assert_string(func): alias = salt.utils.docker.ALIASES_REVMAP.get(name) # Using file paths here because "working_dir" must be passed through # this set of assertions and it requires absolute paths. - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): data = r'c:\foo' else: data = '/foo' @@ -384,7 +385,7 @@ def assert_device_rates(func): continue # Error case: Not an absolute path - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): path = r'foo\bar\baz' else: path = 'foo/bar/baz' @@ -820,7 +821,7 @@ class TranslateInputTestCase(TestCase): expected ) - @assert_stringlist + @assert_string def test_domainname(self): ''' Should be a list of strings or converted to one @@ -1709,7 +1710,7 @@ class TranslateInputTestCase(TestCase): Should be a list of absolute paths ''' # Error case: Not an absolute path - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): path = r'foo\bar\baz' else: path = 'foo/bar/baz' @@ -1735,7 +1736,7 @@ class TranslateInputTestCase(TestCase): Should be a single absolute path ''' # Error case: Not an absolute path - if salt.utils.is_windows(): + if salt.utils.platform.is_windows(): path = r'foo\bar\baz' else: path = 'foo/bar/baz' diff --git a/tests/unit/utils/test_etcd_util.py b/tests/unit/utils/test_etcd_util.py index c199ca35545..f80c7bbdaa0 100644 --- a/tests/unit/utils/test_etcd_util.py +++ b/tests/unit/utils/test_etcd_util.py @@ -16,7 +16,7 @@ from tests.support.mock import ( ) # Import Salt Libs -from salt.utils import etcd_util +import salt.utils.etcd_util as etcd_util try: from urllib3.exceptions import ReadTimeoutError, MaxRetryError HAS_URLLIB3 = True @@ -88,10 +88,11 @@ class EtcdUtilTestCase(TestCase): self.assertEqual(client.get('salt', recurse=True), 'stack') mock.assert_called_with('salt', recursive=True) - mock.side_effect = etcd.EtcdKeyNotFound() + # iter(list(Exception)) works correctly with both mock<1.1 and mock>=1.1 + mock.side_effect = iter([etcd.EtcdKeyNotFound()]) self.assertEqual(client.get('not-found'), None) - mock.side_effect = etcd.EtcdConnectionFailed() + mock.side_effect = iter([etcd.EtcdConnectionFailed()]) self.assertEqual(client.get('watching'), None) # python 2.6 test @@ -126,7 +127,8 @@ class EtcdUtilTestCase(TestCase): mock.assert_any_call('/x') mock.assert_any_call('/x/c') - mock.side_effect = etcd.EtcdKeyNotFound() + # iter(list(Exception)) works correctly with both mock<1.1 and mock>=1.1 + mock.side_effect = iter([etcd.EtcdKeyNotFound()]) self.assertEqual(client.tree('not-found'), None) mock.side_effect = ValueError @@ -149,7 +151,8 @@ class EtcdUtilTestCase(TestCase): self.assertEqual(client.ls('/x'), {'/x': {'/x/a': '1', '/x/b': '2', '/x/c/': {}}}) mock.assert_called_with('/x') - mock.side_effect = etcd.EtcdKeyNotFound() + # iter(list(Exception)) works correctly with both mock<1.1 and mock>=1.1 + mock.side_effect = iter([etcd.EtcdKeyNotFound()]) self.assertEqual(client.ls('/not-found'), {}) mock.side_effect = Exception @@ -327,11 +330,13 @@ class EtcdUtilTestCase(TestCase): {'value': 'stack', 'key': '/some-key', 'mIndex': 1, 'changed': True, 'dir': True}) mock.assert_called_with('/some-dir', wait=True, recursive=True, timeout=5, waitIndex=10) - mock.side_effect = MaxRetryError(None, None) + # iter(list(Exception)) works correctly with both mock<1.1 and mock>=1.1 + mock.side_effect = iter([MaxRetryError(None, None)]) self.assertEqual(client.watch('/some-key'), {}) - mock.side_effect = etcd.EtcdConnectionFailed() + mock.side_effect = iter([etcd.EtcdConnectionFailed()]) self.assertEqual(client.watch('/some-key'), {}) + mock.side_effect = None mock.return_value = None self.assertEqual(client.watch('/some-key'), {}) diff --git a/tests/unit/utils/test_event.py b/tests/unit/utils/test_event.py index c65b397d27c..722f2b4b14f 100644 --- a/tests/unit/utils/test_event.py +++ b/tests/unit/utils/test_event.py @@ -25,9 +25,10 @@ from multiprocessing import Process from tests.support.unit import expectedFailure, skipIf, TestCase # Import salt libs +import salt.utils.event +import salt.utils.stringutils import tests.integration as integration from salt.utils.process import clean_proc -from salt.utils import event, to_bytes # Import 3rd-+arty libs from salt.ext.six.moves import range # pylint: disable=import-error,redefined-builtin @@ -41,7 +42,7 @@ if getattr(zmq, 'IPC_PATH_MAX_LEN', 103) <= 103: @contextmanager def eventpublisher_process(): - proc = event.EventPublisher({'sock_dir': SOCK_DIR}) + proc = salt.utils.event.EventPublisher({'sock_dir': SOCK_DIR}) proc.start() try: if os.environ.get('TRAVIS_PYTHON_VERSION', None) is not None: @@ -62,7 +63,7 @@ class EventSender(Process): self.wait = wait def run(self): - me = event.MasterEvent(SOCK_DIR, listen=False) + me = salt.utils.event.MasterEvent(SOCK_DIR, listen=False) time.sleep(self.wait) me.fire_event(self.data, self.tag) # Wait a few seconds before tearing down the zmq context @@ -98,7 +99,7 @@ class TestSaltEvent(TestCase): self.assertEqual(data[key], evt[key], assertMsg) def test_master_event(self): - me = event.MasterEvent(SOCK_DIR, listen=False) + me = salt.utils.event.MasterEvent(SOCK_DIR, listen=False) self.assertEqual( me.puburi, '{0}'.format( os.path.join(SOCK_DIR, 'master_event_pub.ipc') @@ -113,8 +114,8 @@ class TestSaltEvent(TestCase): def test_minion_event(self): opts = dict(id='foo', sock_dir=SOCK_DIR) - id_hash = hashlib.sha256(to_bytes(opts['id'])).hexdigest()[:10] - me = event.MinionEvent(opts, listen=False) + id_hash = hashlib.sha256(salt.utils.stringutils.to_bytes(opts['id'])).hexdigest()[:10] + me = salt.utils.event.MinionEvent(opts, listen=False) self.assertEqual( me.puburi, '{0}'.format( @@ -134,13 +135,13 @@ class TestSaltEvent(TestCase): def test_minion_event_tcp_ipc_mode(self): opts = dict(id='foo', ipc_mode='tcp') - me = event.MinionEvent(opts, listen=False) + me = salt.utils.event.MinionEvent(opts, listen=False) self.assertEqual(me.puburi, 4510) self.assertEqual(me.pulluri, 4511) def test_minion_event_no_id(self): - me = event.MinionEvent(dict(sock_dir=SOCK_DIR), listen=False) - id_hash = hashlib.sha256(to_bytes('')).hexdigest()[:10] + me = salt.utils.event.MinionEvent(dict(sock_dir=SOCK_DIR), listen=False) + id_hash = hashlib.sha256(salt.utils.stringutils.to_bytes('')).hexdigest()[:10] self.assertEqual( me.puburi, '{0}'.format( @@ -161,7 +162,7 @@ class TestSaltEvent(TestCase): def test_event_single(self): '''Test a single event is received''' with eventpublisher_process(): - me = event.MasterEvent(SOCK_DIR, listen=True) + me = salt.utils.event.MasterEvent(SOCK_DIR, listen=True) me.fire_event({'data': 'foo1'}, 'evt1') evt1 = me.get_event(tag='evt1') self.assertGotEvent(evt1, {'data': 'foo1'}) @@ -169,7 +170,7 @@ class TestSaltEvent(TestCase): def test_event_single_no_block(self): '''Test a single event is received, no block''' with eventpublisher_process(): - me = event.MasterEvent(SOCK_DIR, listen=True) + me = salt.utils.event.MasterEvent(SOCK_DIR, listen=True) start = time.time() finish = start + 5 evt1 = me.get_event(wait=0, tag='evt1', no_block=True) @@ -185,7 +186,7 @@ class TestSaltEvent(TestCase): def test_event_single_wait_0_no_block_False(self): '''Test a single event is received with wait=0 and no_block=False and doesn't spin the while loop''' with eventpublisher_process(): - me = event.MasterEvent(SOCK_DIR, listen=True) + me = salt.utils.event.MasterEvent(SOCK_DIR, listen=True) me.fire_event({'data': 'foo1'}, 'evt1') # This is too fast and will be None but assures we're not blocking evt1 = me.get_event(wait=0, tag='evt1', no_block=False) @@ -194,7 +195,7 @@ class TestSaltEvent(TestCase): def test_event_timeout(self): '''Test no event is received if the timeout is reached''' with eventpublisher_process(): - me = event.MasterEvent(SOCK_DIR, listen=True) + me = salt.utils.event.MasterEvent(SOCK_DIR, listen=True) me.fire_event({'data': 'foo1'}, 'evt1') evt1 = me.get_event(tag='evt1') self.assertGotEvent(evt1, {'data': 'foo1'}) @@ -204,7 +205,7 @@ class TestSaltEvent(TestCase): def test_event_no_timeout(self): '''Test no wait timeout, we should block forever, until we get one ''' with eventpublisher_process(): - me = event.MasterEvent(SOCK_DIR, listen=True) + me = salt.utils.event.MasterEvent(SOCK_DIR, listen=True) with eventsender_process({'data': 'foo2'}, 'evt2', 5): evt = me.get_event(tag='evt2', wait=0, no_block=False) self.assertGotEvent(evt, {'data': 'foo2'}) @@ -212,7 +213,7 @@ class TestSaltEvent(TestCase): def test_event_matching(self): '''Test a startswith match''' with eventpublisher_process(): - me = event.MasterEvent(SOCK_DIR, listen=True) + me = salt.utils.event.MasterEvent(SOCK_DIR, listen=True) me.fire_event({'data': 'foo1'}, 'evt1') evt1 = me.get_event(tag='ev') self.assertGotEvent(evt1, {'data': 'foo1'}) @@ -220,7 +221,7 @@ class TestSaltEvent(TestCase): def test_event_matching_regex(self): '''Test a regex match''' with eventpublisher_process(): - me = event.MasterEvent(SOCK_DIR, listen=True) + me = salt.utils.event.MasterEvent(SOCK_DIR, listen=True) me.fire_event({'data': 'foo1'}, 'evt1') evt1 = me.get_event(tag='^ev', match_type='regex') self.assertGotEvent(evt1, {'data': 'foo1'}) @@ -228,7 +229,7 @@ class TestSaltEvent(TestCase): def test_event_matching_all(self): '''Test an all match''' with eventpublisher_process(): - me = event.MasterEvent(SOCK_DIR, listen=True) + me = salt.utils.event.MasterEvent(SOCK_DIR, listen=True) me.fire_event({'data': 'foo1'}, 'evt1') evt1 = me.get_event(tag='') self.assertGotEvent(evt1, {'data': 'foo1'}) @@ -236,7 +237,7 @@ class TestSaltEvent(TestCase): def test_event_matching_all_when_tag_is_None(self): '''Test event matching all when not passing a tag''' with eventpublisher_process(): - me = event.MasterEvent(SOCK_DIR, listen=True) + me = salt.utils.event.MasterEvent(SOCK_DIR, listen=True) me.fire_event({'data': 'foo1'}, 'evt1') evt1 = me.get_event() self.assertGotEvent(evt1, {'data': 'foo1'}) @@ -244,7 +245,7 @@ class TestSaltEvent(TestCase): def test_event_not_subscribed(self): '''Test get_event drops non-subscribed events''' with eventpublisher_process(): - me = event.MasterEvent(SOCK_DIR, listen=True) + me = salt.utils.event.MasterEvent(SOCK_DIR, listen=True) me.fire_event({'data': 'foo1'}, 'evt1') me.fire_event({'data': 'foo2'}, 'evt2') evt2 = me.get_event(tag='evt2') @@ -255,7 +256,7 @@ class TestSaltEvent(TestCase): def test_event_subscription_cache(self): '''Test subscriptions cache a message until requested''' with eventpublisher_process(): - me = event.MasterEvent(SOCK_DIR, listen=True) + me = salt.utils.event.MasterEvent(SOCK_DIR, listen=True) me.subscribe('evt1') me.fire_event({'data': 'foo1'}, 'evt1') me.fire_event({'data': 'foo2'}, 'evt2') @@ -267,7 +268,7 @@ class TestSaltEvent(TestCase): def test_event_subscriptions_cache_regex(self): '''Test regex subscriptions cache a message until requested''' with eventpublisher_process(): - me = event.MasterEvent(SOCK_DIR, listen=True) + me = salt.utils.event.MasterEvent(SOCK_DIR, listen=True) me.subscribe('e..1$', 'regex') me.fire_event({'data': 'foo1'}, 'evt1') me.fire_event({'data': 'foo2'}, 'evt2') @@ -279,8 +280,8 @@ class TestSaltEvent(TestCase): def test_event_multiple_clients(self): '''Test event is received by multiple clients''' with eventpublisher_process(): - me1 = event.MasterEvent(SOCK_DIR, listen=True) - me2 = event.MasterEvent(SOCK_DIR, listen=True) + me1 = salt.utils.event.MasterEvent(SOCK_DIR, listen=True) + me2 = salt.utils.event.MasterEvent(SOCK_DIR, listen=True) # We need to sleep here to avoid a race condition wherein # the second socket may not be connected by the time the first socket # sends the event. @@ -296,7 +297,7 @@ class TestSaltEvent(TestCase): '''Test nested event subscriptions do not drop events, get event for all tags''' # Show why not to call get_event(tag='') with eventpublisher_process(): - me = event.MasterEvent(SOCK_DIR, listen=True) + me = salt.utils.event.MasterEvent(SOCK_DIR, listen=True) me.fire_event({'data': 'foo1'}, 'evt1') me.fire_event({'data': 'foo2'}, 'evt2') evt2 = me.get_event(tag='') @@ -307,7 +308,7 @@ class TestSaltEvent(TestCase): def test_event_many(self): '''Test a large number of events, one at a time''' with eventpublisher_process(): - me = event.MasterEvent(SOCK_DIR, listen=True) + me = salt.utils.event.MasterEvent(SOCK_DIR, listen=True) for i in range(500): me.fire_event({'data': '{0}'.format(i)}, 'testevents') evt = me.get_event(tag='testevents') @@ -316,7 +317,7 @@ class TestSaltEvent(TestCase): def test_event_many_backlog(self): '''Test a large number of events, send all then recv all''' with eventpublisher_process(): - me = event.MasterEvent(SOCK_DIR, listen=True) + me = salt.utils.event.MasterEvent(SOCK_DIR, listen=True) # Must not exceed zmq HWM for i in range(500): me.fire_event({'data': '{0}'.format(i)}, 'testevents') @@ -329,7 +330,7 @@ class TestSaltEvent(TestCase): def test_send_master_event(self): '''Tests that sending an event through fire_master generates expected event''' with eventpublisher_process(): - me = event.MasterEvent(SOCK_DIR, listen=True) + me = salt.utils.event.MasterEvent(SOCK_DIR, listen=True) data = {'data': 'foo1'} me.fire_master(data, 'test_master') @@ -344,21 +345,21 @@ class TestAsyncEventPublisher(AsyncTestCase): def setUp(self): super(TestAsyncEventPublisher, self).setUp() self.opts = {'sock_dir': SOCK_DIR} - self.publisher = event.AsyncEventPublisher( + self.publisher = salt.utils.event.AsyncEventPublisher( self.opts, self.io_loop, ) - self.event = event.get_event('minion', opts=self.opts, io_loop=self.io_loop) + self.event = salt.utils.event.get_event('minion', opts=self.opts, io_loop=self.io_loop) self.event.subscribe('') self.event.set_event_handler(self._handle_publish) def _handle_publish(self, raw): - self.tag, self.data = event.SaltEvent.unpack(raw) + self.tag, self.data = salt.utils.event.SaltEvent.unpack(raw) self.stop() def test_event_subscription(self): '''Test a single event is received''' - me = event.MinionEvent(self.opts, listen=True) + me = salt.utils.event.MinionEvent(self.opts, listen=True) me.fire_event({'data': 'foo1'}, 'evt1') self.wait() evt1 = me.get_event(tag='evt1') diff --git a/tests/unit/utils/test_extend.py b/tests/unit/utils/test_extend.py index bf3d9b3b53a..691d2991920 100644 --- a/tests/unit/utils/test_extend.py +++ b/tests/unit/utils/test_extend.py @@ -18,9 +18,9 @@ from tests.support.unit import TestCase from tests.support.mock import MagicMock, patch # Import salt libs -import salt.utils.extend import tests.integration as integration -import salt.utils +import salt.utils.extend +import salt.utils.files class ExtendTestCase(TestCase): @@ -44,5 +44,5 @@ class ExtendTestCase(TestCase): self.assertFalse(os.path.exists(os.path.join(out, 'template.yml'))) self.assertTrue(os.path.exists(os.path.join(out, 'directory'))) self.assertTrue(os.path.exists(os.path.join(out, 'directory', 'test.py'))) - with salt.utils.fopen(os.path.join(out, 'directory', 'test.py'), 'r') as test_f: + with salt.utils.files.fopen(os.path.join(out, 'directory', 'test.py'), 'r') as test_f: self.assertEqual(test_f.read(), year) diff --git a/tests/unit/utils/test_files.py b/tests/unit/utils/test_files.py new file mode 100644 index 00000000000..9d092227bb1 --- /dev/null +++ b/tests/unit/utils/test_files.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +''' +Unit Tests for functions located in salt.utils.files.py. +''' + +# Import python libs +from __future__ import absolute_import +import os + +# Import Salt libs +import salt.utils.files + +# Import Salt Testing libs +from tests.support.unit import TestCase, skipIf +from tests.support.mock import ( + patch, + NO_MOCK, + NO_MOCK_REASON +) + + +class FilesUtilTestCase(TestCase): + ''' + Test case for files util. + ''' + + @skipIf(NO_MOCK, NO_MOCK_REASON) + def test_safe_rm(self): + with patch('os.remove') as os_remove_mock: + salt.utils.files.safe_rm('dummy_tgt') + self.assertTrue(os_remove_mock.called) + + @skipIf(os.path.exists('/tmp/no_way_this_is_a_file_nope.sh'), 'Test file exists! Skipping safe_rm_exceptions test!') + def test_safe_rm_exceptions(self): + error = False + try: + salt.utils.files.safe_rm('/tmp/no_way_this_is_a_file_nope.sh') + except (IOError, OSError): + error = True + self.assertFalse(error, 'salt.utils.files.safe_rm raised exception when it should not have') diff --git a/tests/unit/utils/test_find.py b/tests/unit/utils/test_find.py index 86af790b0e3..2599e88bdec 100644 --- a/tests/unit/utils/test_find.py +++ b/tests/unit/utils/test_find.py @@ -13,8 +13,7 @@ from tests.support.unit import skipIf, TestCase from tests.support.paths import TMP # Import salt libs -import salt.ext.six as six -import salt.utils +import salt.utils.files import salt.utils.find # Import 3rd-party libs @@ -121,19 +120,11 @@ class TestFind(TestCase): min_size, max_size = salt.utils.find._parse_size('+1m') self.assertEqual(min_size, 1048576) - # sys.maxint has been removed in Python3. Use maxsize instead. - if six.PY3: - self.assertEqual(max_size, sys.maxsize) - else: - self.assertEqual(max_size, sys.maxint) + self.assertEqual(max_size, sys.maxsize) min_size, max_size = salt.utils.find._parse_size('+1M') self.assertEqual(min_size, 1048576) - # sys.maxint has been removed in Python3. Use maxsize instead. - if six.PY3: - self.assertEqual(max_size, sys.maxsize) - else: - self.assertEqual(max_size, sys.maxint) + self.assertEqual(max_size, sys.maxsize) def test_option_requires(self): option = salt.utils.find.Option() @@ -217,7 +208,7 @@ class TestFind(TestCase): option = salt.utils.find.TypeOption('type', 's') self.assertEqual(option.match('', '', [stat.S_IFSOCK]), True) - @skipIf(sys.platform.startswith('win'), 'No /dev/null on Windows') + @skipIf(sys.platform.startswith('win'), 'pwd not available on Windows') def test_owner_option_requires(self): self.assertRaises( ValueError, salt.utils.find.OwnerOption, 'owner', 'notexist' @@ -226,7 +217,7 @@ class TestFind(TestCase): option = salt.utils.find.OwnerOption('owner', 'root') self.assertEqual(option.requires(), salt.utils.find._REQUIRES_STAT) - @skipIf(sys.platform.startswith('win'), 'No /dev/null on Windows') + @skipIf(sys.platform.startswith('win'), 'pwd not available on Windows') def test_owner_option_match(self): option = salt.utils.find.OwnerOption('owner', 'root') self.assertEqual(option.match('', '', [0] * 5), True) @@ -234,7 +225,7 @@ class TestFind(TestCase): option = salt.utils.find.OwnerOption('owner', '500') self.assertEqual(option.match('', '', [500] * 5), True) - @skipIf(sys.platform.startswith('win'), 'No /dev/null on Windows') + @skipIf(sys.platform.startswith('win'), 'grp not available on Windows') def test_group_option_requires(self): self.assertRaises( ValueError, salt.utils.find.GroupOption, 'group', 'notexist' @@ -247,7 +238,7 @@ class TestFind(TestCase): option = salt.utils.find.GroupOption('group', group_name) self.assertEqual(option.requires(), salt.utils.find._REQUIRES_STAT) - @skipIf(sys.platform.startswith('win'), 'No /dev/null on Windows') + @skipIf(sys.platform.startswith('win'), 'grp not available on Windows') def test_group_option_match(self): if sys.platform.startswith(('darwin', 'freebsd', 'openbsd')): group_name = 'wheel' @@ -313,7 +304,7 @@ class TestGrepOption(TestCase): def test_grep_option_match_regular_file(self): hello_file = os.path.join(self.tmpdir, 'hello.txt') - with salt.utils.fopen(hello_file, 'w') as fp_: + with salt.utils.files.fopen(hello_file, 'w') as fp_: fp_.write('foo') option = salt.utils.find.GrepOption('grep', 'foo') self.assertEqual( @@ -372,7 +363,7 @@ class TestPrintOption(TestCase): def test_print_option_execute(self): hello_file = os.path.join(self.tmpdir, 'hello.txt') - with salt.utils.fopen(hello_file, 'w') as fp_: + with salt.utils.files.fopen(hello_file, 'w') as fp_: fp_.write('foo') option = salt.utils.find.PrintOption('print', '') @@ -412,7 +403,7 @@ class TestPrintOption(TestCase): option.execute('test_name', [0] * 9), [0, 'test_name'] ) - @skipIf(sys.platform.startswith('Windows'), "no /dev/null on windows") + @skipIf(sys.platform.startswith('win'), "pwd not available on Windows") def test_print_user(self): option = salt.utils.find.PrintOption('print', 'user') self.assertEqual(option.execute('', [0] * 10), 'root') @@ -420,7 +411,7 @@ class TestPrintOption(TestCase): option = salt.utils.find.PrintOption('print', 'user') self.assertEqual(option.execute('', [2 ** 31] * 10), 2 ** 31) - @skipIf(sys.platform.startswith('Windows'), "no /dev/null on windows") + @skipIf(sys.platform.startswith('win'), "grp not available on Windows") def test_print_group(self): option = salt.utils.find.PrintOption('print', 'group') if sys.platform.startswith(('darwin', 'freebsd', 'openbsd')): @@ -433,7 +424,7 @@ class TestPrintOption(TestCase): #option = salt.utils.find.PrintOption('print', 'group') #self.assertEqual(option.execute('', [2 ** 31] * 10), 2 ** 31) - @skipIf(sys.platform.startswith('Windows'), "no /dev/null on windows") + @skipIf(sys.platform.startswith('win'), "no /dev/null on windows") def test_print_md5(self): option = salt.utils.find.PrintOption('print', 'md5') self.assertEqual(option.execute('/dev/null', os.stat('/dev/null')), '') @@ -561,7 +552,7 @@ class TestFinder(TestCase): def test_find(self): hello_file = os.path.join(self.tmpdir, 'hello.txt') - with salt.utils.fopen(hello_file, 'w') as fp_: + with salt.utils.files.fopen(hello_file, 'w') as fp_: fp_.write('foo') finder = salt.utils.find.Finder({}) diff --git a/tests/unit/utils/test_gitfs.py b/tests/unit/utils/test_gitfs.py index c8942e46954..070a46fe75e 100644 --- a/tests/unit/utils/test_gitfs.py +++ b/tests/unit/utils/test_gitfs.py @@ -66,7 +66,7 @@ class TestGitFSProvider(TestCase): ('git_pillar', salt.utils.gitfs.GitPillar), ('winrepo', salt.utils.gitfs.WinRepo)): key = '{0}_provider'.format(role_name) - for provider in salt.utils.gitfs.VALID_PROVIDERS: + for provider in salt.utils.gitfs.GIT_PROVIDERS: verify = 'verify_gitpython' mock1 = _get_mock(verify, provider) with patch.object(role_class, verify, mock1): diff --git a/tests/unit/utils/test_http.py b/tests/unit/utils/test_http.py index 993e7975e8d..a6d24b021ce 100644 --- a/tests/unit/utils/test_http.py +++ b/tests/unit/utils/test_http.py @@ -11,7 +11,7 @@ from tests.support.unit import TestCase, skipIf from tests.support.mock import NO_MOCK, NO_MOCK_REASON # Import Salt Libs -from salt.utils import http +import salt.utils.http as http @skipIf(NO_MOCK, NO_MOCK_REASON) diff --git a/tests/unit/utils/test_immutabletypes.py b/tests/unit/utils/test_immutabletypes.py index 4626866b137..f6842963242 100644 --- a/tests/unit/utils/test_immutabletypes.py +++ b/tests/unit/utils/test_immutabletypes.py @@ -16,7 +16,7 @@ from __future__ import absolute_import from tests.support.unit import TestCase # Import salt libs -from salt.utils import immutabletypes +import salt.utils.immutabletypes as immutabletypes class ImmutableTypesTestCase(TestCase): diff --git a/tests/unit/utils/test_locales.py b/tests/unit/utils/test_locales.py index 45232544cea..60e2bd8151c 100644 --- a/tests/unit/utils/test_locales.py +++ b/tests/unit/utils/test_locales.py @@ -1,16 +1,17 @@ # coding: utf-8 -# python libs +# Import Python libs from __future__ import absolute_import -# salt testing libs + +# Import Salt libs +import salt.utils.locales as locales from tests.support.unit import TestCase, skipIf from tests.support.mock import patch, NO_MOCK, NO_MOCK_REASON -# salt libs -import salt.ext.six as six +# Import 3rd-part libs +from salt.ext import six from salt.ext.six.moves import reload_module -from salt.utils import locales @skipIf(NO_MOCK, NO_MOCK_REASON) diff --git a/tests/unit/utils/test_mac_utils.py b/tests/unit/utils/test_mac_utils.py index 3b7bc32d33d..4c60e928ed8 100644 --- a/tests/unit/utils/test_mac_utils.py +++ b/tests/unit/utils/test_mac_utils.py @@ -9,12 +9,14 @@ from __future__ import absolute_import # Import Salt Testing Libs from tests.support.unit import TestCase, skipIf from tests.support.mock import MagicMock, patch, NO_MOCK, NO_MOCK_REASON -from salt.ext.six.moves import range -# Import Salt Libs -from salt.utils import mac_utils +# Import Salt libs +import salt.utils.mac_utils as mac_utils from salt.exceptions import SaltInvocationError, CommandExecutionError +# Import 3rd-party libs +from salt.ext.six.moves import range + @skipIf(NO_MOCK, NO_MOCK_REASON) class MacUtilsTestCase(TestCase): diff --git a/tests/unit/utils/test_minions.py b/tests/unit/utils/test_minions.py index 3a025960cc9..fbec4e2c8b4 100644 --- a/tests/unit/utils/test_minions.py +++ b/tests/unit/utils/test_minions.py @@ -4,10 +4,14 @@ from __future__ import absolute_import # Import Salt Libs -from salt.utils import minions +import salt.utils.minions as minions # Import Salt Testing Libs from tests.support.unit import TestCase +from tests.support.mock import ( + patch, + MagicMock, +) NODEGROUPS = { 'group1': 'L@host1,host2,host3', @@ -36,3 +40,329 @@ class MinionsTestCase(TestCase): expected = EXPECTED[nodegroup] ret = minions.nodegroup_comp(nodegroup, NODEGROUPS) self.assertEqual(ret, expected) + + +class CkMinionsTestCase(TestCase): + ''' + TestCase for salt.utils.minions.CkMinions class + ''' + def setUp(self): + self.ckminions = minions.CkMinions({}) + + def test_spec_check(self): + # Test spec-only rule + auth_list = ['@runner'] + ret = self.ckminions.spec_check(auth_list, 'test.arg', {}, 'runner') + self.assertTrue(ret) + ret = self.ckminions.spec_check(auth_list, 'test.arg', {}, 'wheel') + self.assertFalse(ret) + ret = self.ckminions.spec_check(auth_list, 'testarg', {}, 'runner') + mock_ret = {'error': {'name': 'SaltInvocationError', + 'message': 'A command invocation error occurred: Check syntax.'}} + self.assertDictEqual(mock_ret, ret) + + # Test spec in plural form + auth_list = ['@runners'] + ret = self.ckminions.spec_check(auth_list, 'test.arg', {}, 'runner') + self.assertTrue(ret) + ret = self.ckminions.spec_check(auth_list, 'test.arg', {}, 'wheel') + self.assertFalse(ret) + + # Test spec with module.function restriction + auth_list = [{'@runner': 'test.arg'}] + ret = self.ckminions.spec_check(auth_list, 'test.arg', {}, 'runner') + self.assertTrue(ret) + ret = self.ckminions.spec_check(auth_list, 'test.arg', {}, 'wheel') + self.assertFalse(ret) + ret = self.ckminions.spec_check(auth_list, 'tes.arg', {}, 'runner') + self.assertFalse(ret) + ret = self.ckminions.spec_check(auth_list, 'test.ar', {}, 'runner') + self.assertFalse(ret) + + # Test function name is a regex + auth_list = [{'@runner': 'test.arg.*some'}] + ret = self.ckminions.spec_check(auth_list, 'test.arg', {}, 'runner') + self.assertFalse(ret) + ret = self.ckminions.spec_check(auth_list, 'test.argsome', {}, 'runner') + self.assertTrue(ret) + ret = self.ckminions.spec_check(auth_list, 'test.arg_aaa_some', {}, 'runner') + self.assertTrue(ret) + + # Test a list of funcs + auth_list = [{'@runner': ['test.arg', 'jobs.active']}] + ret = self.ckminions.spec_check(auth_list, 'test.arg', {}, 'runner') + self.assertTrue(ret) + ret = self.ckminions.spec_check(auth_list, 'jobs.active', {}, 'runner') + self.assertTrue(ret) + ret = self.ckminions.spec_check(auth_list, 'test.active', {}, 'runner') + self.assertFalse(ret) + ret = self.ckminions.spec_check(auth_list, 'jobs.arg', {}, 'runner') + self.assertFalse(ret) + + # Test args-kwargs rules + auth_list = [{ + '@runner': { + 'test.arg': { + 'args': ['1', '2'], + 'kwargs': { + 'aaa': 'bbb', + 'ccc': 'ddd' + } + } + } + }] + ret = self.ckminions.spec_check(auth_list, 'test.arg', {}, 'runner') + self.assertFalse(ret) + args = { + 'arg': ['1', '2'], + 'kwarg': {'aaa': 'bbb', 'ccc': 'ddd'} + } + ret = self.ckminions.spec_check(auth_list, 'test.arg', args, 'runner') + self.assertTrue(ret) + args = { + 'arg': ['1', '2', '3'], + 'kwarg': {'aaa': 'bbb', 'ccc': 'ddd'} + } + ret = self.ckminions.spec_check(auth_list, 'test.arg', args, 'runner') + self.assertTrue(ret) + args = { + 'arg': ['1', '2'], + 'kwarg': {'aaa': 'bbb', 'ccc': 'ddd', 'zzz': 'zzz'} + } + ret = self.ckminions.spec_check(auth_list, 'test.arg', args, 'runner') + self.assertTrue(ret) + args = { + 'arg': ['1', '2'], + 'kwarg': {'aaa': 'bbb', 'ccc': 'ddc'} + } + ret = self.ckminions.spec_check(auth_list, 'test.arg', args, 'runner') + self.assertFalse(ret) + args = { + 'arg': ['1', '2'], + 'kwarg': {'aaa': 'bbb'} + } + ret = self.ckminions.spec_check(auth_list, 'test.arg', args, 'runner') + self.assertFalse(ret) + args = { + 'arg': ['1', '3'], + 'kwarg': {'aaa': 'bbb', 'ccc': 'ddd'} + } + ret = self.ckminions.spec_check(auth_list, 'test.arg', args, 'runner') + self.assertFalse(ret) + args = { + 'arg': ['1'], + 'kwarg': {'aaa': 'bbb', 'ccc': 'ddd'} + } + ret = self.ckminions.spec_check(auth_list, 'test.arg', args, 'runner') + self.assertFalse(ret) + args = { + 'kwarg': {'aaa': 'bbb', 'ccc': 'ddd'} + } + ret = self.ckminions.spec_check(auth_list, 'test.arg', args, 'runner') + self.assertFalse(ret) + args = { + 'arg': ['1', '2'], + } + ret = self.ckminions.spec_check(auth_list, 'test.arg', args, 'runner') + self.assertFalse(ret) + + # Test kwargs only + auth_list = [{ + '@runner': { + 'test.arg': { + 'kwargs': { + 'aaa': 'bbb', + 'ccc': 'ddd' + } + } + } + }] + ret = self.ckminions.spec_check(auth_list, 'test.arg', {}, 'runner') + self.assertFalse(ret) + args = { + 'arg': ['1', '2'], + 'kwarg': {'aaa': 'bbb', 'ccc': 'ddd'} + } + ret = self.ckminions.spec_check(auth_list, 'test.arg', args, 'runner') + self.assertTrue(ret) + + # Test args only + auth_list = [{ + '@runner': { + 'test.arg': { + 'args': ['1', '2'] + } + } + }] + ret = self.ckminions.spec_check(auth_list, 'test.arg', {}, 'runner') + self.assertFalse(ret) + args = { + 'arg': ['1', '2'], + 'kwarg': {'aaa': 'bbb', 'ccc': 'ddd'} + } + ret = self.ckminions.spec_check(auth_list, 'test.arg', args, 'runner') + self.assertTrue(ret) + + # Test list of args + auth_list = [{'@runner': [{'test.arg': [{'args': ['1', '2'], + 'kwargs': {'aaa': 'bbb', + 'ccc': 'ddd' + } + }, + {'args': ['2', '3'], + 'kwargs': {'aaa': 'aaa', + 'ccc': 'ccc' + } + }] + }] + }] + args = { + 'arg': ['1', '2'], + 'kwarg': {'aaa': 'bbb', 'ccc': 'ddd'} + } + ret = self.ckminions.spec_check(auth_list, 'test.arg', args, 'runner') + self.assertTrue(ret) + args = { + 'arg': ['2', '3'], + 'kwarg': {'aaa': 'aaa', 'ccc': 'ccc'} + } + ret = self.ckminions.spec_check(auth_list, 'test.arg', args, 'runner') + self.assertTrue(ret) + + # Test @module form + auth_list = ['@jobs'] + ret = self.ckminions.spec_check(auth_list, 'jobs.active', {}, 'runner') + self.assertTrue(ret) + ret = self.ckminions.spec_check(auth_list, 'jobs.active', {}, 'wheel') + self.assertTrue(ret) + ret = self.ckminions.spec_check(auth_list, 'test.arg', {}, 'runner') + self.assertFalse(ret) + ret = self.ckminions.spec_check(auth_list, 'job.arg', {}, 'runner') + self.assertFalse(ret) + + # Test @module: function + auth_list = [{'@jobs': 'active'}] + ret = self.ckminions.spec_check(auth_list, 'jobs.active', {}, 'runner') + self.assertTrue(ret) + ret = self.ckminions.spec_check(auth_list, 'jobs.active', {}, 'wheel') + self.assertTrue(ret) + ret = self.ckminions.spec_check(auth_list, 'jobs.active_jobs', {}, 'runner') + self.assertTrue(ret) + ret = self.ckminions.spec_check(auth_list, 'jobs.activ', {}, 'runner') + self.assertFalse(ret) + + # Test @module: [functions] + auth_list = [{'@jobs': ['active', 'li']}] + ret = self.ckminions.spec_check(auth_list, 'jobs.active', {}, 'runner') + self.assertTrue(ret) + ret = self.ckminions.spec_check(auth_list, 'jobs.list_jobs', {}, 'runner') + self.assertTrue(ret) + ret = self.ckminions.spec_check(auth_list, 'jobs.last_run', {}, 'runner') + self.assertFalse(ret) + + # Test @module: function with args + auth_list = [{'@jobs': {'active': {'args': ['1', '2'], + 'kwargs': {'a': 'b', 'c': 'd'}}}}] + args = {'arg': ['1', '2'], + 'kwarg': {'a': 'b', 'c': 'd'}} + ret = self.ckminions.spec_check(auth_list, 'jobs.active', args, 'runner') + self.assertTrue(ret) + ret = self.ckminions.spec_check(auth_list, 'jobs.active', {}, 'runner') + self.assertFalse(ret) + + @patch('salt.utils.minions.CkMinions._pki_minions', MagicMock(return_value=['alpha', 'beta', 'gamma'])) + def test_auth_check(self): + # Test function-only rule + auth_list = ['test.ping'] + ret = self.ckminions.auth_check(auth_list, 'test.ping', None, 'alpha') + self.assertTrue(ret) + ret = self.ckminions.auth_check(auth_list, 'test.arg', None, 'alpha') + self.assertFalse(ret) + + # Test minion and function + auth_list = [{'alpha': 'test.ping'}] + ret = self.ckminions.auth_check(auth_list, 'test.ping', None, 'alpha') + self.assertTrue(ret) + ret = self.ckminions.auth_check(auth_list, 'test.arg', None, 'alpha') + self.assertFalse(ret) + ret = self.ckminions.auth_check(auth_list, 'test.ping', None, 'beta') + self.assertFalse(ret) + + # Test function list + auth_list = [{'*': ['test.*', 'saltutil.cmd']}] + ret = self.ckminions.auth_check(auth_list, 'test.arg', None, 'alpha') + self.assertTrue(ret) + ret = self.ckminions.auth_check(auth_list, 'test.ping', None, 'beta') + self.assertTrue(ret) + ret = self.ckminions.auth_check(auth_list, 'saltutil.cmd', None, 'gamma') + self.assertTrue(ret) + ret = self.ckminions.auth_check(auth_list, 'saltutil.running', None, 'gamma') + self.assertFalse(ret) + + # Test an args and kwargs rule + auth_list = [{ + 'alpha': { + 'test.arg': { + 'args': ['1', '2'], + 'kwargs': { + 'aaa': 'bbb', + 'ccc': 'ddd' + } + } + } + }] + ret = self.ckminions.auth_check(auth_list, 'test.arg', None, 'runner') + self.assertFalse(ret) + ret = self.ckminions.auth_check(auth_list, 'test.arg', [], 'runner') + self.assertFalse(ret) + args = ['1', '2', {'aaa': 'bbb', 'ccc': 'ddd', '__kwarg__': True}] + ret = self.ckminions.auth_check(auth_list, 'test.arg', args, 'runner') + self.assertTrue(ret) + args = ['1', '2', '3', {'aaa': 'bbb', 'ccc': 'ddd', 'eee': 'fff', '__kwarg__': True}] + ret = self.ckminions.auth_check(auth_list, 'test.arg', args, 'runner') + self.assertTrue(ret) + args = ['1', {'aaa': 'bbb', 'ccc': 'ddd', '__kwarg__': True}] + ret = self.ckminions.auth_check(auth_list, 'test.arg', args, 'runner') + self.assertFalse(ret) + args = ['1', '2', {'aaa': 'bbb', '__kwarg__': True}] + ret = self.ckminions.auth_check(auth_list, 'test.arg', args, 'runner') + self.assertFalse(ret) + args = ['1', '3', {'aaa': 'bbb', 'ccc': 'ddd', '__kwarg__': True}] + ret = self.ckminions.auth_check(auth_list, 'test.arg', args, 'runner') + self.assertFalse(ret) + args = ['1', '2', {'aaa': 'bbb', 'ccc': 'fff', '__kwarg__': True}] + ret = self.ckminions.auth_check(auth_list, 'test.arg', args, 'runner') + self.assertFalse(ret) + + # Test kwargs only rule + auth_list = [{ + 'alpha': { + 'test.arg': { + 'kwargs': { + 'aaa': 'bbb', + 'ccc': 'ddd' + } + } + } + }] + args = ['1', '2', {'aaa': 'bbb', 'ccc': 'ddd', '__kwarg__': True}] + ret = self.ckminions.auth_check(auth_list, 'test.arg', args, 'runner') + self.assertTrue(ret) + args = [{'aaa': 'bbb', 'ccc': 'ddd', 'eee': 'fff', '__kwarg__': True}] + ret = self.ckminions.auth_check(auth_list, 'test.arg', args, 'runner') + self.assertTrue(ret) + + # Test args only rule + auth_list = [{ + 'alpha': { + 'test.arg': { + 'args': ['1', '2'], + } + } + }] + args = ['1', '2', {'aaa': 'bbb', 'ccc': 'ddd', '__kwarg__': True}] + ret = self.ckminions.auth_check(auth_list, 'test.arg', args, 'runner') + self.assertTrue(ret) + args = ['1', '2'] + ret = self.ckminions.auth_check(auth_list, 'test.arg', args, 'runner') + self.assertTrue(ret) diff --git a/tests/unit/utils/test_network.py b/tests/unit/utils/test_network.py index 4c1bb7fd830..fdb70a44039 100644 --- a/tests/unit/utils/test_network.py +++ b/tests/unit/utils/test_network.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # Import Python libs from __future__ import absolute_import +import socket # Import Salt Testing libs from tests.support.unit import skipIf @@ -8,7 +9,7 @@ from tests.support.unit import TestCase from tests.support.mock import NO_MOCK, NO_MOCK_REASON, patch, MagicMock # Import salt libs -from salt.utils import network +import salt.utils.network as network LINUX = '''\ eth0 Link encap:Ethernet HWaddr e0:3f:49:85:6a:af @@ -104,8 +105,36 @@ class NetworkTestCase(TestCase): self.assertEqual(ret, '10.1.2.3') def test_host_to_ips(self): - ret = network.host_to_ips('www.saltstack.com') - self.assertEqual(ret, ['104.199.122.13']) + ''' + NOTE: When this test fails it's usually because the IP address has + changed. In these cases, we just need to update the IP address in the + assertion. + ''' + def _side_effect(host, *args): + try: + return { + 'github.com': [ + (2, 1, 6, '', ('192.30.255.112', 0)), + (2, 1, 6, '', ('192.30.255.113', 0)), + ], + 'ipv6host.foo': [ + (socket.AF_INET6, 1, 6, '', ('2001:a71::1', 0, 0, 0)), + ], + }[host] + except KeyError: + raise socket.gaierror(-2, 'Name or service not known') + + getaddrinfo_mock = MagicMock(side_effect=_side_effect) + with patch.object(socket, 'getaddrinfo', getaddrinfo_mock): + # Test host that can be resolved + ret = network.host_to_ips('github.com') + self.assertEqual(ret, ['192.30.255.112', '192.30.255.113']) + # Test ipv6 + ret = network.host_to_ips('ipv6host.foo') + self.assertEqual(ret, ['2001:a71::1']) + # Test host that can't be resolved + ret = network.host_to_ips('someothersite.com') + self.assertEqual(ret, None) def test_generate_minion_id(self): self.assertTrue(network.generate_minion_id()) @@ -198,7 +227,7 @@ class NetworkTestCase(TestCase): ) def test_interfaces_ifconfig_solaris(self): - with patch('salt.utils.is_sunos', lambda: True): + with patch('salt.utils.platform.is_sunos', lambda: True): interfaces = network._interfaces_ifconfig(SOLARIS) expected_interfaces = {'ilbint0': {'inet6': [], @@ -233,16 +262,16 @@ class NetworkTestCase(TestCase): self.assertEqual(interfaces, expected_interfaces) def test_freebsd_remotes_on(self): - with patch('salt.utils.is_sunos', lambda: False): - with patch('salt.utils.is_freebsd', lambda: True): + with patch('salt.utils.platform.is_sunos', lambda: False): + with patch('salt.utils.platform.is_freebsd', lambda: True): with patch('subprocess.check_output', return_value=FREEBSD_SOCKSTAT): remotes = network._freebsd_remotes_on('4506', 'remote') self.assertEqual(remotes, set(['127.0.0.1'])) def test_freebsd_remotes_on_with_fat_pid(self): - with patch('salt.utils.is_sunos', lambda: False): - with patch('salt.utils.is_freebsd', lambda: True): + with patch('salt.utils.platform.is_sunos', lambda: False): + with patch('salt.utils.platform.is_freebsd', lambda: True): with patch('subprocess.check_output', return_value=FREEBSD_SOCKSTAT_WITH_FAT_PID): remotes = network._freebsd_remotes_on('4506', 'remote') @@ -258,7 +287,7 @@ class NetworkTestCase(TestCase): patch('socket.gethostname', MagicMock(return_value='hostname')), \ patch('socket.getfqdn', MagicMock(return_value='hostname.domainname.blank')), \ patch('socket.getaddrinfo', MagicMock(return_value=[(2, 3, 0, 'attrname', ('127.0.1.1', 0))])), \ - patch('salt.utils.fopen', MagicMock(return_value=False)), \ + patch('salt.utils.files.fopen', MagicMock(return_value=False)), \ patch('os.path.exists', MagicMock(return_value=False)), \ patch('salt.utils.network.ip_addrs', MagicMock(return_value=['1.2.3.4', '5.6.7.8'])): self.assertEqual(network._generate_minion_id(), @@ -274,7 +303,7 @@ class NetworkTestCase(TestCase): patch('socket.gethostname', MagicMock(return_value='127')), \ patch('socket.getfqdn', MagicMock(return_value='127.domainname.blank')), \ patch('socket.getaddrinfo', MagicMock(return_value=[(2, 3, 0, 'attrname', ('127.0.1.1', 0))])), \ - patch('salt.utils.fopen', MagicMock(return_value=False)), \ + patch('salt.utils.files.fopen', MagicMock(return_value=False)), \ patch('os.path.exists', MagicMock(return_value=False)), \ patch('salt.utils.network.ip_addrs', MagicMock(return_value=['1.2.3.4', '5.6.7.8'])): self.assertEqual(network._generate_minion_id(), @@ -290,7 +319,7 @@ class NetworkTestCase(TestCase): patch('socket.gethostname', MagicMock(return_value='127890')), \ patch('socket.getfqdn', MagicMock(return_value='127890.domainname.blank')), \ patch('socket.getaddrinfo', MagicMock(return_value=[(2, 3, 0, 'attrname', ('127.0.1.1', 0))])), \ - patch('salt.utils.fopen', MagicMock(return_value=False)), \ + patch('salt.utils.files.fopen', MagicMock(return_value=False)), \ patch('os.path.exists', MagicMock(return_value=False)), \ patch('salt.utils.network.ip_addrs', MagicMock(return_value=['1.2.3.4', '5.6.7.8'])): self.assertEqual(network._generate_minion_id(), @@ -306,7 +335,7 @@ class NetworkTestCase(TestCase): patch('socket.gethostname', MagicMock(return_value='hostname')), \ patch('socket.getfqdn', MagicMock(return_value='hostname')), \ patch('socket.getaddrinfo', MagicMock(return_value=[(2, 3, 0, 'hostname', ('127.0.1.1', 0))])), \ - patch('salt.utils.fopen', MagicMock(return_value=False)), \ + patch('salt.utils.files.fopen', MagicMock(return_value=False)), \ patch('os.path.exists', MagicMock(return_value=False)), \ patch('salt.utils.network.ip_addrs', MagicMock(return_value=['1.2.3.4', '1.2.3.4', '1.2.3.4'])): self.assertEqual(network._generate_minion_id(), ['hostname', '1.2.3.4']) @@ -322,7 +351,7 @@ class NetworkTestCase(TestCase): patch('socket.gethostname', MagicMock(return_value='hostname')), \ patch('socket.getfqdn', MagicMock(return_value='')), \ patch('socket.getaddrinfo', MagicMock(return_value=[(2, 3, 0, 'hostname', ('127.0.1.1', 0))])), \ - patch('salt.utils.fopen', MagicMock(return_value=False)), \ + patch('salt.utils.files.fopen', MagicMock(return_value=False)), \ patch('os.path.exists', MagicMock(return_value=False)), \ patch('salt.utils.network.ip_addrs', MagicMock(return_value=['1.2.3.4', '1.2.3.4', '1.2.3.4'])): self.assertEqual(network.generate_minion_id(), 'very.long.and.complex.domain.name') @@ -337,7 +366,7 @@ class NetworkTestCase(TestCase): patch('socket.gethostname', MagicMock(return_value='pick.me')), \ patch('socket.getfqdn', MagicMock(return_value='hostname.domainname.blank')), \ patch('socket.getaddrinfo', MagicMock(return_value=[(2, 3, 0, 'hostname', ('127.0.1.1', 0))])), \ - patch('salt.utils.fopen', MagicMock(return_value=False)), \ + patch('salt.utils.files.fopen', MagicMock(return_value=False)), \ patch('os.path.exists', MagicMock(return_value=False)), \ patch('salt.utils.network.ip_addrs', MagicMock(return_value=['1.2.3.4', '1.2.3.4', '1.2.3.4'])): self.assertEqual(network.generate_minion_id(), 'hostname.domainname.blank') @@ -352,7 +381,7 @@ class NetworkTestCase(TestCase): patch('socket.gethostname', MagicMock(return_value='ip6-loopback')), \ patch('socket.getfqdn', MagicMock(return_value='ip6-localhost')), \ patch('socket.getaddrinfo', MagicMock(return_value=[(2, 3, 0, 'localhost', ('127.0.1.1', 0))])), \ - patch('salt.utils.fopen', MagicMock(return_value=False)), \ + patch('salt.utils.files.fopen', MagicMock(return_value=False)), \ patch('os.path.exists', MagicMock(return_value=False)), \ patch('salt.utils.network.ip_addrs', MagicMock(return_value=['127.0.0.1', '::1', 'fe00::0', 'fe02::1', '1.2.3.4'])): self.assertEqual(network.generate_minion_id(), '1.2.3.4') @@ -367,7 +396,7 @@ class NetworkTestCase(TestCase): patch('socket.gethostname', MagicMock(return_value='ip6-loopback')), \ patch('socket.getfqdn', MagicMock(return_value='ip6-localhost')), \ patch('socket.getaddrinfo', MagicMock(return_value=[(2, 3, 0, 'localhost', ('127.0.1.1', 0))])), \ - patch('salt.utils.fopen', MagicMock(return_value=False)), \ + patch('salt.utils.files.fopen', MagicMock(return_value=False)), \ patch('os.path.exists', MagicMock(return_value=False)), \ patch('salt.utils.network.ip_addrs', MagicMock(return_value=['127.0.0.1', '::1', 'fe00::0', 'fe02::1'])): self.assertEqual(network.generate_minion_id(), 'localhost') @@ -382,7 +411,7 @@ class NetworkTestCase(TestCase): patch('socket.gethostname', MagicMock(return_value='ip6-loopback')), \ patch('socket.getfqdn', MagicMock(return_value='pick.me')), \ patch('socket.getaddrinfo', MagicMock(return_value=[(2, 3, 0, 'localhost', ('127.0.1.1', 0))])), \ - patch('salt.utils.fopen', MagicMock(return_value=False)), \ + patch('salt.utils.files.fopen', MagicMock(return_value=False)), \ patch('os.path.exists', MagicMock(return_value=False)), \ patch('salt.utils.network.ip_addrs', MagicMock(return_value=['127.0.0.1', '::1', 'fe00::0', 'fe02::1'])): self.assertEqual(network.generate_minion_id(), 'pick.me') @@ -397,7 +426,7 @@ class NetworkTestCase(TestCase): patch('socket.gethostname', MagicMock(return_value='ip6-loopback')), \ patch('socket.getfqdn', MagicMock(return_value='ip6-localhost')), \ patch('socket.getaddrinfo', MagicMock(return_value=[(2, 3, 0, 'pick.me', ('127.0.1.1', 0))])), \ - patch('salt.utils.fopen', MagicMock(return_value=False)), \ + patch('salt.utils.files.fopen', MagicMock(return_value=False)), \ patch('os.path.exists', MagicMock(return_value=False)), \ patch('salt.utils.network.ip_addrs', MagicMock(return_value=['127.0.0.1', '::1', 'fe00::0', 'fe02::1'])): self.assertEqual(network.generate_minion_id(), 'pick.me') @@ -412,7 +441,7 @@ class NetworkTestCase(TestCase): patch('socket.gethostname', MagicMock(return_value='ip6-loopback')), \ patch('socket.getfqdn', MagicMock(return_value='ip6-localhost')), \ patch('socket.getaddrinfo', MagicMock(return_value=[(2, 3, 0, 'localhost', ('127.0.1.1', 0))])), \ - patch('salt.utils.fopen', MagicMock(return_value=False)), \ + patch('salt.utils.files.fopen', MagicMock(return_value=False)), \ patch('os.path.exists', MagicMock(return_value=False)), \ patch('salt.utils.network.ip_addrs', MagicMock(return_value=['127.0.0.1', '::1', 'fe00::0', 'fe02::1', '1.2.3.4'])): self.assertEqual(network.generate_minion_id(), '1.2.3.4') diff --git a/tests/unit/utils/test_path_join.py b/tests/unit/utils/test_path.py similarity index 74% rename from tests/unit/utils/test_path_join.py rename to tests/unit/utils/test_path.py index ddba151e435..c9bc6761a37 100644 --- a/tests/unit/utils/test_path_join.py +++ b/tests/unit/utils/test_path.py @@ -3,11 +3,11 @@ :codeauthor: :email:`Pedro Algarvio (pedro@algarvio.me)` - tests.unit.utils.path_join_test + tests.unit.utils.salt.utils.path.join_test ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ''' -# Import python libs +# Import Python libs from __future__ import absolute_import import os import sys @@ -17,14 +17,15 @@ import platform import tempfile # Import Salt Testing libs -import salt.utils from tests.support.unit import TestCase, skipIf +from tests.support.mock import patch, NO_MOCK, NO_MOCK_REASON -# Import salt libs -from salt.utils import path_join +# Import Salt libs +import salt.utils.path +import salt.utils.platform # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six class PathJoinTestCase(TestCase): @@ -52,10 +53,10 @@ class PathJoinTestCase(TestCase): def test_nix_paths(self): if platform.system().lower() == "windows": self.skipTest( - "Windows platform found. not running *nix path_join tests" + "Windows platform found. not running *nix salt.utils.path.join tests" ) for idx, (parts, expected) in enumerate(self.NIX_PATHS): - path = path_join(*parts) + path = salt.utils.path.join(*parts) self.assertEqual( '{0}: {1}'.format(idx, path), '{0}: {1}'.format(idx, expected) @@ -66,11 +67,11 @@ class PathJoinTestCase(TestCase): if platform.system().lower() != "windows": self.skipTest( 'Non windows platform found. not running non patched os.path ' - 'path_join tests' + 'salt.utils.path.join tests' ) for idx, (parts, expected) in enumerate(self.WIN_PATHS): - path = path_join(*parts) + path = salt.utils.path.join(*parts) self.assertEqual( '{0}: {1}'.format(idx, path), '{0}: {1}'.format(idx, expected) @@ -81,13 +82,13 @@ class PathJoinTestCase(TestCase): if platform.system().lower() == "windows": self.skipTest( 'Windows platform found. not running patched os.path ' - 'path_join tests' + 'salt.utils.path.join tests' ) self.__patch_path() for idx, (parts, expected) in enumerate(self.WIN_PATHS): - path = path_join(*parts) + path = salt.utils.path.join(*parts) self.assertEqual( '{0}: {1}'.format(idx, path), '{0}: {1}'.format(idx, expected) @@ -95,7 +96,7 @@ class PathJoinTestCase(TestCase): self.__unpatch_path() - @skipIf(salt.utils.is_windows(), '*nix-only test') + @skipIf(salt.utils.platform.is_windows(), '*nix-only test') def test_mixed_unicode_and_binary(self): ''' This tests joining paths that contain a mix of components with unicode @@ -109,7 +110,7 @@ class PathJoinTestCase(TestCase): a = u'/foo/bar' b = 'Д' expected = u'/foo/bar/\u0414' - actual = path_join(a, b) + actual = salt.utils.path.join(a, b) self.assertEqual(actual, expected) def __patch_path(self): @@ -136,3 +137,22 @@ class PathJoinTestCase(TestCase): for module in (posixpath, os, os.path, tempfile, platform): reload(module) + + +@skipIf(NO_MOCK, NO_MOCK_REASON) +class WhichTestCase(TestCase): + def test_which_bin(self): + ret = salt.utils.path.which_bin('str') + self.assertIs(None, ret) + + test_exes = ['ls', 'echo'] + with patch('salt.utils.path.which', return_value='/tmp/dummy_path'): + ret = salt.utils.path.which_bin(test_exes) + self.assertEqual(ret, '/tmp/dummy_path') + + ret = salt.utils.path.which_bin([]) + self.assertIs(None, ret) + + with patch('salt.utils.path.which', return_value=''): + ret = salt.utils.path.which_bin(test_exes) + self.assertIs(None, ret) diff --git a/tests/unit/utils/test_process.py b/tests/unit/utils/test_process.py index 72215cd98a6..25be0b7a1b7 100644 --- a/tests/unit/utils/test_process.py +++ b/tests/unit/utils/test_process.py @@ -16,7 +16,7 @@ import salt.utils import salt.utils.process # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six from salt.ext.six.moves import range # pylint: disable=import-error,redefined-builtin diff --git a/tests/unit/utils/test_reactor.py b/tests/unit/utils/test_reactor.py index 7a969009771..765b11d80ca 100644 --- a/tests/unit/utils/test_reactor.py +++ b/tests/unit/utils/test_reactor.py @@ -8,7 +8,7 @@ import os from contextlib import contextmanager -import salt.utils +import salt.utils.files from salt.utils.process import clean_proc import salt.utils.reactor as reactor @@ -45,7 +45,7 @@ class TestReactor(TestCase, AdaptedConfigurationTestCaseMixin): self.opts = self.get_temp_config('master') self.tempdir = tempfile.mkdtemp(dir=TMP) self.sls_name = os.path.join(self.tempdir, 'test.sls') - with salt.utils.fopen(self.sls_name, 'w') as fh: + with salt.utils.files.fopen(self.sls_name, 'w') as fh: fh.write(''' update_fileserver: runner.fileserver.update diff --git a/tests/unit/utils/test_schema.py b/tests/unit/utils/test_schema.py index 3319839c366..edf8841b6b6 100644 --- a/tests/unit/utils/test_schema.py +++ b/tests/unit/utils/test_schema.py @@ -13,7 +13,7 @@ import yaml from tests.support.unit import TestCase, skipIf # Import Salt Libs -from salt.utils import schema +import salt.utils.schema as schema from salt.utils.versions import LooseVersion as _LooseVersion # Import 3rd-party libs @@ -136,7 +136,7 @@ class ConfigTestCase(TestCase): def test_optional_requirements_config(self): class BaseRequirements(schema.Schema): - driver = schema.StringItem(default='digital_ocean', format='hidden') + driver = schema.StringItem(default='digitalocean', format='hidden') class SSHKeyFileSchema(schema.Schema): ssh_key_file = schema.StringItem( @@ -149,13 +149,13 @@ class ConfigTestCase(TestCase): ssh_key_names = schema.StringItem( title='SSH Key Names', description='The names of an SSH key being managed on ' - 'Digital Ocean account which will be used to ' + 'DigitalOcean account which will be used to ' 'authenticate on the deployed VMs', ) class Requirements(BaseRequirements): - title = 'Digital Ocean' - description = 'Digital Ocean Cloud VM configuration requirements.' + title = 'DigitalOcean' + description = 'DigitalOcean Cloud VM configuration requirements.' personal_access_token = schema.StringItem( title='Personal Access Token', @@ -174,12 +174,12 @@ class ConfigTestCase(TestCase): expected = { '$schema': 'http://json-schema.org/draft-04/schema#', - 'title': 'Digital Ocean', - 'description': 'Digital Ocean Cloud VM configuration requirements.', + 'title': 'DigitalOcean', + 'description': 'DigitalOcean Cloud VM configuration requirements.', 'type': 'object', 'properties': { 'driver': { - 'default': 'digital_ocean', + 'default': 'digitalocean', 'format': 'hidden', 'type': 'string', 'title': 'driver' @@ -198,9 +198,8 @@ class ConfigTestCase(TestCase): }, 'ssh_key_names': { 'type': 'string', - 'description': 'The names of an SSH key being managed on Digital ' - 'Ocean account which will be used to authenticate ' - 'on the deployed VMs', + 'description': 'The names of an SSH key being managed on DigitalOcean ' + 'account which will be used to authenticate on the deployed VMs', 'title': 'SSH Key Names' } }, @@ -222,8 +221,8 @@ class ConfigTestCase(TestCase): self.assertDictEqual(expected, Requirements.serialize()) class Requirements2(BaseRequirements): - title = 'Digital Ocean' - description = 'Digital Ocean Cloud VM configuration requirements.' + title = 'DigitalOcean' + description = 'DigitalOcean Cloud VM configuration requirements.' personal_access_token = schema.StringItem( title='Personal Access Token', @@ -239,7 +238,7 @@ class ConfigTestCase(TestCase): ssh_key_names = schema.StringItem( title='SSH Key Names', description='The names of an SSH key being managed on ' - 'Digital Ocean account which will be used to ' + 'DigitalOcean account which will be used to ' 'authenticate on the deployed VMs') requirements_definition = schema.AnyOfItem( @@ -251,12 +250,12 @@ class ConfigTestCase(TestCase): expected = { '$schema': 'http://json-schema.org/draft-04/schema#', - 'title': 'Digital Ocean', - 'description': 'Digital Ocean Cloud VM configuration requirements.', + 'title': 'DigitalOcean', + 'description': 'DigitalOcean Cloud VM configuration requirements.', 'type': 'object', 'properties': { 'driver': { - 'default': 'digital_ocean', + 'default': 'digitalocean', 'format': 'hidden', 'type': 'string', 'title': 'driver' @@ -275,9 +274,8 @@ class ConfigTestCase(TestCase): }, 'ssh_key_names': { 'type': 'string', - 'description': 'The names of an SSH key being managed on Digital ' - 'Ocean account which will be used to authenticate ' - 'on the deployed VMs', + 'description': 'The names of an SSH key being managed on DigitalOcean ' + 'account which will be used to authenticate on the deployed VMs', 'title': 'SSH Key Names' } }, @@ -299,19 +297,19 @@ class ConfigTestCase(TestCase): self.assertDictContainsSubset(expected, Requirements2.serialize()) class Requirements3(schema.Schema): - title = 'Digital Ocean' - description = 'Digital Ocean Cloud VM configuration requirements.' + title = 'DigitalOcean' + description = 'DigitalOcean Cloud VM configuration requirements.' merge_reqs = Requirements(flatten=True) expected = { '$schema': 'http://json-schema.org/draft-04/schema#', - 'title': 'Digital Ocean', - 'description': 'Digital Ocean Cloud VM configuration requirements.', + 'title': 'DigitalOcean', + 'description': 'DigitalOcean Cloud VM configuration requirements.', 'type': 'object', 'properties': { 'driver': { - 'default': 'digital_ocean', + 'default': 'digitalocean', 'format': 'hidden', 'type': 'string', 'title': 'driver' @@ -330,9 +328,8 @@ class ConfigTestCase(TestCase): }, 'ssh_key_names': { 'type': 'string', - 'description': 'The names of an SSH key being managed on Digital ' - 'Ocean account which will be used to authenticate ' - 'on the deployed VMs', + 'description': 'The names of an SSH key being managed on DigitalOcean ' + 'account which will be used to authenticate on the deployed VMs', 'title': 'SSH Key Names' } }, @@ -354,8 +351,8 @@ class ConfigTestCase(TestCase): self.assertDictContainsSubset(expected, Requirements3.serialize()) class Requirements4(schema.Schema): - title = 'Digital Ocean' - description = 'Digital Ocean Cloud VM configuration requirements.' + title = 'DigitalOcean' + description = 'DigitalOcean Cloud VM configuration requirements.' merge_reqs = Requirements(flatten=True) @@ -367,7 +364,7 @@ class ConfigTestCase(TestCase): ssh_key_names_2 = schema.StringItem( title='SSH Key Names', description='The names of an SSH key being managed on ' - 'Digital Ocean account which will be used to ' + 'DigitalOcean account which will be used to ' 'authenticate on the deployed VMs') requirements_definition_2 = schema.AnyOfItem( @@ -379,12 +376,12 @@ class ConfigTestCase(TestCase): expected = { '$schema': 'http://json-schema.org/draft-04/schema#', - 'title': 'Digital Ocean', - 'description': 'Digital Ocean Cloud VM configuration requirements.', + 'title': 'DigitalOcean', + 'description': 'DigitalOcean Cloud VM configuration requirements.', 'type': 'object', 'properties': { 'driver': { - 'default': 'digital_ocean', + 'default': 'digitalocean', 'format': 'hidden', 'type': 'string', 'title': 'driver' @@ -403,9 +400,8 @@ class ConfigTestCase(TestCase): }, 'ssh_key_names': { 'type': 'string', - 'description': 'The names of an SSH key being managed on Digital ' - 'Ocean account which will be used to authenticate ' - 'on the deployed VMs', + 'description': 'The names of an SSH key being managed on DigitalOcean ' + 'account which will be used to authenticate on the deployed VMs', 'title': 'SSH Key Names' }, 'ssh_key_file_2': { @@ -416,9 +412,8 @@ class ConfigTestCase(TestCase): }, 'ssh_key_names_2': { 'type': 'string', - 'description': 'The names of an SSH key being managed on Digital ' - 'Ocean account which will be used to authenticate ' - 'on the deployed VMs', + 'description': 'The names of an SSH key being managed on DigitalOcean ' + 'account which will be used to authenticate on the deployed VMs', 'title': 'SSH Key Names' } }, @@ -446,7 +441,7 @@ class ConfigTestCase(TestCase): @skipIf(HAS_JSONSCHEMA is False, 'The \'jsonschema\' library is missing') def test_optional_requirements_config_validation(self): class BaseRequirements(schema.Schema): - driver = schema.StringItem(default='digital_ocean', format='hidden') + driver = schema.StringItem(default='digitalocean', format='hidden') class SSHKeyFileSchema(schema.Schema): ssh_key_file = schema.StringItem( @@ -462,8 +457,8 @@ class ConfigTestCase(TestCase): 'authenticate on the deployed VMs') class Requirements(BaseRequirements): - title = 'Digital Ocean' - description = 'Digital Ocean Cloud VM configuration requirements.' + title = 'DigitalOcean' + description = 'DigitalOcean Cloud VM configuration requirements.' personal_access_token = schema.StringItem( title='Personal Access Token', diff --git a/tests/unit/utils/test_state.py b/tests/unit/utils/test_state.py new file mode 100644 index 00000000000..18a7c2c9af4 --- /dev/null +++ b/tests/unit/utils/test_state.py @@ -0,0 +1,688 @@ +# -*- coding: utf-8 -*- +''' +Unit Tests for functions located in salt.utils.state.py. +''' + +# Import python libs +from __future__ import absolute_import +import copy +import textwrap + +# Import Salt libs +from salt.ext import six +import salt.utils.odict +import salt.utils.state + +# Import Salt Testing libs +from tests.support.unit import TestCase + + +class StateUtilTestCase(TestCase): + ''' + Test case for state util. + ''' + def test_check_result(self): + self.assertFalse(salt.utils.state.check_result(None), + 'Failed to handle None as an invalid data type.') + self.assertFalse(salt.utils.state.check_result([]), + 'Failed to handle an invalid data type.') + self.assertFalse(salt.utils.state.check_result({}), + 'Failed to handle an empty dictionary.') + self.assertFalse(salt.utils.state.check_result({'host1': []}), + 'Failed to handle an invalid host data structure.') + test_valid_state = {'host1': {'test_state': {'result': 'We have liftoff!'}}} + self.assertTrue(salt.utils.state.check_result(test_valid_state)) + test_valid_false_states = { + 'test1': salt.utils.odict.OrderedDict([ + ('host1', + salt.utils.odict.OrderedDict([ + ('test_state0', {'result': True}), + ('test_state', {'result': False}), + ])), + ]), + 'test2': salt.utils.odict.OrderedDict([ + ('host1', + salt.utils.odict.OrderedDict([ + ('test_state0', {'result': True}), + ('test_state', {'result': True}), + ])), + ('host2', + salt.utils.odict.OrderedDict([ + ('test_state0', {'result': True}), + ('test_state', {'result': False}), + ])), + ]), + 'test3': ['a'], + 'test4': salt.utils.odict.OrderedDict([ + ('asup', salt.utils.odict.OrderedDict([ + ('host1', + salt.utils.odict.OrderedDict([ + ('test_state0', {'result': True}), + ('test_state', {'result': True}), + ])), + ('host2', + salt.utils.odict.OrderedDict([ + ('test_state0', {'result': True}), + ('test_state', {'result': False}), + ])) + ])) + ]), + 'test5': salt.utils.odict.OrderedDict([ + ('asup', salt.utils.odict.OrderedDict([ + ('host1', + salt.utils.odict.OrderedDict([ + ('test_state0', {'result': True}), + ('test_state', {'result': True}), + ])), + ('host2', salt.utils.odict.OrderedDict([])) + ])) + ]) + } + for test, data in six.iteritems(test_valid_false_states): + self.assertFalse( + salt.utils.state.check_result(data), + msg='{0} failed'.format(test)) + test_valid_true_states = { + 'test1': salt.utils.odict.OrderedDict([ + ('host1', + salt.utils.odict.OrderedDict([ + ('test_state0', {'result': True}), + ('test_state', {'result': True}), + ])), + ]), + 'test3': salt.utils.odict.OrderedDict([ + ('host1', + salt.utils.odict.OrderedDict([ + ('test_state0', {'result': True}), + ('test_state', {'result': True}), + ])), + ('host2', + salt.utils.odict.OrderedDict([ + ('test_state0', {'result': True}), + ('test_state', {'result': True}), + ])), + ]), + 'test4': salt.utils.odict.OrderedDict([ + ('asup', salt.utils.odict.OrderedDict([ + ('host1', + salt.utils.odict.OrderedDict([ + ('test_state0', {'result': True}), + ('test_state', {'result': True}), + ])), + ('host2', + salt.utils.odict.OrderedDict([ + ('test_state0', {'result': True}), + ('test_state', {'result': True}), + ])) + ])) + ]), + 'test2': salt.utils.odict.OrderedDict([ + ('host1', + salt.utils.odict.OrderedDict([ + ('test_state0', {'result': None}), + ('test_state', {'result': True}), + ])), + ('host2', + salt.utils.odict.OrderedDict([ + ('test_state0', {'result': True}), + ('test_state', {'result': 'abc'}), + ])) + ]) + } + for test, data in six.iteritems(test_valid_true_states): + self.assertTrue( + salt.utils.state.check_result(data), + msg='{0} failed'.format(test)) + test_invalid_true_ht_states = { + 'test_onfail_simple2': ( + salt.utils.odict.OrderedDict([ + ('host1', + salt.utils.odict.OrderedDict([ + ('test_vstate0', {'result': False}), + ('test_vstate1', {'result': True}), + ])), + ]), + { + 'test_vstate0': { + '__env__': 'base', + '__sls__': u'a', + 'cmd': [salt.utils.odict.OrderedDict([('name', '/bin/true')]), + 'run', + {'order': 10002}]}, + 'test_vstate1': { + '__env__': 'base', + '__sls__': u'a', + 'cmd': [salt.utils.odict.OrderedDict([('name', '/bin/true')]), + salt.utils.odict.OrderedDict([ + ('onfail_stop', True), + ('onfail', + [salt.utils.odict.OrderedDict([('cmd', 'test_vstate0')])]) + ]), + 'run', + {'order': 10004}]}, + } + ), + 'test_onfail_integ2': ( + salt.utils.odict.OrderedDict([ + ('host1', + salt.utils.odict.OrderedDict([ + ('t_|-test_ivstate0_|-echo_|-run', { + 'result': False}), + ('cmd_|-test_ivstate0_|-echo_|-run', { + 'result': False}), + ('cmd_|-test_ivstate1_|-echo_|-run', { + 'result': False}), + ])), + ]), + { + 'test_ivstate0': { + '__env__': 'base', + '__sls__': u'a', + 'cmd': [salt.utils.odict.OrderedDict([('name', '/bin/true')]), + 'run', + {'order': 10002}], + 't': [salt.utils.odict.OrderedDict([('name', '/bin/true')]), + 'run', + {'order': 10002}]}, + 'test_ivstate1': { + '__env__': 'base', + '__sls__': u'a', + 'cmd': [salt.utils.odict.OrderedDict([('name', '/bin/true')]), + salt.utils.odict.OrderedDict([ + ('onfail_stop', False), + ('onfail', + [salt.utils.odict.OrderedDict([('cmd', 'test_ivstate0')])]) + ]), + 'run', + {'order': 10004}]}, + } + ), + 'test_onfail_integ3': ( + salt.utils.odict.OrderedDict([ + ('host1', + salt.utils.odict.OrderedDict([ + ('t_|-test_ivstate0_|-echo_|-run', { + 'result': True}), + ('cmd_|-test_ivstate0_|-echo_|-run', { + 'result': False}), + ('cmd_|-test_ivstate1_|-echo_|-run', { + 'result': False}), + ])), + ]), + { + 'test_ivstate0': { + '__env__': 'base', + '__sls__': u'a', + 'cmd': [salt.utils.odict.OrderedDict([('name', '/bin/true')]), + 'run', + {'order': 10002}], + 't': [salt.utils.odict.OrderedDict([('name', '/bin/true')]), + 'run', + {'order': 10002}]}, + 'test_ivstate1': { + '__env__': 'base', + '__sls__': u'a', + 'cmd': [salt.utils.odict.OrderedDict([('name', '/bin/true')]), + salt.utils.odict.OrderedDict([ + ('onfail_stop', False), + ('onfail', + [salt.utils.odict.OrderedDict([('cmd', 'test_ivstate0')])]) + ]), + 'run', + {'order': 10004}]}, + } + ), + 'test_onfail_integ4': ( + salt.utils.odict.OrderedDict([ + ('host1', + salt.utils.odict.OrderedDict([ + ('t_|-test_ivstate0_|-echo_|-run', { + 'result': False}), + ('cmd_|-test_ivstate0_|-echo_|-run', { + 'result': False}), + ('cmd_|-test_ivstate1_|-echo_|-run', { + 'result': True}), + ])), + ]), + { + 'test_ivstate0': { + '__env__': 'base', + '__sls__': u'a', + 'cmd': [salt.utils.odict.OrderedDict([('name', '/bin/true')]), + 'run', + {'order': 10002}], + 't': [salt.utils.odict.OrderedDict([('name', '/bin/true')]), + 'run', + {'order': 10002}]}, + 'test_ivstate1': { + '__env__': 'base', + '__sls__': u'a', + 'cmd': [salt.utils.odict.OrderedDict([('name', '/bin/true')]), + salt.utils.odict.OrderedDict([ + ('onfail_stop', False), + ('onfail', + [salt.utils.odict.OrderedDict([('cmd', 'test_ivstate0')])]) + ]), + 'run', + {'order': 10004}]}, + 'test_ivstate2': { + '__env__': 'base', + '__sls__': u'a', + 'cmd': [salt.utils.odict.OrderedDict([('name', '/bin/true')]), + salt.utils.odict.OrderedDict([ + ('onfail_stop', True), + ('onfail', + [salt.utils.odict.OrderedDict([('cmd', 'test_ivstate0')])]) + ]), + 'run', + {'order': 10004}]}, + } + ), + 'test_onfail': ( + salt.utils.odict.OrderedDict([ + ('host1', + salt.utils.odict.OrderedDict([ + ('test_state0', {'result': False}), + ('test_state', {'result': True}), + ])), + ]), + None + ), + 'test_onfail_d': ( + salt.utils.odict.OrderedDict([ + ('host1', + salt.utils.odict.OrderedDict([ + ('test_state0', {'result': False}), + ('test_state', {'result': True}), + ])), + ]), + {} + ) + } + for test, testdata in six.iteritems(test_invalid_true_ht_states): + data, ht = testdata + for t_ in [a for a in data['host1']]: + tdata = data['host1'][t_] + if '_|-' in t_: + t_ = t_.split('_|-')[1] + tdata['__id__'] = t_ + self.assertFalse( + salt.utils.state.check_result(data, highstate=ht), + msg='{0} failed'.format(test)) + + test_valid_true_ht_states = { + 'test_onfail_integ': ( + salt.utils.odict.OrderedDict([ + ('host1', + salt.utils.odict.OrderedDict([ + ('cmd_|-test_ivstate0_|-echo_|-run', { + 'result': False}), + ('cmd_|-test_ivstate1_|-echo_|-run', { + 'result': True}), + ])), + ]), + { + 'test_ivstate0': { + '__env__': 'base', + '__sls__': u'a', + 'cmd': [salt.utils.odict.OrderedDict([('name', '/bin/true')]), + 'run', + {'order': 10002}]}, + 'test_ivstate1': { + '__env__': 'base', + '__sls__': u'a', + 'cmd': [salt.utils.odict.OrderedDict([('name', '/bin/true')]), + salt.utils.odict.OrderedDict([ + ('onfail_stop', False), + ('onfail', + [salt.utils.odict.OrderedDict([('cmd', 'test_ivstate0')])]) + ]), + 'run', + {'order': 10004}]}, + } + ), + 'test_onfail_intega3': ( + salt.utils.odict.OrderedDict([ + ('host1', + salt.utils.odict.OrderedDict([ + ('t_|-test_ivstate0_|-echo_|-run', { + 'result': True}), + ('cmd_|-test_ivstate0_|-echo_|-run', { + 'result': False}), + ('cmd_|-test_ivstate1_|-echo_|-run', { + 'result': True}), + ])), + ]), + { + 'test_ivstate0': { + '__env__': 'base', + '__sls__': u'a', + 'cmd': [salt.utils.odict.OrderedDict([('name', '/bin/true')]), + 'run', + {'order': 10002}], + 't': [salt.utils.odict.OrderedDict([('name', '/bin/true')]), + 'run', + {'order': 10002}]}, + 'test_ivstate1': { + '__env__': 'base', + '__sls__': u'a', + 'cmd': [salt.utils.odict.OrderedDict([('name', '/bin/true')]), + salt.utils.odict.OrderedDict([ + ('onfail_stop', False), + ('onfail', + [salt.utils.odict.OrderedDict([('cmd', 'test_ivstate0')])]) + ]), + 'run', + {'order': 10004}]}, + } + ), + 'test_onfail_simple': ( + salt.utils.odict.OrderedDict([ + ('host1', + salt.utils.odict.OrderedDict([ + ('test_vstate0', {'result': False}), + ('test_vstate1', {'result': True}), + ])), + ]), + { + 'test_vstate0': { + '__env__': 'base', + '__sls__': u'a', + 'cmd': [salt.utils.odict.OrderedDict([('name', '/bin/true')]), + 'run', + {'order': 10002}]}, + 'test_vstate1': { + '__env__': 'base', + '__sls__': u'a', + 'cmd': [salt.utils.odict.OrderedDict([('name', '/bin/true')]), + salt.utils.odict.OrderedDict([ + ('onfail_stop', False), + ('onfail', + [salt.utils.odict.OrderedDict([('cmd', 'test_vstate0')])]) + ]), + 'run', + {'order': 10004}]}, + } + ), # order is different + 'test_onfail_simple_rev': ( + salt.utils.odict.OrderedDict([ + ('host1', + salt.utils.odict.OrderedDict([ + ('test_vstate0', {'result': False}), + ('test_vstate1', {'result': True}), + ])), + ]), + { + 'test_vstate0': { + '__env__': 'base', + '__sls__': u'a', + 'cmd': [salt.utils.odict.OrderedDict([('name', '/bin/true')]), + 'run', + {'order': 10002}]}, + 'test_vstate1': { + '__env__': 'base', + '__sls__': u'a', + 'cmd': [salt.utils.odict.OrderedDict([('name', '/bin/true')]), + salt.utils.odict.OrderedDict([ + ('onfail', + [salt.utils.odict.OrderedDict([('cmd', 'test_vstate0')])]) + ]), + salt.utils.odict.OrderedDict([('onfail_stop', False)]), + 'run', + {'order': 10004}]}, + } + ) + } + for test, testdata in six.iteritems(test_valid_true_ht_states): + data, ht = testdata + for t_ in [a for a in data['host1']]: + tdata = data['host1'][t_] + if '_|-' in t_: + t_ = t_.split('_|-')[1] + tdata['__id__'] = t_ + self.assertTrue( + salt.utils.state.check_result(data, highstate=ht), + msg='{0} failed'.format(test)) + test_valid_false_state = {'host1': {'test_state': {'result': False}}} + self.assertFalse(salt.utils.state.check_result(test_valid_false_state)) + + +class UtilStateMergeSubreturnTestcase(TestCase): + ''' + Test cases for salt.utils.state.merge_subreturn function. + ''' + main_ret = { + 'name': 'primary', + # result may be missing, as primarysalt.utils.state is still in progress + 'comment': '', + 'changes': {}, + } + sub_ret = { + 'name': 'secondary', + 'result': True, + 'comment': '', + 'changes': {}, + } + + def test_merge_result(self): + # result not created if not needed + for no_effect_result in [True, None]: + m = copy.deepcopy(self.main_ret) + s = copy.deepcopy(self.sub_ret) + s['result'] = no_effect_result + res = salt.utils.state.merge_subreturn(m, s) + self.assertNotIn('result', res) + + # False subresult is propagated to existing result + for original_result in [True, None, False]: + m = copy.deepcopy(self.main_ret) + m['result'] = original_result + s = copy.deepcopy(self.sub_ret) + s['result'] = False + res = salt.utils.state.merge_subreturn(m, s) + self.assertFalse(res['result']) + + # False result cannot be overriden + for any_result in [True, None, False]: + m = copy.deepcopy(self.main_ret) + m['result'] = False + s = copy.deepcopy(self.sub_ret) + s['result'] = any_result + res = salt.utils.state.merge_subreturn(m, s) + self.assertFalse(res['result']) + + def test_merge_changes(self): + # The main changes dict should always already exist, + # and there should always be a changes dict in the secondary. + primary_changes = {'old': None, 'new': 'my_resource'} + secondary_changes = {'old': None, 'new': ['alarm-1', 'alarm-2']} + + # No changes case + m = copy.deepcopy(self.main_ret) + s = copy.deepcopy(self.sub_ret) + res = salt.utils.state.merge_subreturn(m, s) + self.assertDictEqual(res['changes'], {}) + + # New changes don't get rid of existing changes + m = copy.deepcopy(self.main_ret) + m['changes'] = copy.deepcopy(primary_changes) + s = copy.deepcopy(self.sub_ret) + s['changes'] = copy.deepcopy(secondary_changes) + res = salt.utils.state.merge_subreturn(m, s) + self.assertDictEqual(res['changes'], { + 'old': None, + 'new': 'my_resource', + 'secondary': secondary_changes, + }) + + # The subkey parameter is respected + m = copy.deepcopy(self.main_ret) + m['changes'] = copy.deepcopy(primary_changes) + s = copy.deepcopy(self.sub_ret) + s['changes'] = copy.deepcopy(secondary_changes) + res = salt.utils.state.merge_subreturn(m, s, subkey='alarms') + self.assertDictEqual(res['changes'], { + 'old': None, + 'new': 'my_resource', + 'alarms': secondary_changes, + }) + + def test_merge_pchanges(self): + primary_pchanges = {'old': None, 'new': 'my_resource'} + secondary_pchanges = {'old': None, 'new': ['alarm-1', 'alarm-2']} + + # Neither main nor sub pchanges case + m = copy.deepcopy(self.main_ret) + s = copy.deepcopy(self.sub_ret) + res = salt.utils.state.merge_subreturn(m, s) + self.assertNotIn('pchanges', res) + + # No main pchanges, sub pchanges + m = copy.deepcopy(self.main_ret) + s = copy.deepcopy(self.sub_ret) + s['pchanges'] = copy.deepcopy(secondary_pchanges) + res = salt.utils.state.merge_subreturn(m, s) + self.assertDictEqual(res['pchanges'], { + 'secondary': secondary_pchanges + }) + + # Main pchanges, no sub pchanges + m = copy.deepcopy(self.main_ret) + m['pchanges'] = copy.deepcopy(primary_pchanges) + s = copy.deepcopy(self.sub_ret) + res = salt.utils.state.merge_subreturn(m, s) + self.assertDictEqual(res['pchanges'], primary_pchanges) + + # Both main and sub pchanges, new pchanges don't affect existing ones + m = copy.deepcopy(self.main_ret) + m['pchanges'] = copy.deepcopy(primary_pchanges) + s = copy.deepcopy(self.sub_ret) + s['pchanges'] = copy.deepcopy(secondary_pchanges) + res = salt.utils.state.merge_subreturn(m, s) + self.assertDictEqual(res['pchanges'], { + 'old': None, + 'new': 'my_resource', + 'secondary': secondary_pchanges, + }) + + # The subkey parameter is respected + m = copy.deepcopy(self.main_ret) + m['pchanges'] = copy.deepcopy(primary_pchanges) + s = copy.deepcopy(self.sub_ret) + s['pchanges'] = copy.deepcopy(secondary_pchanges) + res = salt.utils.state.merge_subreturn(m, s, subkey='alarms') + self.assertDictEqual(res['pchanges'], { + 'old': None, + 'new': 'my_resource', + 'alarms': secondary_pchanges, + }) + + def test_merge_comments(self): + main_comment_1 = 'First primary comment.' + main_comment_2 = 'Second primary comment.' + sub_comment_1 = 'First secondary comment,\nwhich spans two lines.' + sub_comment_2 = 'Second secondary comment: {0}'.format( + 'some error\n And a traceback', + ) + final_comment = textwrap.dedent('''\ + First primary comment. + Second primary comment. + First secondary comment, + which spans two lines. + Second secondary comment: some error + And a traceback + '''.rstrip()) + + # Joining two strings + m = copy.deepcopy(self.main_ret) + m['comment'] = main_comment_1 + u'\n' + main_comment_2 + s = copy.deepcopy(self.sub_ret) + s['comment'] = sub_comment_1 + u'\n' + sub_comment_2 + res = salt.utils.state.merge_subreturn(m, s) + self.assertMultiLineEqual(res['comment'], final_comment) + + # Joining string and a list + m = copy.deepcopy(self.main_ret) + m['comment'] = main_comment_1 + u'\n' + main_comment_2 + s = copy.deepcopy(self.sub_ret) + s['comment'] = [sub_comment_1, sub_comment_2] + res = salt.utils.state.merge_subreturn(m, s) + self.assertMultiLineEqual(res['comment'], final_comment) + + # For tests where output is a list, + # also test that final joined output will match + # Joining list and a string + m = copy.deepcopy(self.main_ret) + m['comment'] = [main_comment_1, main_comment_2] + s = copy.deepcopy(self.sub_ret) + s['comment'] = sub_comment_1 + u'\n' + sub_comment_2 + res = salt.utils.state.merge_subreturn(m, s) + self.assertEqual(res['comment'], [ + main_comment_1, + main_comment_2, + sub_comment_1 + u'\n' + sub_comment_2, + ]) + self.assertMultiLineEqual(u'\n'.join(res['comment']), final_comment) + + # Joining two lists + m = copy.deepcopy(self.main_ret) + m['comment'] = [main_comment_1, main_comment_2] + s = copy.deepcopy(self.sub_ret) + s['comment'] = [sub_comment_1, sub_comment_2] + res = salt.utils.state.merge_subreturn(m, s) + self.assertEqual(res['comment'], [ + main_comment_1, + main_comment_2, + sub_comment_1, + sub_comment_2, + ]) + self.assertMultiLineEqual(u'\n'.join(res['comment']), final_comment) + + def test_merge_empty_comments(self): + # Since the primarysalt.utils.state is in progress, + # the main comment may be empty, either '' or []. + # Note that [''] is a degenerate case and should never happen, + # hence the behavior is left unspecified in that case. + # The secondary comment should never be empty, + # because thatsalt.utils.state has already returned, + # so we leave the behavior unspecified in that case. + sub_comment_1 = 'Secondary comment about changes:' + sub_comment_2 = 'A diff that goes with the previous comment' + # No contributions from primary + final_comment = sub_comment_1 + u'\n' + sub_comment_2 + + # Joining empty string and a string + m = copy.deepcopy(self.main_ret) + m['comment'] = '' + s = copy.deepcopy(self.sub_ret) + s['comment'] = sub_comment_1 + u'\n' + sub_comment_2 + res = salt.utils.state.merge_subreturn(m, s) + self.assertEqual(res['comment'], final_comment) + + # Joining empty string and a list + m = copy.deepcopy(self.main_ret) + m['comment'] = '' + s = copy.deepcopy(self.sub_ret) + s['comment'] = [sub_comment_1, sub_comment_2] + res = salt.utils.state.merge_subreturn(m, s) + self.assertEqual(res['comment'], final_comment) + + # For tests where output is a list, + # also test that final joined output will match + # Joining empty list and a string + m = copy.deepcopy(self.main_ret) + m['comment'] = [] + s = copy.deepcopy(self.sub_ret) + s['comment'] = sub_comment_1 + u'\n' + sub_comment_2 + res = salt.utils.state.merge_subreturn(m, s) + self.assertEqual(res['comment'], [final_comment]) + self.assertEqual(u'\n'.join(res['comment']), final_comment) + + # Joining empty list and a list + m = copy.deepcopy(self.main_ret) + m['comment'] = [] + s = copy.deepcopy(self.sub_ret) + s['comment'] = [sub_comment_1, sub_comment_2] + res = salt.utils.state.merge_subreturn(m, s) + self.assertEqual(res['comment'], [sub_comment_1, sub_comment_2]) + self.assertEqual(u'\n'.join(res['comment']), final_comment) diff --git a/tests/unit/utils/test_stringutils.py b/tests/unit/utils/test_stringutils.py new file mode 100644 index 00000000000..4c9249b7f4c --- /dev/null +++ b/tests/unit/utils/test_stringutils.py @@ -0,0 +1,99 @@ +# -*- coding: utf-8 -*- + +from __future__ import absolute_import + +# Import Salt libs +from tests.support.unit import TestCase +from tests.unit.utils.test_utils import LOREM_IPSUM +import salt.utils.stringutils + +# Import 3rd-party libs +from salt.ext import six +from salt.ext.six.moves import range # pylint: disable=redefined-builtin + + +class StringutilsTestCase(TestCase): + def test_contains_whitespace(self): + does_contain_whitespace = 'A brown fox jumped over the red hen.' + does_not_contain_whitespace = 'Abrownfoxjumpedovertheredhen.' + + self.assertTrue(salt.utils.stringutils.contains_whitespace(does_contain_whitespace)) + self.assertFalse(salt.utils.stringutils.contains_whitespace(does_not_contain_whitespace)) + + def test_to_num(self): + self.assertEqual(7, salt.utils.stringutils.to_num('7')) + self.assertIsInstance(salt.utils.stringutils.to_num('7'), int) + self.assertEqual(7, salt.utils.stringutils.to_num('7.0')) + self.assertIsInstance(salt.utils.stringutils.to_num('7.0'), float) + self.assertEqual(salt.utils.stringutils.to_num('Seven'), 'Seven') + self.assertIsInstance(salt.utils.stringutils.to_num('Seven'), str) + + def test_is_binary(self): + self.assertFalse(salt.utils.stringutils.is_binary(LOREM_IPSUM)) + + zero_str = '{0}{1}'.format(LOREM_IPSUM, '\0') + self.assertTrue(salt.utils.stringutils.is_binary(zero_str)) + + # To to ensure safe exit if str passed doesn't evaluate to True + self.assertFalse(salt.utils.stringutils.is_binary('')) + + nontext = 3 * (''.join([chr(x) for x in range(1, 32) if x not in (8, 9, 10, 12, 13)])) + almost_bin_str = '{0}{1}'.format(LOREM_IPSUM[:100], nontext[:42]) + self.assertFalse(salt.utils.stringutils.is_binary(almost_bin_str)) + + bin_str = almost_bin_str + '\x01' + self.assertTrue(salt.utils.stringutils.is_binary(bin_str)) + + def test_to_str(self): + for x in (123, (1, 2, 3), [1, 2, 3], {1: 23}, None): + self.assertRaises(TypeError, salt.utils.stringutils.to_str, x) + if six.PY3: + self.assertEqual(salt.utils.stringutils.to_str('plugh'), 'plugh') + self.assertEqual(salt.utils.stringutils.to_str('áéíóúý', 'utf-8'), 'áéíóúý') + un = '\u4e2d\u56fd\u8a9e (\u7e41\u4f53)' # pylint: disable=anomalous-unicode-escape-in-string + ut = bytes((0xe4, 0xb8, 0xad, 0xe5, 0x9b, 0xbd, 0xe8, 0xaa, 0x9e, 0x20, 0x28, 0xe7, 0xb9, 0x81, 0xe4, 0xbd, 0x93, 0x29)) + self.assertEqual(salt.utils.stringutils.to_str(ut, 'utf-8'), un) + self.assertEqual(salt.utils.stringutils.to_str(bytearray(ut), 'utf-8'), un) + # Test situation when a minion returns incorrect utf-8 string because of... million reasons + ut2 = b'\x9c' + self.assertEqual(salt.utils.stringutils.to_str(ut2, 'utf-8'), u'\ufffd') + self.assertEqual(salt.utils.stringutils.to_str(bytearray(ut2), 'utf-8'), u'\ufffd') + else: + self.assertEqual(salt.utils.stringutils.to_str('plugh'), 'plugh') + self.assertEqual(salt.utils.stringutils.to_str(u'áéíóúý', 'utf-8'), 'áéíóúý') + un = u'\u4e2d\u56fd\u8a9e (\u7e41\u4f53)' + ut = '\xe4\xb8\xad\xe5\x9b\xbd\xe8\xaa\x9e (\xe7\xb9\x81\xe4\xbd\x93)' + self.assertEqual(salt.utils.stringutils.to_str(un, 'utf-8'), ut) + self.assertEqual(salt.utils.stringutils.to_str(bytearray(ut), 'utf-8'), ut) + + def test_to_bytes(self): + for x in (123, (1, 2, 3), [1, 2, 3], {1: 23}, None): + self.assertRaises(TypeError, salt.utils.stringutils.to_bytes, x) + if six.PY3: + self.assertEqual(salt.utils.stringutils.to_bytes('xyzzy'), b'xyzzy') + ut = bytes((0xe4, 0xb8, 0xad, 0xe5, 0x9b, 0xbd, 0xe8, 0xaa, 0x9e, 0x20, 0x28, 0xe7, 0xb9, 0x81, 0xe4, 0xbd, 0x93, 0x29)) + un = '\u4e2d\u56fd\u8a9e (\u7e41\u4f53)' # pylint: disable=anomalous-unicode-escape-in-string + self.assertEqual(salt.utils.stringutils.to_bytes(ut), ut) + self.assertEqual(salt.utils.stringutils.to_bytes(bytearray(ut)), ut) + self.assertEqual(salt.utils.stringutils.to_bytes(un, 'utf-8'), ut) + else: + self.assertEqual(salt.utils.stringutils.to_bytes('xyzzy'), 'xyzzy') + ut = ''.join([chr(x) for x in (0xe4, 0xb8, 0xad, 0xe5, 0x9b, 0xbd, 0xe8, 0xaa, 0x9e, 0x20, 0x28, 0xe7, 0xb9, 0x81, 0xe4, 0xbd, 0x93, 0x29)]) + un = u'\u4e2d\u56fd\u8a9e (\u7e41\u4f53)' # pylint: disable=anomalous-unicode-escape-in-string + self.assertEqual(salt.utils.stringutils.to_bytes(ut), ut) + self.assertEqual(salt.utils.stringutils.to_bytes(bytearray(ut)), ut) + self.assertEqual(salt.utils.stringutils.to_bytes(un, 'utf-8'), ut) + + def test_to_unicode(self): + if six.PY3: + self.assertEqual(salt.utils.stringutils.to_unicode('plugh'), 'plugh') + self.assertEqual(salt.utils.stringutils.to_unicode('áéíóúý'), 'áéíóúý') + un = '\u4e2d\u56fd\u8a9e (\u7e41\u4f53)' # pylint: disable=anomalous-unicode-escape-in-string + ut = bytes((0xe4, 0xb8, 0xad, 0xe5, 0x9b, 0xbd, 0xe8, 0xaa, 0x9e, 0x20, 0x28, 0xe7, 0xb9, 0x81, 0xe4, 0xbd, 0x93, 0x29)) + self.assertEqual(salt.utils.stringutils.to_unicode(ut, 'utf-8'), un) + self.assertEqual(salt.utils.stringutils.to_unicode(bytearray(ut), 'utf-8'), un) + else: + self.assertEqual(salt.utils.stringutils.to_unicode('xyzzy', 'utf-8'), u'xyzzy') + ut = '\xe4\xb8\xad\xe5\x9b\xbd\xe8\xaa\x9e (\xe7\xb9\x81\xe4\xbd\x93)' + un = u'\u4e2d\u56fd\u8a9e (\u7e41\u4f53)' + self.assertEqual(salt.utils.stringutils.to_unicode(ut, 'utf-8'), un) diff --git a/tests/unit/utils/test_systemd.py b/tests/unit/utils/test_systemd.py index e3ea053ced2..8f3fce96609 100644 --- a/tests/unit/utils/test_systemd.py +++ b/tests/unit/utils/test_systemd.py @@ -10,8 +10,8 @@ from tests.support.unit import TestCase, skipIf from tests.support.mock import Mock, patch, NO_MOCK, NO_MOCK_REASON # Import Salt libs +import salt.utils.systemd as _systemd from salt.exceptions import SaltInvocationError -from salt.utils import systemd as _systemd def _booted_effect(path): diff --git a/tests/unit/utils/test_url.py b/tests/unit/utils/test_url.py index 95caa0253e1..800d8a090d8 100644 --- a/tests/unit/utils/test_url.py +++ b/tests/unit/utils/test_url.py @@ -4,6 +4,7 @@ from __future__ import absolute_import # Import Salt Libs +import salt.utils.platform import salt.utils.url # Import Salt Testing Libs @@ -38,6 +39,8 @@ class UrlTestCase(TestCase): ''' path = '?funny/path with {interesting|chars}' url = 'salt://' + path + if salt.utils.platform.is_windows(): + path = '_funny/path with {interesting_chars}' self.assertEqual(salt.utils.url.parse(url), (path, None)) @@ -48,6 +51,8 @@ class UrlTestCase(TestCase): saltenv = 'ambience' path = '?funny/path&with {interesting|chars}' url = 'salt://' + path + '?saltenv=' + saltenv + if salt.utils.platform.is_windows(): + path = '_funny/path&with {interesting_chars}' self.assertEqual(salt.utils.url.parse(url), (path, saltenv)) @@ -59,6 +64,8 @@ class UrlTestCase(TestCase): ''' path = '? interesting/&path.filetype' url = 'salt://' + path + if salt.utils.platform.is_windows(): + url = 'salt://_ interesting/&path.filetype' self.assertEqual(salt.utils.url.create(path), url) @@ -68,6 +75,8 @@ class UrlTestCase(TestCase): ''' saltenv = 'raumklang' path = '? interesting/&path.filetype' + if salt.utils.platform.is_windows(): + path = '_ interesting/&path.filetype' url = 'salt://' + path + '?saltenv=' + saltenv @@ -81,7 +90,7 @@ class UrlTestCase(TestCase): ''' url = 'salt://dir/file.ini' - with patch('salt.utils.is_windows', MagicMock(return_value=True)): + with patch('salt.utils.platform.is_windows', MagicMock(return_value=True)): self.assertFalse(salt.utils.url.is_escaped(url)) def test_is_escaped_escaped_path(self): @@ -132,7 +141,7 @@ class UrlTestCase(TestCase): ''' url = 'salt://dir/file.ini' - with patch('salt.utils.is_windows', MagicMock(return_value=True)): + with patch('salt.utils.platform.is_windows', MagicMock(return_value=True)): self.assertEqual(salt.utils.url.escape(url), url) def test_escape_escaped_path(self): @@ -149,6 +158,8 @@ class UrlTestCase(TestCase): ''' path = 'dir/file.conf' escaped_path = '|' + path + if salt.utils.platform.is_windows(): + escaped_path = path self.assertEqual(salt.utils.url.escape(path), escaped_path) @@ -167,6 +178,8 @@ class UrlTestCase(TestCase): path = 'dir/file.conf' url = 'salt://' + path escaped_url = 'salt://|' + path + if salt.utils.platform.is_windows(): + escaped_url = url self.assertEqual(salt.utils.url.escape(url), escaped_url) @@ -186,7 +199,7 @@ class UrlTestCase(TestCase): ''' url = 'salt://dir/file.ini' - with patch('salt.utils.is_windows', MagicMock(return_value=True)): + with patch('salt.utils.platform.is_windows', MagicMock(return_value=True)): self.assertEqual(salt.utils.url.unescape(url), url) def test_unescape_escaped_path(self): diff --git a/tests/unit/utils/test_utils.py b/tests/unit/utils/test_utils.py index 948c47a6b8a..f13e9cdf737 100644 --- a/tests/unit/utils/test_utils.py +++ b/tests/unit/utils/test_utils.py @@ -9,35 +9,32 @@ from __future__ import absolute_import # Import Salt Testing libs from tests.support.unit import TestCase, skipIf from tests.support.mock import ( - patch, DEFAULT, - create_autospec, + patch, NO_MOCK, NO_MOCK_REASON ) # Import Salt libs -from salt.utils import args -from salt.utils.odict import OrderedDict -from salt.exceptions import (SaltInvocationError, SaltSystemExit, CommandNotFoundError) -from salt import utils +import salt.utils +import salt.utils.jid +import salt.utils.yamlencoding +import salt.utils.zeromq +from salt.exceptions import SaltSystemExit, CommandNotFoundError # Import Python libraries -import os import datetime +import os import yaml import zmq -from collections import namedtuple # Import 3rd-party libs -import salt.ext.six as six -from salt.ext.six.moves import range try: import timelib # pylint: disable=import-error,unused-import HAS_TIMELIB = True except ImportError: HAS_TIMELIB = False -LORUM_IPSUM = 'Lorem ipsum dolor sit amet, consectetur adipiscing elit. Quisque eget urna a arcu lacinia sagittis. \n' \ +LOREM_IPSUM = 'Lorem ipsum dolor sit amet, consectetur adipiscing elit. Quisque eget urna a arcu lacinia sagittis. \n' \ 'Sed scelerisque, lacus eget malesuada vestibulum, justo diam facilisis tortor, in sodales dolor \n' \ 'nibh eu urna. Aliquam iaculis massa risus, sed elementum risus accumsan id. Suspendisse mattis, \n' \ 'metus sed lacinia dictum, leo orci dapibus sapien, at porttitor sapien nulla ac velit. \n' \ @@ -56,117 +53,55 @@ class UtilsTestCase(TestCase): ' dolor \n' \ '[...]\n' \ '---' - ret = utils.get_context(LORUM_IPSUM, 1, num_lines=1) + ret = salt.utils.get_context(LOREM_IPSUM, 1, num_lines=1) self.assertEqual(ret, expected_context) def test_jid_to_time(self): test_jid = 20131219110700123489 expected_jid = '2013, Dec 19 11:07:00.123489' - self.assertEqual(utils.jid.jid_to_time(test_jid), expected_jid) + self.assertEqual(salt.utils.jid.jid_to_time(test_jid), expected_jid) # Test incorrect lengths incorrect_jid_length = 2012 - self.assertEqual(utils.jid.jid_to_time(incorrect_jid_length), '') + self.assertEqual(salt.utils.jid.jid_to_time(incorrect_jid_length), '') @skipIf(NO_MOCK, NO_MOCK_REASON) def test_gen_mac(self): with patch('random.randint', return_value=1) as random_mock: self.assertEqual(random_mock.return_value, 1) - ret = utils.gen_mac('00:16:3E') + ret = salt.utils.gen_mac('00:16:3E') expected_mac = '00:16:3E:01:01:01' self.assertEqual(ret, expected_mac) def test_mac_str_to_bytes(self): - self.assertRaises(ValueError, utils.mac_str_to_bytes, '31337') - self.assertRaises(ValueError, utils.mac_str_to_bytes, '0001020304056') - self.assertRaises(ValueError, utils.mac_str_to_bytes, '00:01:02:03:04:056') - self.assertRaises(ValueError, utils.mac_str_to_bytes, 'a0:b0:c0:d0:e0:fg') - self.assertEqual(b'\x10\x08\x06\x04\x02\x00', utils.mac_str_to_bytes('100806040200')) - self.assertEqual(b'\xf8\xe7\xd6\xc5\xb4\xa3', utils.mac_str_to_bytes('f8e7d6c5b4a3')) + self.assertRaises(ValueError, salt.utils.mac_str_to_bytes, '31337') + self.assertRaises(ValueError, salt.utils.mac_str_to_bytes, '0001020304056') + self.assertRaises(ValueError, salt.utils.mac_str_to_bytes, '00:01:02:03:04:056') + self.assertRaises(ValueError, salt.utils.mac_str_to_bytes, 'a0:b0:c0:d0:e0:fg') + self.assertEqual(b'\x10\x08\x06\x04\x02\x00', salt.utils.mac_str_to_bytes('100806040200')) + self.assertEqual(b'\xf8\xe7\xd6\xc5\xb4\xa3', salt.utils.mac_str_to_bytes('f8e7d6c5b4a3')) def test_ip_bracket(self): test_ipv4 = '127.0.0.1' test_ipv6 = '::1' - self.assertEqual(test_ipv4, utils.ip_bracket(test_ipv4)) - self.assertEqual('[{0}]'.format(test_ipv6), utils.ip_bracket(test_ipv6)) + self.assertEqual(test_ipv4, salt.utils.ip_bracket(test_ipv4)) + self.assertEqual('[{0}]'.format(test_ipv6), salt.utils.ip_bracket(test_ipv6)) def test_is_jid(self): - self.assertTrue(utils.jid.is_jid('20131219110700123489')) # Valid JID - self.assertFalse(utils.jid.is_jid(20131219110700123489)) # int - self.assertFalse(utils.jid.is_jid('2013121911070012348911111')) # Wrong length - - @skipIf(NO_MOCK, NO_MOCK_REASON) - def test_path_join(self): - with patch('salt.utils.is_windows', return_value=False) as is_windows_mock: - self.assertFalse(is_windows_mock.return_value) - expected_path = '/a/b/c/d' - ret = utils.path_join('/a/b/c', 'd') - self.assertEqual(ret, expected_path) + self.assertTrue(salt.utils.jid.is_jid('20131219110700123489')) # Valid JID + self.assertFalse(salt.utils.jid.is_jid(20131219110700123489)) # int + self.assertFalse(salt.utils.jid.is_jid('2013121911070012348911111')) # Wrong length def test_build_whitespace_split_regex(self): expected_regex = '(?m)^(?:[\\s]+)?Lorem(?:[\\s]+)?ipsum(?:[\\s]+)?dolor(?:[\\s]+)?sit(?:[\\s]+)?amet\\,' \ '(?:[\\s]+)?$' - ret = utils.build_whitespace_split_regex(' '.join(LORUM_IPSUM.split()[:5])) + ret = salt.utils.build_whitespace_split_regex(' '.join(LOREM_IPSUM.split()[:5])) self.assertEqual(ret, expected_regex) - def test_get_function_argspec(self): - def dummy_func(first, second, third, fourth='fifth'): - pass - - expected_argspec = namedtuple('ArgSpec', 'args varargs keywords defaults')( - args=['first', 'second', 'third', 'fourth'], varargs=None, keywords=None, defaults=('fifth',)) - ret = utils.args.get_function_argspec(dummy_func) - - self.assertEqual(ret, expected_argspec) - - def test_arg_lookup(self): - def dummy_func(first, second, third, fourth='fifth'): - pass - - expected_dict = {'args': ['first', 'second', 'third'], 'kwargs': {'fourth': 'fifth'}} - ret = utils.arg_lookup(dummy_func) - self.assertEqual(expected_dict, ret) - - @skipIf(NO_MOCK, NO_MOCK_REASON) - def test_safe_rm(self): - with patch('os.remove') as os_remove_mock: - utils.safe_rm('dummy_tgt') - self.assertTrue(os_remove_mock.called) - - @skipIf(os.path.exists('/tmp/no_way_this_is_a_file_nope.sh'), 'Test file exists! Skipping safe_rm_exceptions test!') - def test_safe_rm_exceptions(self): - error = False - try: - utils.safe_rm('/tmp/no_way_this_is_a_file_nope.sh') - except (IOError, OSError): - error = True - self.assertFalse(error, 'utils.safe_rm raised exception when it should not have') - - @skipIf(NO_MOCK, NO_MOCK_REASON) - def test_format_call(self): - with patch('salt.utils.arg_lookup') as arg_lookup: - def dummy_func(first=None, second=None, third=None): - pass - arg_lookup.return_value = {'args': ['first', 'second', 'third'], 'kwargs': {}} - get_function_argspec = DEFAULT - get_function_argspec.return_value = namedtuple('ArgSpec', 'args varargs keywords defaults')( - args=['first', 'second', 'third', 'fourth'], varargs=None, keywords=None, defaults=('fifth',)) - - # Make sure we raise an error if we don't pass in the requisite number of arguments - self.assertRaises(SaltInvocationError, utils.format_call, dummy_func, {'1': 2}) - - # Make sure we warn on invalid kwargs - ret = utils.format_call(dummy_func, {'first': 2, 'second': 2, 'third': 3}) - self.assertGreaterEqual(len(ret['warnings']), 1) - - ret = utils.format_call(dummy_func, {'first': 2, 'second': 2, 'third': 3}, - expected_extra_kws=('first', 'second', 'third')) - self.assertDictEqual(ret, {'args': [], 'kwargs': {}}) - def test_isorted(self): test_list = ['foo', 'Foo', 'bar', 'Bar'] expected_list = ['bar', 'Bar', 'foo', 'Foo'] - self.assertEqual(utils.isorted(test_list), expected_list) + self.assertEqual(salt.utils.isorted(test_list), expected_list) def test_mysql_to_dict(self): test_mysql_output = ['+----+------+-----------+------+---------+------+-------+------------------+', @@ -175,28 +110,13 @@ class UtilsTestCase(TestCase): '| 7 | root | localhost | NULL | Query | 0 | init | show processlist |', '+----+------+-----------+------+---------+------+-------+------------------+'] - ret = utils.mysql_to_dict(test_mysql_output, 'Info') + ret = salt.utils.mysql_to_dict(test_mysql_output, 'Info') expected_dict = { 'show processlist': {'Info': 'show processlist', 'db': 'NULL', 'State': 'init', 'Host': 'localhost', 'Command': 'Query', 'User': 'root', 'Time': 0, 'Id': 7}} self.assertDictEqual(ret, expected_dict) - def test_contains_whitespace(self): - does_contain_whitespace = 'A brown fox jumped over the red hen.' - does_not_contain_whitespace = 'Abrownfoxjumpedovertheredhen.' - - self.assertTrue(utils.contains_whitespace(does_contain_whitespace)) - self.assertFalse(utils.contains_whitespace(does_not_contain_whitespace)) - - def test_str_to_num(self): - self.assertEqual(7, utils.str_to_num('7')) - self.assertIsInstance(utils.str_to_num('7'), int) - self.assertEqual(7, utils.str_to_num('7.0')) - self.assertIsInstance(utils.str_to_num('7.0'), float) - self.assertEqual(utils.str_to_num('Seven'), 'Seven') - self.assertIsInstance(utils.str_to_num('Seven'), str) - def test_subdict_match(self): test_two_level_dict = {'foo': {'bar': 'baz'}} test_two_level_comb_dict = {'foo': {'bar': 'baz:woz'}} @@ -206,21 +126,21 @@ class UtilsTestCase(TestCase): test_three_level_dict = {'a': {'b': {'c': 'v'}}} self.assertTrue( - utils.subdict_match( + salt.utils.subdict_match( test_two_level_dict, 'foo:bar:baz' ) ) # In test_two_level_comb_dict, 'foo:bar' corresponds to 'baz:woz', not # 'baz'. This match should return False. self.assertFalse( - utils.subdict_match( + salt.utils.subdict_match( test_two_level_comb_dict, 'foo:bar:baz' ) ) # This tests matching with the delimiter in the value part (in other # words, that the path 'foo:bar' corresponds to the string 'baz:woz'). self.assertTrue( - utils.subdict_match( + salt.utils.subdict_match( test_two_level_comb_dict, 'foo:bar:baz:woz' ) ) @@ -228,7 +148,7 @@ class UtilsTestCase(TestCase): # to 'baz:woz:wiz', or if there was more deep nesting. But it does not, # so this should return False. self.assertFalse( - utils.subdict_match( + salt.utils.subdict_match( test_two_level_comb_dict, 'foo:bar:baz:woz:wiz' ) ) @@ -240,7 +160,7 @@ class UtilsTestCase(TestCase): # salt.utils.traverse_list_and_dict() so this particular assertion is a # sanity check. self.assertTrue( - utils.subdict_match( + salt.utils.subdict_match( test_two_level_dict_and_list, 'abc:ghi' ) ) @@ -248,25 +168,25 @@ class UtilsTestCase(TestCase): # list, embedded in a dict. This is a rather absurd case, but it # confirms that match recursion works properly. self.assertTrue( - utils.subdict_match( + salt.utils.subdict_match( test_two_level_dict_and_list, 'abc:lorem:ipsum:dolor:sit' ) ) # Test four level dict match for reference self.assertTrue( - utils.subdict_match( + salt.utils.subdict_match( test_three_level_dict, 'a:b:c:v' ) ) self.assertFalse( # Test regression in 2015.8 where 'a:c:v' would match 'a:b:c:v' - utils.subdict_match( + salt.utils.subdict_match( test_three_level_dict, 'a:c:v' ) ) # Test wildcard match self.assertTrue( - utils.subdict_match( + salt.utils.subdict_match( test_three_level_dict, 'a:*:c:v' ) ) @@ -276,13 +196,13 @@ class UtilsTestCase(TestCase): self.assertDictEqual( {'not_found': 'nope'}, - utils.traverse_dict( + salt.utils.traverse_dict( test_two_level_dict, 'foo:bar:baz', {'not_found': 'nope'} ) ) self.assertEqual( 'baz', - utils.traverse_dict( + salt.utils.traverse_dict( test_two_level_dict, 'foo:bar', {'not_found': 'not_found'} ) ) @@ -298,472 +218,45 @@ class UtilsTestCase(TestCase): # corresponding to the key path foo:bar. self.assertDictEqual( {'not_found': 'nope'}, - utils.traverse_dict_and_list( + salt.utils.traverse_dict_and_list( test_two_level_dict, 'foo:bar:baz', {'not_found': 'nope'} ) ) # Now check to ensure that foo:bar corresponds to baz self.assertEqual( 'baz', - utils.traverse_dict_and_list( + salt.utils.traverse_dict_and_list( test_two_level_dict, 'foo:bar', {'not_found': 'not_found'} ) ) # Check traversing too far self.assertDictEqual( {'not_found': 'nope'}, - utils.traverse_dict_and_list( + salt.utils.traverse_dict_and_list( test_two_level_dict_and_list, 'foo:bar', {'not_found': 'nope'} ) ) # Check index 1 (2nd element) of list corresponding to path 'foo' self.assertEqual( 'baz', - utils.traverse_dict_and_list( + salt.utils.traverse_dict_and_list( test_two_level_dict_and_list, 'foo:1', {'not_found': 'not_found'} ) ) # Traverse a couple times into dicts embedded in lists self.assertEqual( 'sit', - utils.traverse_dict_and_list( + salt.utils.traverse_dict_and_list( test_two_level_dict_and_list, 'foo:lorem:ipsum:dolor', {'not_found': 'not_found'} ) ) - def test_clean_kwargs(self): - self.assertDictEqual(utils.clean_kwargs(foo='bar'), {'foo': 'bar'}) - self.assertDictEqual(utils.clean_kwargs(__pub_foo='bar'), {}) - self.assertDictEqual(utils.clean_kwargs(__foo_bar='gwar'), {}) - self.assertDictEqual(utils.clean_kwargs(foo_bar='gwar'), {'foo_bar': 'gwar'}) - def test_sanitize_win_path_string(self): p = '\\windows\\system' - self.assertEqual(utils.sanitize_win_path_string('\\windows\\system'), '\\windows\\system') - self.assertEqual(utils.sanitize_win_path_string('\\bo:g|us\\p?at*h>'), '\\bo_g_us\\p_at_h_') - - def test_check_state_result(self): - self.assertFalse(utils.check_state_result(None), "Failed to handle None as an invalid data type.") - self.assertFalse(utils.check_state_result([]), "Failed to handle an invalid data type.") - self.assertFalse(utils.check_state_result({}), "Failed to handle an empty dictionary.") - self.assertFalse(utils.check_state_result({'host1': []}), "Failed to handle an invalid host data structure.") - test_valid_state = {'host1': {'test_state': {'result': 'We have liftoff!'}}} - self.assertTrue(utils.check_state_result(test_valid_state)) - test_valid_false_states = { - 'test1': OrderedDict([ - ('host1', - OrderedDict([ - ('test_state0', {'result': True}), - ('test_state', {'result': False}), - ])), - ]), - 'test2': OrderedDict([ - ('host1', - OrderedDict([ - ('test_state0', {'result': True}), - ('test_state', {'result': True}), - ])), - ('host2', - OrderedDict([ - ('test_state0', {'result': True}), - ('test_state', {'result': False}), - ])), - ]), - 'test3': ['a'], - 'test4': OrderedDict([ - ('asup', OrderedDict([ - ('host1', - OrderedDict([ - ('test_state0', {'result': True}), - ('test_state', {'result': True}), - ])), - ('host2', - OrderedDict([ - ('test_state0', {'result': True}), - ('test_state', {'result': False}), - ])) - ])) - ]), - 'test5': OrderedDict([ - ('asup', OrderedDict([ - ('host1', - OrderedDict([ - ('test_state0', {'result': True}), - ('test_state', {'result': True}), - ])), - ('host2', OrderedDict([])) - ])) - ]) - } - for test, data in six.iteritems(test_valid_false_states): - self.assertFalse( - utils.check_state_result(data), - msg='{0} failed'.format(test)) - test_valid_true_states = { - 'test1': OrderedDict([ - ('host1', - OrderedDict([ - ('test_state0', {'result': True}), - ('test_state', {'result': True}), - ])), - ]), - 'test3': OrderedDict([ - ('host1', - OrderedDict([ - ('test_state0', {'result': True}), - ('test_state', {'result': True}), - ])), - ('host2', - OrderedDict([ - ('test_state0', {'result': True}), - ('test_state', {'result': True}), - ])), - ]), - 'test4': OrderedDict([ - ('asup', OrderedDict([ - ('host1', - OrderedDict([ - ('test_state0', {'result': True}), - ('test_state', {'result': True}), - ])), - ('host2', - OrderedDict([ - ('test_state0', {'result': True}), - ('test_state', {'result': True}), - ])) - ])) - ]), - 'test2': OrderedDict([ - ('host1', - OrderedDict([ - ('test_state0', {'result': None}), - ('test_state', {'result': True}), - ])), - ('host2', - OrderedDict([ - ('test_state0', {'result': True}), - ('test_state', {'result': 'abc'}), - ])) - ]) - } - for test, data in six.iteritems(test_valid_true_states): - self.assertTrue( - utils.check_state_result(data), - msg='{0} failed'.format(test)) - test_invalid_true_ht_states = { - 'test_onfail_simple2': ( - OrderedDict([ - ('host1', - OrderedDict([ - ('test_vstate0', {'result': False}), - ('test_vstate1', {'result': True}), - ])), - ]), - { - 'test_vstate0': { - '__env__': 'base', - '__sls__': u'a', - 'cmd': [OrderedDict([('name', '/bin/true')]), - 'run', - {'order': 10002}]}, - 'test_vstate1': { - '__env__': 'base', - '__sls__': u'a', - 'cmd': [OrderedDict([('name', '/bin/true')]), - OrderedDict([ - ('onfail_stop', True), - ('onfail', - [OrderedDict([('cmd', 'test_vstate0')])]) - ]), - 'run', - {'order': 10004}]}, - } - ), - 'test_onfail_integ2': ( - OrderedDict([ - ('host1', - OrderedDict([ - ('t_|-test_ivstate0_|-echo_|-run', { - 'result': False}), - ('cmd_|-test_ivstate0_|-echo_|-run', { - 'result': False}), - ('cmd_|-test_ivstate1_|-echo_|-run', { - 'result': False}), - ])), - ]), - { - 'test_ivstate0': { - '__env__': 'base', - '__sls__': u'a', - 'cmd': [OrderedDict([('name', '/bin/true')]), - 'run', - {'order': 10002}], - 't': [OrderedDict([('name', '/bin/true')]), - 'run', - {'order': 10002}]}, - 'test_ivstate1': { - '__env__': 'base', - '__sls__': u'a', - 'cmd': [OrderedDict([('name', '/bin/true')]), - OrderedDict([ - ('onfail_stop', False), - ('onfail', - [OrderedDict([('cmd', 'test_ivstate0')])]) - ]), - 'run', - {'order': 10004}]}, - } - ), - 'test_onfail_integ3': ( - OrderedDict([ - ('host1', - OrderedDict([ - ('t_|-test_ivstate0_|-echo_|-run', { - 'result': True}), - ('cmd_|-test_ivstate0_|-echo_|-run', { - 'result': False}), - ('cmd_|-test_ivstate1_|-echo_|-run', { - 'result': False}), - ])), - ]), - { - 'test_ivstate0': { - '__env__': 'base', - '__sls__': u'a', - 'cmd': [OrderedDict([('name', '/bin/true')]), - 'run', - {'order': 10002}], - 't': [OrderedDict([('name', '/bin/true')]), - 'run', - {'order': 10002}]}, - 'test_ivstate1': { - '__env__': 'base', - '__sls__': u'a', - 'cmd': [OrderedDict([('name', '/bin/true')]), - OrderedDict([ - ('onfail_stop', False), - ('onfail', - [OrderedDict([('cmd', 'test_ivstate0')])]) - ]), - 'run', - {'order': 10004}]}, - } - ), - 'test_onfail_integ4': ( - OrderedDict([ - ('host1', - OrderedDict([ - ('t_|-test_ivstate0_|-echo_|-run', { - 'result': False}), - ('cmd_|-test_ivstate0_|-echo_|-run', { - 'result': False}), - ('cmd_|-test_ivstate1_|-echo_|-run', { - 'result': True}), - ])), - ]), - { - 'test_ivstate0': { - '__env__': 'base', - '__sls__': u'a', - 'cmd': [OrderedDict([('name', '/bin/true')]), - 'run', - {'order': 10002}], - 't': [OrderedDict([('name', '/bin/true')]), - 'run', - {'order': 10002}]}, - 'test_ivstate1': { - '__env__': 'base', - '__sls__': u'a', - 'cmd': [OrderedDict([('name', '/bin/true')]), - OrderedDict([ - ('onfail_stop', False), - ('onfail', - [OrderedDict([('cmd', 'test_ivstate0')])]) - ]), - 'run', - {'order': 10004}]}, - 'test_ivstate2': { - '__env__': 'base', - '__sls__': u'a', - 'cmd': [OrderedDict([('name', '/bin/true')]), - OrderedDict([ - ('onfail_stop', True), - ('onfail', - [OrderedDict([('cmd', 'test_ivstate0')])]) - ]), - 'run', - {'order': 10004}]}, - } - ), - 'test_onfail': ( - OrderedDict([ - ('host1', - OrderedDict([ - ('test_state0', {'result': False}), - ('test_state', {'result': True}), - ])), - ]), - None - ), - 'test_onfail_d': ( - OrderedDict([ - ('host1', - OrderedDict([ - ('test_state0', {'result': False}), - ('test_state', {'result': True}), - ])), - ]), - {} - ) - } - for test, testdata in six.iteritems(test_invalid_true_ht_states): - data, ht = testdata - for t_ in [a for a in data['host1']]: - tdata = data['host1'][t_] - if '_|-' in t_: - t_ = t_.split('_|-')[1] - tdata['__id__'] = t_ - self.assertFalse( - utils.check_state_result(data, highstate=ht), - msg='{0} failed'.format(test)) - - test_valid_true_ht_states = { - 'test_onfail_integ': ( - OrderedDict([ - ('host1', - OrderedDict([ - ('cmd_|-test_ivstate0_|-echo_|-run', { - 'result': False}), - ('cmd_|-test_ivstate1_|-echo_|-run', { - 'result': True}), - ])), - ]), - { - 'test_ivstate0': { - '__env__': 'base', - '__sls__': u'a', - 'cmd': [OrderedDict([('name', '/bin/true')]), - 'run', - {'order': 10002}]}, - 'test_ivstate1': { - '__env__': 'base', - '__sls__': u'a', - 'cmd': [OrderedDict([('name', '/bin/true')]), - OrderedDict([ - ('onfail_stop', False), - ('onfail', - [OrderedDict([('cmd', 'test_ivstate0')])]) - ]), - 'run', - {'order': 10004}]}, - } - ), - 'test_onfail_intega3': ( - OrderedDict([ - ('host1', - OrderedDict([ - ('t_|-test_ivstate0_|-echo_|-run', { - 'result': True}), - ('cmd_|-test_ivstate0_|-echo_|-run', { - 'result': False}), - ('cmd_|-test_ivstate1_|-echo_|-run', { - 'result': True}), - ])), - ]), - { - 'test_ivstate0': { - '__env__': 'base', - '__sls__': u'a', - 'cmd': [OrderedDict([('name', '/bin/true')]), - 'run', - {'order': 10002}], - 't': [OrderedDict([('name', '/bin/true')]), - 'run', - {'order': 10002}]}, - 'test_ivstate1': { - '__env__': 'base', - '__sls__': u'a', - 'cmd': [OrderedDict([('name', '/bin/true')]), - OrderedDict([ - ('onfail_stop', False), - ('onfail', - [OrderedDict([('cmd', 'test_ivstate0')])]) - ]), - 'run', - {'order': 10004}]}, - } - ), - 'test_onfail_simple': ( - OrderedDict([ - ('host1', - OrderedDict([ - ('test_vstate0', {'result': False}), - ('test_vstate1', {'result': True}), - ])), - ]), - { - 'test_vstate0': { - '__env__': 'base', - '__sls__': u'a', - 'cmd': [OrderedDict([('name', '/bin/true')]), - 'run', - {'order': 10002}]}, - 'test_vstate1': { - '__env__': 'base', - '__sls__': u'a', - 'cmd': [OrderedDict([('name', '/bin/true')]), - OrderedDict([ - ('onfail_stop', False), - ('onfail', - [OrderedDict([('cmd', 'test_vstate0')])]) - ]), - 'run', - {'order': 10004}]}, - } - ), # order is different - 'test_onfail_simple_rev': ( - OrderedDict([ - ('host1', - OrderedDict([ - ('test_vstate0', {'result': False}), - ('test_vstate1', {'result': True}), - ])), - ]), - { - 'test_vstate0': { - '__env__': 'base', - '__sls__': u'a', - 'cmd': [OrderedDict([('name', '/bin/true')]), - 'run', - {'order': 10002}]}, - 'test_vstate1': { - '__env__': 'base', - '__sls__': u'a', - 'cmd': [OrderedDict([('name', '/bin/true')]), - OrderedDict([ - ('onfail', - [OrderedDict([('cmd', 'test_vstate0')])]) - ]), - OrderedDict([('onfail_stop', False)]), - 'run', - {'order': 10004}]}, - } - ) - } - for test, testdata in six.iteritems(test_valid_true_ht_states): - data, ht = testdata - for t_ in [a for a in data['host1']]: - tdata = data['host1'][t_] - if '_|-' in t_: - t_ = t_.split('_|-')[1] - tdata['__id__'] = t_ - self.assertTrue( - utils.check_state_result(data, highstate=ht), - msg='{0} failed'.format(test)) - test_valid_false_state = {'host1': {'test_state': {'result': False}}} - self.assertFalse(utils.check_state_result(test_valid_false_state)) + self.assertEqual(salt.utils.sanitize_win_path_string('\\windows\\system'), '\\windows\\system') + self.assertEqual(salt.utils.sanitize_win_path_string('\\bo:g|us\\p?at*h>'), '\\bo_g_us\\p_at_h_') @skipIf(NO_MOCK, NO_MOCK_REASON) @skipIf(not hasattr(zmq, 'IPC_PATH_MAX_LEN'), "ZMQ does not have max length support.") @@ -772,56 +265,33 @@ class UtilsTestCase(TestCase): Ensure we throw an exception if we have a too-long IPC URI ''' with patch('zmq.IPC_PATH_MAX_LEN', 1): - self.assertRaises(SaltSystemExit, utils.zeromq.check_ipc_path_max_len, '1' * 1024) + self.assertRaises(SaltSystemExit, salt.utils.zeromq.check_ipc_path_max_len, '1' * 1024) def test_test_mode(self): - self.assertTrue(utils.test_mode(test=True)) - self.assertTrue(utils.test_mode(Test=True)) - self.assertTrue(utils.test_mode(tEsT=True)) + self.assertTrue(salt.utils.test_mode(test=True)) + self.assertTrue(salt.utils.test_mode(Test=True)) + self.assertTrue(salt.utils.test_mode(tEsT=True)) def test_option(self): test_two_level_dict = {'foo': {'bar': 'baz'}} - self.assertDictEqual({'not_found': 'nope'}, utils.option('foo:bar', {'not_found': 'nope'})) - self.assertEqual('baz', utils.option('foo:bar', {'not_found': 'nope'}, opts=test_two_level_dict)) - self.assertEqual('baz', utils.option('foo:bar', {'not_found': 'nope'}, pillar={'master': test_two_level_dict})) - self.assertEqual('baz', utils.option('foo:bar', {'not_found': 'nope'}, pillar=test_two_level_dict)) - - def test_parse_docstring(self): - test_keystone_str = '''Management of Keystone users - ============================ - - :depends: - keystoneclient Python module - :configuration: See :py:mod:`salt.modules.keystone` for setup instructions. -''' - - ret = utils.parse_docstring(test_keystone_str) - expected_dict = {'deps': ['keystoneclient'], - 'full': 'Management of Keystone users\n ' - '============================\n\n ' - ':depends: - keystoneclient Python module\n ' - ':configuration: See :py:mod:`salt.modules.keystone` for setup instructions.\n'} - self.assertDictEqual(ret, expected_dict) + self.assertDictEqual({'not_found': 'nope'}, salt.utils.option('foo:bar', {'not_found': 'nope'})) + self.assertEqual('baz', salt.utils.option('foo:bar', {'not_found': 'nope'}, opts=test_two_level_dict)) + self.assertEqual('baz', salt.utils.option('foo:bar', {'not_found': 'nope'}, pillar={'master': test_two_level_dict})) + self.assertEqual('baz', salt.utils.option('foo:bar', {'not_found': 'nope'}, pillar=test_two_level_dict)) def test_get_hash_exception(self): - self.assertRaises(ValueError, utils.get_hash, '/tmp/foo/', form='INVALID') - - def test_parse_kwarg(self): - ret = args.parse_kwarg('foo=bar') - self.assertEqual(ret, ('foo', 'bar')) - - ret = args.parse_kwarg('foobar') - self.assertEqual(ret, (None, None)) + self.assertRaises(ValueError, salt.utils.get_hash, '/tmp/foo/', form='INVALID') @skipIf(NO_MOCK, NO_MOCK_REASON) def test_date_cast(self): now = datetime.datetime.now() with patch('datetime.datetime'): datetime.datetime.now.return_value = now - self.assertEqual(now, utils.date_cast(None)) - self.assertEqual(now, utils.date_cast(now)) + self.assertEqual(now, salt.utils.date_cast(None)) + self.assertEqual(now, salt.utils.date_cast(now)) try: - ret = utils.date_cast('Mon Dec 23 10:19:15 MST 2013') + ret = salt.utils.date_cast('Mon Dec 23 10:19:15 MST 2013') expected_ret = datetime.datetime(2013, 12, 23, 10, 19, 15) self.assertEqual(ret, expected_ret) except RuntimeError: @@ -836,67 +306,56 @@ class UtilsTestCase(TestCase): expected_ret = '2002-12-25' src = datetime.datetime(2002, 12, 25, 12, 00, 00, 00) - ret = utils.date_format(src) + ret = salt.utils.date_format(src) self.assertEqual(ret, expected_ret) src = '2002/12/25' - ret = utils.date_format(src) + ret = salt.utils.date_format(src) self.assertEqual(ret, expected_ret) src = 1040814000 - ret = utils.date_format(src) + ret = salt.utils.date_format(src) self.assertEqual(ret, expected_ret) src = '1040814000' - ret = utils.date_format(src) + ret = salt.utils.date_format(src) self.assertEqual(ret, expected_ret) def test_yaml_dquote(self): for teststr in (r'"\ []{}"',): - self.assertEqual(teststr, yaml.safe_load(utils.yamlencoding.yaml_dquote(teststr))) + self.assertEqual(teststr, yaml.safe_load(salt.utils.yamlencoding.yaml_dquote(teststr))) def test_yaml_dquote_doesNotAddNewLines(self): teststr = '"' * 100 - self.assertNotIn('\n', utils.yamlencoding.yaml_dquote(teststr)) + self.assertNotIn('\n', salt.utils.yamlencoding.yaml_dquote(teststr)) def test_yaml_squote(self): - ret = utils.yamlencoding.yaml_squote(r'"') + ret = salt.utils.yamlencoding.yaml_squote(r'"') self.assertEqual(ret, r"""'"'""") def test_yaml_squote_doesNotAddNewLines(self): teststr = "'" * 100 - self.assertNotIn('\n', utils.yamlencoding.yaml_squote(teststr)) + self.assertNotIn('\n', salt.utils.yamlencoding.yaml_squote(teststr)) def test_yaml_encode(self): for testobj in (None, True, False, '[7, 5]', '"monkey"', 5, 7.5, "2014-06-02 15:30:29.7"): - self.assertEqual(testobj, yaml.safe_load(utils.yamlencoding.yaml_encode(testobj))) + self.assertEqual(testobj, yaml.safe_load(salt.utils.yamlencoding.yaml_encode(testobj))) for testobj in ({}, [], set()): - self.assertRaises(TypeError, utils.yamlencoding.yaml_encode, testobj) + self.assertRaises(TypeError, salt.utils.yamlencoding.yaml_encode, testobj) def test_compare_dicts(self): - ret = utils.compare_dicts(old={'foo': 'bar'}, new={'foo': 'bar'}) + ret = salt.utils.compare_dicts(old={'foo': 'bar'}, new={'foo': 'bar'}) self.assertEqual(ret, {}) - ret = utils.compare_dicts(old={'foo': 'bar'}, new={'foo': 'woz'}) + ret = salt.utils.compare_dicts(old={'foo': 'bar'}, new={'foo': 'woz'}) expected_ret = {'foo': {'new': 'woz', 'old': 'bar'}} self.assertDictEqual(ret, expected_ret) - @skipIf(NO_MOCK, NO_MOCK_REASON) - def test_argspec_report(self): - def _test_spec(arg1, arg2, kwarg1=None): - pass - - sys_mock = create_autospec(_test_spec) - test_functions = {'test_module.test_spec': sys_mock} - ret = utils.argspec_report(test_functions, 'test_module.test_spec') - self.assertDictEqual(ret, {'test_module.test_spec': - {'kwargs': True, 'args': None, 'defaults': None, 'varargs': True}}) - def test_decode_list(self): test_data = [u'unicode_str', [u'unicode_item_in_list', 'second_item_in_list'], {'dict_key': u'dict_val'}] expected_ret = ['unicode_str', ['unicode_item_in_list', 'second_item_in_list'], {'dict_key': 'dict_val'}] - ret = utils.decode_list(test_data) + ret = salt.utils.decode_list(test_data) self.assertEqual(ret, expected_ret) def test_decode_dict(self): @@ -906,7 +365,7 @@ class UtilsTestCase(TestCase): expected_ret = {'test_unicode_key': 'test_unicode_val', 'test_list_key': ['list_1', 'unicode_list_two'], 'test_dict_key': {'test_sub_dict_key': 'test_sub_dict_val'}} - ret = utils.decode_dict(test_data) + ret = salt.utils.decode_dict(test_data) self.assertDictEqual(ret, expected_ret) def test_find_json(self): @@ -942,32 +401,16 @@ class UtilsTestCase(TestCase): 'Abbrev': 'ISO 8879:1986', 'ID': 'SGML'}}, 'title': 'S'}, 'title': 'example glossary'}} # First test the valid JSON - ret = utils.find_json(test_sample_json) + ret = salt.utils.find_json(test_sample_json) self.assertDictEqual(ret, expected_ret) # Now pre-pend some garbage and re-test - garbage_prepend_json = '{0}{1}'.format(LORUM_IPSUM, test_sample_json) - ret = utils.find_json(garbage_prepend_json) + garbage_prepend_json = '{0}{1}'.format(LOREM_IPSUM, test_sample_json) + ret = salt.utils.find_json(garbage_prepend_json) self.assertDictEqual(ret, expected_ret) # Test to see if a ValueError is raised if no JSON is passed in - self.assertRaises(ValueError, utils.find_json, LORUM_IPSUM) - - def test_is_bin_str(self): - self.assertFalse(utils.is_bin_str(LORUM_IPSUM)) - - zero_str = '{0}{1}'.format(LORUM_IPSUM, '\0') - self.assertTrue(utils.is_bin_str(zero_str)) - - # To to ensure safe exit if str passed doesn't evaluate to True - self.assertFalse(utils.is_bin_str('')) - - nontext = 3 * (''.join([chr(x) for x in range(1, 32) if x not in (8, 9, 10, 12, 13)])) - almost_bin_str = '{0}{1}'.format(LORUM_IPSUM[:100], nontext[:42]) - self.assertFalse(utils.is_bin_str(almost_bin_str)) - - bin_str = almost_bin_str + '\x01' - self.assertTrue(utils.is_bin_str(bin_str)) + self.assertRaises(ValueError, salt.utils.find_json, LOREM_IPSUM) def test_repack_dict(self): list_of_one_element_dicts = [{'dict_key_1': 'dict_val_1'}, @@ -976,146 +419,53 @@ class UtilsTestCase(TestCase): expected_ret = {'dict_key_1': 'dict_val_1', 'dict_key_2': 'dict_val_2', 'dict_key_3': 'dict_val_3'} - ret = utils.repack_dictlist(list_of_one_element_dicts) + ret = salt.utils.repack_dictlist(list_of_one_element_dicts) self.assertDictEqual(ret, expected_ret) # Try with yaml yaml_key_val_pair = '- key1: val1' - ret = utils.repack_dictlist(yaml_key_val_pair) + ret = salt.utils.repack_dictlist(yaml_key_val_pair) self.assertDictEqual(ret, {'key1': 'val1'}) # Make sure we handle non-yaml junk data - ret = utils.repack_dictlist(LORUM_IPSUM) + ret = salt.utils.repack_dictlist(LOREM_IPSUM) self.assertDictEqual(ret, {}) - def test_get_colors(self): - ret = utils.get_colors() - self.assertEqual('\x1b[0;37m', str(ret['LIGHT_GRAY'])) - - ret = utils.get_colors(use=False) - self.assertDictContainsSubset({'LIGHT_GRAY': ''}, ret) - - ret = utils.get_colors(use='LIGHT_GRAY') - # LIGHT_YELLOW now == LIGHT_GRAY - self.assertEqual(str(ret['LIGHT_YELLOW']), str(ret['LIGHT_GRAY'])) - @skipIf(NO_MOCK, NO_MOCK_REASON) def test_daemonize_if(self): # pylint: disable=assignment-from-none with patch('sys.argv', ['salt-call']): - ret = utils.daemonize_if({}) + ret = salt.utils.daemonize_if({}) self.assertEqual(None, ret) - ret = utils.daemonize_if({'multiprocessing': False}) + ret = salt.utils.daemonize_if({'multiprocessing': False}) self.assertEqual(None, ret) with patch('sys.platform', 'win'): - ret = utils.daemonize_if({}) + ret = salt.utils.daemonize_if({}) self.assertEqual(None, ret) with patch('salt.utils.daemonize'): - utils.daemonize_if({}) - self.assertTrue(utils.daemonize.called) + salt.utils.daemonize_if({}) + self.assertTrue(salt.utils.daemonize.called) # pylint: enable=assignment-from-none - @skipIf(NO_MOCK, NO_MOCK_REASON) - def test_which_bin(self): - ret = utils.which_bin('str') - self.assertIs(None, ret) - - test_exes = ['ls', 'echo'] - with patch('salt.utils.which', return_value='/tmp/dummy_path'): - ret = utils.which_bin(test_exes) - self.assertEqual(ret, '/tmp/dummy_path') - - ret = utils.which_bin([]) - self.assertIs(None, ret) - - with patch('salt.utils.which', return_value=''): - ret = utils.which_bin(test_exes) - self.assertIs(None, ret) - @skipIf(NO_MOCK, NO_MOCK_REASON) def test_gen_jid(self): now = datetime.datetime(2002, 12, 25, 12, 00, 00, 00) with patch('datetime.datetime'): datetime.datetime.now.return_value = now - ret = utils.jid.gen_jid() + ret = salt.utils.jid.gen_jid({}) self.assertEqual(ret, '20021225120000000000') + salt.utils.jid.LAST_JID_DATETIME = None + ret = salt.utils.jid.gen_jid({'unique_jid': True}) + self.assertEqual(ret, '20021225120000000000_{0}'.format(os.getpid())) + ret = salt.utils.jid.gen_jid({'unique_jid': True}) + self.assertEqual(ret, '20021225120000000001_{0}'.format(os.getpid())) @skipIf(NO_MOCK, NO_MOCK_REASON) def test_check_or_die(self): - self.assertRaises(CommandNotFoundError, utils.check_or_die, None) + self.assertRaises(CommandNotFoundError, salt.utils.check_or_die, None) - with patch('salt.utils.which', return_value=False): - self.assertRaises(CommandNotFoundError, utils.check_or_die, 'FAKE COMMAND') - - @skipIf(NO_MOCK, NO_MOCK_REASON) - def test_compare_versions(self): - ret = utils.compare_versions('1.0', '==', '1.0') - self.assertTrue(ret) - - ret = utils.compare_versions('1.0', '!=', '1.0') - self.assertFalse(ret) - - with patch('salt.utils.log') as log_mock: - ret = utils.compare_versions('1.0', 'HAH I AM NOT A COMP OPERATOR! I AM YOUR FATHER!', '1.0') - self.assertTrue(log_mock.error.called) - - def test_kwargs_warn_until(self): - # Test invalid version arg - self.assertRaises(RuntimeError, utils.kwargs_warn_until, {}, []) - - def test_to_str(self): - for x in (123, (1, 2, 3), [1, 2, 3], {1: 23}, None): - self.assertRaises(TypeError, utils.to_str, x) - if six.PY3: - self.assertEqual(utils.to_str('plugh'), 'plugh') - self.assertEqual(utils.to_str('áéíóúý', 'utf-8'), 'áéíóúý') - un = '\u4e2d\u56fd\u8a9e (\u7e41\u4f53)' # pylint: disable=anomalous-unicode-escape-in-string - ut = bytes((0xe4, 0xb8, 0xad, 0xe5, 0x9b, 0xbd, 0xe8, 0xaa, 0x9e, 0x20, 0x28, 0xe7, 0xb9, 0x81, 0xe4, 0xbd, 0x93, 0x29)) - self.assertEqual(utils.to_str(ut, 'utf-8'), un) - self.assertEqual(utils.to_str(bytearray(ut), 'utf-8'), un) - # Test situation when a minion returns incorrect utf-8 string because of... million reasons - ut2 = b'\x9c' - self.assertEqual(utils.to_str(ut2, 'utf-8'), u'\ufffd') - self.assertEqual(utils.to_str(bytearray(ut2), 'utf-8'), u'\ufffd') - else: - self.assertEqual(utils.to_str('plugh'), 'plugh') - self.assertEqual(utils.to_str(u'áéíóúý', 'utf-8'), 'áéíóúý') - un = u'\u4e2d\u56fd\u8a9e (\u7e41\u4f53)' - ut = '\xe4\xb8\xad\xe5\x9b\xbd\xe8\xaa\x9e (\xe7\xb9\x81\xe4\xbd\x93)' - self.assertEqual(utils.to_str(un, 'utf-8'), ut) - self.assertEqual(utils.to_str(bytearray(ut), 'utf-8'), ut) - - def test_to_bytes(self): - for x in (123, (1, 2, 3), [1, 2, 3], {1: 23}, None): - self.assertRaises(TypeError, utils.to_bytes, x) - if six.PY3: - self.assertEqual(utils.to_bytes('xyzzy'), b'xyzzy') - ut = bytes((0xe4, 0xb8, 0xad, 0xe5, 0x9b, 0xbd, 0xe8, 0xaa, 0x9e, 0x20, 0x28, 0xe7, 0xb9, 0x81, 0xe4, 0xbd, 0x93, 0x29)) - un = '\u4e2d\u56fd\u8a9e (\u7e41\u4f53)' # pylint: disable=anomalous-unicode-escape-in-string - self.assertEqual(utils.to_bytes(ut), ut) - self.assertEqual(utils.to_bytes(bytearray(ut)), ut) - self.assertEqual(utils.to_bytes(un, 'utf-8'), ut) - else: - self.assertEqual(utils.to_bytes('xyzzy'), 'xyzzy') - ut = ''.join([chr(x) for x in (0xe4, 0xb8, 0xad, 0xe5, 0x9b, 0xbd, 0xe8, 0xaa, 0x9e, 0x20, 0x28, 0xe7, 0xb9, 0x81, 0xe4, 0xbd, 0x93, 0x29)]) - un = u'\u4e2d\u56fd\u8a9e (\u7e41\u4f53)' # pylint: disable=anomalous-unicode-escape-in-string - self.assertEqual(utils.to_bytes(ut), ut) - self.assertEqual(utils.to_bytes(bytearray(ut)), ut) - self.assertEqual(utils.to_bytes(un, 'utf-8'), ut) - - def test_to_unicode(self): - if six.PY3: - self.assertEqual(utils.to_unicode('plugh'), 'plugh') - self.assertEqual(utils.to_unicode('áéíóúý'), 'áéíóúý') - un = '\u4e2d\u56fd\u8a9e (\u7e41\u4f53)' # pylint: disable=anomalous-unicode-escape-in-string - ut = bytes((0xe4, 0xb8, 0xad, 0xe5, 0x9b, 0xbd, 0xe8, 0xaa, 0x9e, 0x20, 0x28, 0xe7, 0xb9, 0x81, 0xe4, 0xbd, 0x93, 0x29)) - self.assertEqual(utils.to_unicode(ut, 'utf-8'), un) - self.assertEqual(utils.to_unicode(bytearray(ut), 'utf-8'), un) - else: - self.assertEqual(utils.to_unicode('xyzzy', 'utf-8'), u'xyzzy') - ut = '\xe4\xb8\xad\xe5\x9b\xbd\xe8\xaa\x9e (\xe7\xb9\x81\xe4\xbd\x93)' - un = u'\u4e2d\u56fd\u8a9e (\u7e41\u4f53)' - self.assertEqual(utils.to_unicode(ut, 'utf-8'), un) + with patch('salt.utils.path.which', return_value=False): + self.assertRaises(CommandNotFoundError, salt.utils.check_or_die, 'FAKE COMMAND') diff --git a/tests/unit/utils/test_verify.py b/tests/unit/utils/test_verify.py index 795298877dd..d1bd3a2ea55 100644 --- a/tests/unit/utils/test_verify.py +++ b/tests/unit/utils/test_verify.py @@ -29,7 +29,7 @@ from tests.support.mock import ( ) # Import salt libs -import salt.utils +import salt.utils.files from salt.utils.verify import ( check_user, verify_env, @@ -156,7 +156,7 @@ class TestVerify(TestCase): for n in range(prev, newmax): kpath = os.path.join(keys_dir, str(n)) - with salt.utils.fopen(kpath, 'w') as fp_: + with salt.utils.files.fopen(kpath, 'w') as fp_: fp_.write(str(n)) opts = { @@ -191,7 +191,7 @@ class TestVerify(TestCase): newmax = mof_test for n in range(prev, newmax): kpath = os.path.join(keys_dir, str(n)) - with salt.utils.fopen(kpath, 'w') as fp_: + with salt.utils.files.fopen(kpath, 'w') as fp_: fp_.write(str(n)) opts = { diff --git a/tests/unit/utils/test_versions.py b/tests/unit/utils/test_versions.py index bc6a665ba6e..d6b6f0f165a 100644 --- a/tests/unit/utils/test_versions.py +++ b/tests/unit/utils/test_versions.py @@ -10,15 +10,20 @@ # Import python libs from __future__ import absolute_import +import sys +import warnings # Import Salt Testing libs -from tests.support.unit import TestCase +from tests.support.unit import TestCase, skipIf +from tests.support.mock import patch, NO_MOCK, NO_MOCK_REASON # Import Salt libs +import salt.version +import salt.utils.versions from salt.utils.versions import LooseVersion, StrictVersion # Import 3rd-party libs -import salt.ext.six as six +from salt.ext import six if six.PY2: cmp_method = '__cmp__' @@ -89,3 +94,162 @@ class VersionTestCase(TestCase): self.assertEqual(res, wanted, 'cmp(%s, %s) should be %s, got %s' % (v1, v2, wanted, res)) + + +class VersionFuncsTestCase(TestCase): + + @skipIf(NO_MOCK, NO_MOCK_REASON) + def test_compare(self): + ret = salt.utils.versions.compare('1.0', '==', '1.0') + self.assertTrue(ret) + + ret = salt.utils.versions.compare('1.0', '!=', '1.0') + self.assertFalse(ret) + + with patch.object(salt.utils.versions, 'log') as log_mock: + ret = salt.utils.versions.compare('1.0', 'HAH I AM NOT A COMP OPERATOR! I AM YOUR FATHER!', '1.0') + self.assertTrue(log_mock.error.called) + + def test_kwargs_warn_until(self): + # Test invalid version arg + self.assertRaises(RuntimeError, salt.utils.versions.kwargs_warn_until, {}, []) + + def test_warn_until_warning_raised(self): + # We *always* want *all* warnings thrown on this module + warnings.filterwarnings('always', '', DeprecationWarning, __name__) + + def raise_warning(_version_info_=(0, 16, 0)): + salt.utils.versions.warn_until( + (0, 17), 'Deprecation Message!', + _version_info_=_version_info_ + + ) + + def raise_named_version_warning(_version_info_=(0, 16, 0)): + salt.utils.versions.warn_until( + 'Hydrogen', 'Deprecation Message!', + _version_info_=_version_info_ + ) + + # raise_warning should show warning until version info is >= (0, 17) + with warnings.catch_warnings(record=True) as recorded_warnings: + raise_warning() + self.assertEqual( + 'Deprecation Message!', str(recorded_warnings[0].message) + ) + + # raise_warning should show warning until version info is >= (0, 17) + with warnings.catch_warnings(record=True) as recorded_warnings: + raise_named_version_warning() + self.assertEqual( + 'Deprecation Message!', str(recorded_warnings[0].message) + ) + + # the deprecation warning is not issued because we passed + # _dont_call_warning + with warnings.catch_warnings(record=True) as recorded_warnings: + salt.utils.versions.warn_until( + (0, 17), 'Foo', _dont_call_warnings=True, + _version_info_=(0, 16) + ) + self.assertEqual(0, len(recorded_warnings)) + + # Let's set version info to (0, 17), a RuntimeError should be raised + with self.assertRaisesRegex( + RuntimeError, + r'The warning triggered on filename \'(.*)test_versions.py\', ' + r'line number ([\d]+), is supposed to be shown until version ' + r'0.17.0 is released. Current version is now 0.17.0. ' + r'Please remove the warning.'): + raise_warning(_version_info_=(0, 17, 0)) + + # Let's set version info to (0, 17), a RuntimeError should be raised + with self.assertRaisesRegex( + RuntimeError, + r'The warning triggered on filename \'(.*)test_versions.py\', ' + r'line number ([\d]+), is supposed to be shown until version ' + r'(.*) is released. Current version is now ' + r'([\d.]+). Please remove the warning.'): + raise_named_version_warning(_version_info_=(getattr(sys, 'maxint', None) or getattr(sys, 'maxsize'), 16, 0)) + + # Even though we're calling warn_until, we pass _dont_call_warnings + # because we're only after the RuntimeError + with self.assertRaisesRegex( + RuntimeError, + r'The warning triggered on filename \'(.*)test_versions.py\', ' + r'line number ([\d]+), is supposed to be shown until version ' + r'0.17.0 is released. Current version is now ' + r'(.*). Please remove the warning.'): + salt.utils.versions.warn_until( + (0, 17), 'Foo', _dont_call_warnings=True + ) + + with self.assertRaisesRegex( + RuntimeError, + r'The warning triggered on filename \'(.*)test_versions.py\', ' + r'line number ([\d]+), is supposed to be shown until version ' + r'(.*) is released. Current version is now ' + r'(.*). Please remove the warning.'): + salt.utils.versions.warn_until( + 'Hydrogen', 'Foo', _dont_call_warnings=True, + _version_info_=(getattr(sys, 'maxint', None) or getattr(sys, 'maxsize'), 16, 0) + ) + + # version on the deprecation message gets properly formatted + with warnings.catch_warnings(record=True) as recorded_warnings: + vrs = salt.version.SaltStackVersion.from_name('Helium') + salt.utils.versions.warn_until( + 'Helium', 'Deprecation Message until {version}!', + _version_info_=(vrs.major - 1, 0) + ) + self.assertEqual( + 'Deprecation Message until {0}!'.format(vrs.formatted_version), + str(recorded_warnings[0].message) + ) + + def test_kwargs_warn_until_warning_raised(self): + # We *always* want *all* warnings thrown on this module + warnings.filterwarnings('always', '', DeprecationWarning, __name__) + + def raise_warning(**kwargs): + _version_info_ = kwargs.pop('_version_info_', (0, 16, 0)) + salt.utils.versions.kwargs_warn_until( + kwargs, + (0, 17), + _version_info_=_version_info_ + ) + + # raise_warning({...}) should show warning until version info is >= (0, 17) + with warnings.catch_warnings(record=True) as recorded_warnings: + raise_warning(foo=42) # with a kwarg + self.assertEqual( + 'The following parameter(s) have been deprecated and ' + 'will be removed in \'0.17.0\': \'foo\'.', + str(recorded_warnings[0].message) + ) + # With no **kwargs, should not show warning until version info is >= (0, 17) + with warnings.catch_warnings(record=True) as recorded_warnings: + salt.utils.versions.kwargs_warn_until( + {}, # no kwargs + (0, 17), + _version_info_=(0, 16, 0) + ) + self.assertEqual(0, len(recorded_warnings)) + + # Let's set version info to (0, 17), a RuntimeError should be raised + # regardless of whether or not we pass any **kwargs. + with self.assertRaisesRegex( + RuntimeError, + r'The warning triggered on filename \'(.*)test_versions.py\', ' + r'line number ([\d]+), is supposed to be shown until version ' + r'0.17.0 is released. Current version is now 0.17.0. ' + r'Please remove the warning.'): + raise_warning(_version_info_=(0, 17)) # no kwargs + + with self.assertRaisesRegex( + RuntimeError, + r'The warning triggered on filename \'(.*)test_versions.py\', ' + r'line number ([\d]+), is supposed to be shown until version ' + r'0.17.0 is released. Current version is now 0.17.0. ' + r'Please remove the warning.'): + raise_warning(bar='baz', qux='quux', _version_info_=(0, 17)) # some kwargs diff --git a/tests/unit/utils/test_vsan.py b/tests/unit/utils/test_vsan.py new file mode 100644 index 00000000000..197ba517dec --- /dev/null +++ b/tests/unit/utils/test_vsan.py @@ -0,0 +1,218 @@ +# -*- coding: utf-8 -*- +''' + :codeauthor: :email:`Alexandru Bleotu ` + + Tests functions in salt.utils.vsan +''' + +# Import python libraries +from __future__ import absolute_import +import logging + +# Import Salt testing libraries +from tests.support.mixins import LoaderModuleMockMixin +from tests.support.unit import TestCase, skipIf +from tests.support.mock import NO_MOCK, NO_MOCK_REASON, patch, MagicMock, \ + PropertyMock + +# Import Salt libraries +from salt.exceptions import VMwareApiError, VMwareRuntimeError +from salt.utils import vsan + +try: + from pyVmomi import vim, vmodl + HAS_PYVMOMI = True +except ImportError: + HAS_PYVMOMI = False +HAS_PYVSAN = vsan.HAS_PYVSAN + + +# Get Logging Started +log = logging.getLogger(__name__) + + +@skipIf(NO_MOCK, NO_MOCK_REASON) +@skipIf(not HAS_PYVMOMI, 'The \'pyvmomi\' library is missing') +@skipIf(not HAS_PYVSAN, 'The \'vsan\' ext library is missing') +class VsanSupportedTestCase(TestCase): + '''Tests for salt.utils.vsan.vsan_supported''' + + def test_supported_api_version(self): + mock_si = MagicMock(content=MagicMock(about=MagicMock())) + type(mock_si.content.about).apiVersion = \ + PropertyMock(return_value='6.0') + self.assertTrue(vsan.vsan_supported(mock_si)) + + def test_unsupported_api_version(self): + mock_si = MagicMock(content=MagicMock(about=MagicMock())) + type(mock_si.content.about).apiVersion = \ + PropertyMock(return_value='5.0') + self.assertFalse(vsan.vsan_supported(mock_si)) + + def test_api_version_raises_no_permission(self): + exc = vim.fault.NoPermission() + exc.privilegeId = 'Fake privilege' + mock_si = MagicMock(content=MagicMock(about=MagicMock())) + type(mock_si.content.about).apiVersion = PropertyMock(side_effect=exc) + with self.assertRaises(VMwareApiError) as excinfo: + vsan.vsan_supported(mock_si) + self.assertEqual(excinfo.exception.strerror, + 'Not enough permissions. Required privilege: ' + 'Fake privilege') + + def test_api_version_raises_vim_fault(self): + exc = vim.fault.VimFault() + exc.msg = 'VimFault msg' + mock_si = MagicMock(content=MagicMock(about=MagicMock())) + type(mock_si.content.about).apiVersion = PropertyMock(side_effect=exc) + with self.assertRaises(VMwareApiError) as excinfo: + vsan.vsan_supported(mock_si) + self.assertEqual(excinfo.exception.strerror, 'VimFault msg') + + def test_api_version_raises_runtime_fault(self): + exc = vmodl.RuntimeFault() + exc.msg = 'RuntimeFault msg' + mock_si = MagicMock(content=MagicMock(about=MagicMock())) + type(mock_si.content.about).apiVersion = PropertyMock(side_effect=exc) + with self.assertRaises(VMwareRuntimeError) as excinfo: + vsan.vsan_supported(mock_si) + self.assertEqual(excinfo.exception.strerror, 'RuntimeFault msg') + + +@skipIf(NO_MOCK, NO_MOCK_REASON) +@skipIf(not HAS_PYVMOMI, 'The \'pyvmomi\' library is missing') +@skipIf(not HAS_PYVSAN, 'The \'vsan\' ext library is missing') +class GetVsanClusterConfigSystemTestCase(TestCase, LoaderModuleMockMixin): + '''Tests for salt.utils.vsan.get_vsan_cluster_config_system''' + def setup_loader_modules(self): + return {vsan: { + '__virtual__': MagicMock(return_value='vsan'), + 'sys': MagicMock(), + 'ssl': MagicMock()}} + + def setUp(self): + self.mock_si = MagicMock() + self.mock_ret = MagicMock() + patches = (('salt.utils.vsan.vsanapiutils.GetVsanVcMos', + MagicMock( + return_value={'vsan-cluster-config-system': + self.mock_ret})),) + for mod, mock in patches: + patcher = patch(mod, mock) + patcher.start() + self.addCleanup(patcher.stop) + + type(vsan.sys).version_info = PropertyMock(return_value=(2, 7, 9)) + self.mock_context = MagicMock() + self.mock_create_default_context = \ + MagicMock(return_value=self.mock_context) + vsan.ssl.create_default_context = self.mock_create_default_context + + def tearDown(self): + for attr in ('mock_si', 'mock_ret', 'mock_context', + 'mock_create_default_context'): + delattr(self, attr) + + def test_ssl_default_context_loaded(self): + vsan.get_vsan_cluster_config_system(self.mock_si) + self.mock_create_default_context.assert_called_once_with() + self.assertFalse(self.mock_context.check_hostname) + self.assertEqual(self.mock_context.verify_mode, vsan.ssl.CERT_NONE) + + def test_ssl_default_context_not_loaded(self): + type(vsan.sys).version_info = PropertyMock(return_value=(2, 7, 8)) + vsan.get_vsan_cluster_config_system(self.mock_si) + self.assertEqual(self.mock_create_default_context.call_count, 0) + + def test_GetVsanVcMos_call(self): + mock_get_vsan_vc_mos = MagicMock() + with patch('salt.utils.vsan.vsanapiutils.GetVsanVcMos', + mock_get_vsan_vc_mos): + vsan.get_vsan_cluster_config_system(self.mock_si) + mock_get_vsan_vc_mos.assert_called_once_with(self.mock_si._stub, + context=self.mock_context) + + def test_return(self): + ret = vsan.get_vsan_cluster_config_system(self.mock_si) + self.assertEqual(ret, self.mock_ret) + + +@skipIf(NO_MOCK, NO_MOCK_REASON) +@skipIf(not HAS_PYVMOMI, 'The \'pyvmomi\' library is missing') +@skipIf(not HAS_PYVSAN, 'The \'vsan\' ext library is missing') +class GetClusterVsanInfoTestCase(TestCase, LoaderModuleMockMixin): + '''Tests for salt.utils.vsan.get_cluster_vsan_info''' + def setup_loader_modules(self): + return {vsan: { + '__virtual__': MagicMock(return_value='vsan')}} + + def setUp(self): + self.mock_cl_ref = MagicMock() + self.mock_si = MagicMock() + patches = ( + ('salt.utils.vmware.get_managed_object_name', MagicMock()), + ('salt.utils.vmware.get_service_instance_from_managed_object', + MagicMock(return_value=self.mock_si)), + ('salt.utils.vsan.get_vsan_cluster_config_system', MagicMock())) + for mod, mock in patches: + patcher = patch(mod, mock) + patcher.start() + self.addCleanup(patcher.stop) + + def tearDown(self): + for attr in ('mock_si', 'mock_cl_ref'): + delattr(self, attr) + + def test_get_managed_object_name_call(self): + mock_get_managed_object_name = MagicMock() + with patch('salt.utils.vmware.get_managed_object_name', + mock_get_managed_object_name): + vsan.get_cluster_vsan_info(self.mock_cl_ref) + mock_get_managed_object_name.assert_called_once_with(self.mock_cl_ref) + + def test_get_vsan_cluster_config_system_call(self): + mock_get_vsan_cl_syst = MagicMock() + with patch('salt.utils.vsan.get_vsan_cluster_config_system', + mock_get_vsan_cl_syst): + vsan.get_cluster_vsan_info(self.mock_cl_ref) + mock_get_vsan_cl_syst.assert_called_once_with(self.mock_si) + + def test_VsanClusterGetConfig_call(self): + mock_vsan_sys = MagicMock() + with patch('salt.utils.vsan.get_vsan_cluster_config_system', + MagicMock(return_value=mock_vsan_sys)): + vsan.get_cluster_vsan_info(self.mock_cl_ref) + mock_vsan_sys.VsanClusterGetConfig.assert_called_once_with( + self.mock_cl_ref) + + def test_VsanClusterGetConfig_raises_no_permission(self): + exc = vim.fault.NoPermission() + exc.privilegeId = 'Fake privilege' + with patch('salt.utils.vsan.get_vsan_cluster_config_system', + MagicMock(return_value=MagicMock( + VsanClusterGetConfig=MagicMock(side_effect=exc)))): + with self.assertRaises(VMwareApiError) as excinfo: + vsan.get_cluster_vsan_info(self.mock_cl_ref) + self.assertEqual(excinfo.exception.strerror, + 'Not enough permissions. Required privilege: ' + 'Fake privilege') + + def test_VsanClusterGetConfig_raises_vim_fault(self): + exc = vim.fault.VimFault() + exc.msg = 'VimFault msg' + with patch('salt.utils.vsan.get_vsan_cluster_config_system', + MagicMock(return_value=MagicMock( + VsanClusterGetConfig=MagicMock(side_effect=exc)))): + with self.assertRaises(VMwareApiError) as excinfo: + vsan.get_cluster_vsan_info(self.mock_cl_ref) + self.assertEqual(excinfo.exception.strerror, 'VimFault msg') + + def test_VsanClusterGetConfig_raises_runtime_fault(self): + exc = vmodl.RuntimeFault() + exc.msg = 'RuntimeFault msg' + with patch('salt.utils.vsan.get_vsan_cluster_config_system', + MagicMock(return_value=MagicMock( + VsanClusterGetConfig=MagicMock(side_effect=exc)))): + with self.assertRaises(VMwareRuntimeError) as excinfo: + vsan.get_cluster_vsan_info(self.mock_cl_ref) + self.assertEqual(excinfo.exception.strerror, 'RuntimeFault msg') diff --git a/tests/unit/utils/test_vt.py b/tests/unit/utils/test_vt.py index 7ba4282a194..b65f904c5ce 100644 --- a/tests/unit/utils/test_vt.py +++ b/tests/unit/utils/test_vt.py @@ -9,7 +9,7 @@ VirtualTerminal tests ''' -# Import python libs +# Import Python libs from __future__ import absolute_import import os import sys @@ -20,8 +20,9 @@ import time # Import Salt Testing libs from tests.support.unit import TestCase, skipIf -# Import salt libs -import salt.utils +# Import Salt libs +import salt.utils.files +import salt.utils.platform import salt.utils.vt # Import 3rd-party libs @@ -60,7 +61,7 @@ class VTTestCase(TestCase): # Get current number of PTY's try: if os.path.exists('/proc/sys/kernel/pty/nr'): - with salt.utils.fopen('/proc/sys/kernel/pty/nr') as fh_: + with salt.utils.files.fopen('/proc/sys/kernel/pty/nr') as fh_: return int(fh_.read().strip()) proc = subprocess.Popen( @@ -71,7 +72,7 @@ class VTTestCase(TestCase): stdout, _ = proc.communicate() return int(stdout.strip()) except (ValueError, OSError, IOError): - if salt.utils.is_darwin(): + if salt.utils.platform.is_darwin(): # We're unable to findout how many PTY's are open self.skipTest( 'Unable to find out how many PTY\'s are open on Darwin - ' diff --git a/tests/unit/utils/test_warnings.py b/tests/unit/utils/test_warnings.py deleted file mode 100644 index e658def5045..00000000000 --- a/tests/unit/utils/test_warnings.py +++ /dev/null @@ -1,165 +0,0 @@ -# -*- coding: utf-8 -*- -''' - :codeauthor: :email:`Pedro Algarvio (pedro@algarvio.me)` - - - tests.unit.utils.test_warnings - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - Test ``salt.utils.warn_until`` and ``salt.utils.kwargs_warn_until`` -''' - -# Import python libs -from __future__ import absolute_import -import sys -import warnings - -# Import Salt Testing libs -from tests.support.unit import TestCase - -# Import salt libs -from salt.utils import warn_until, kwargs_warn_until -from salt.version import SaltStackVersion - - -class WarnUntilTestCase(TestCase): - - def test_warn_until_warning_raised(self): - # We *always* want *all* warnings thrown on this module - warnings.filterwarnings('always', '', DeprecationWarning, __name__) - - def raise_warning(_version_info_=(0, 16, 0)): - warn_until( - (0, 17), 'Deprecation Message!', - _version_info_=_version_info_ - - ) - - def raise_named_version_warning(_version_info_=(0, 16, 0)): - warn_until( - 'Hydrogen', 'Deprecation Message!', - _version_info_=_version_info_ - ) - - # raise_warning should show warning until version info is >= (0, 17) - with warnings.catch_warnings(record=True) as recorded_warnings: - raise_warning() - self.assertEqual( - 'Deprecation Message!', str(recorded_warnings[0].message) - ) - - # raise_warning should show warning until version info is >= (0, 17) - with warnings.catch_warnings(record=True) as recorded_warnings: - raise_named_version_warning() - self.assertEqual( - 'Deprecation Message!', str(recorded_warnings[0].message) - ) - - # the deprecation warning is not issued because we passed - # _dont_call_warning - with warnings.catch_warnings(record=True) as recorded_warnings: - warn_until( - (0, 17), 'Foo', _dont_call_warnings=True, - _version_info_=(0, 16) - ) - self.assertEqual(0, len(recorded_warnings)) - - # Let's set version info to (0, 17), a RuntimeError should be raised - with self.assertRaisesRegex( - RuntimeError, - r'The warning triggered on filename \'(.*)test_warnings.py\', ' - r'line number ([\d]+), is supposed to be shown until version ' - r'0.17.0 is released. Current version is now 0.17.0. ' - r'Please remove the warning.'): - raise_warning(_version_info_=(0, 17, 0)) - - # Let's set version info to (0, 17), a RuntimeError should be raised - with self.assertRaisesRegex( - RuntimeError, - r'The warning triggered on filename \'(.*)test_warnings.py\', ' - r'line number ([\d]+), is supposed to be shown until version ' - r'(.*) is released. Current version is now ' - r'([\d.]+). Please remove the warning.'): - raise_named_version_warning(_version_info_=(getattr(sys, 'maxint', None) or getattr(sys, 'maxsize'), 16, 0)) - - # Even though we're calling warn_until, we pass _dont_call_warnings - # because we're only after the RuntimeError - with self.assertRaisesRegex( - RuntimeError, - r'The warning triggered on filename \'(.*)test_warnings.py\', ' - r'line number ([\d]+), is supposed to be shown until version ' - r'0.17.0 is released. Current version is now ' - r'(.*). Please remove the warning.'): - warn_until( - (0, 17), 'Foo', _dont_call_warnings=True - ) - - with self.assertRaisesRegex( - RuntimeError, - r'The warning triggered on filename \'(.*)test_warnings.py\', ' - r'line number ([\d]+), is supposed to be shown until version ' - r'(.*) is released. Current version is now ' - r'(.*). Please remove the warning.'): - warn_until( - 'Hydrogen', 'Foo', _dont_call_warnings=True, - _version_info_=(getattr(sys, 'maxint', None) or getattr(sys, 'maxsize'), 16, 0) - ) - - # version on the deprecation message gets properly formatted - with warnings.catch_warnings(record=True) as recorded_warnings: - vrs = SaltStackVersion.from_name('Helium') - warn_until( - 'Helium', 'Deprecation Message until {version}!', - _version_info_=(vrs.major - 1, 0) - ) - self.assertEqual( - 'Deprecation Message until {0}!'.format(vrs.formatted_version), - str(recorded_warnings[0].message) - ) - - def test_kwargs_warn_until_warning_raised(self): - # We *always* want *all* warnings thrown on this module - warnings.filterwarnings('always', '', DeprecationWarning, __name__) - - def raise_warning(**kwargs): - _version_info_ = kwargs.pop('_version_info_', (0, 16, 0)) - kwargs_warn_until( - kwargs, - (0, 17), - _version_info_=_version_info_ - ) - - # raise_warning({...}) should show warning until version info is >= (0, 17) - with warnings.catch_warnings(record=True) as recorded_warnings: - raise_warning(foo=42) # with a kwarg - self.assertEqual( - 'The following parameter(s) have been deprecated and ' - 'will be removed in \'0.17.0\': \'foo\'.', - str(recorded_warnings[0].message) - ) - # With no **kwargs, should not show warning until version info is >= (0, 17) - with warnings.catch_warnings(record=True) as recorded_warnings: - kwargs_warn_until( - {}, # no kwargs - (0, 17), - _version_info_=(0, 16, 0) - ) - self.assertEqual(0, len(recorded_warnings)) - - # Let's set version info to (0, 17), a RuntimeError should be raised - # regardless of whether or not we pass any **kwargs. - with self.assertRaisesRegex( - RuntimeError, - r'The warning triggered on filename \'(.*)test_warnings.py\', ' - r'line number ([\d]+), is supposed to be shown until version ' - r'0.17.0 is released. Current version is now 0.17.0. ' - r'Please remove the warning.'): - raise_warning(_version_info_=(0, 17)) # no kwargs - - with self.assertRaisesRegex( - RuntimeError, - r'The warning triggered on filename \'(.*)test_warnings.py\', ' - r'line number ([\d]+), is supposed to be shown until version ' - r'0.17.0 is released. Current version is now 0.17.0. ' - r'Please remove the warning.'): - raise_warning(bar='baz', qux='quux', _version_info_=(0, 17)) # some kwargs diff --git a/tests/unit/utils/test_which.py b/tests/unit/utils/test_which.py index 9ab674791d4..8788a1658a6 100644 --- a/tests/unit/utils/test_which.py +++ b/tests/unit/utils/test_which.py @@ -9,29 +9,29 @@ from tests.support.unit import skipIf, TestCase from tests.support.mock import NO_MOCK, NO_MOCK_REASON, patch # Import salt libs -import salt.utils +import salt.utils.path @skipIf(NO_MOCK, NO_MOCK_REASON) class TestWhich(TestCase): ''' - Tests salt.utils.which function to ensure that it returns True as + Tests salt.utils.path.which function to ensure that it returns True as expected. ''' # The mock patch below will make sure that ALL calls to the which function # returns None def test_missing_binary_in_linux(self): - with patch('salt.utils.which', lambda exe: None): + with patch('salt.utils.path.which', lambda exe: None): self.assertTrue( - salt.utils.which('this-binary-does-not-exist') is None + salt.utils.path.which('this-binary-does-not-exist') is None ) # The mock patch below will make sure that ALL calls to the which function # return whatever is sent to it def test_existing_binary_in_linux(self): - with patch('salt.utils.which', lambda exe: exe): - self.assertTrue(salt.utils.which('this-binary-exists-under-linux')) + with patch('salt.utils.path.which', lambda exe: exe): + self.assertTrue(salt.utils.path.which('this-binary-exists-under-linux')) def test_existing_binary_in_windows(self): with patch('os.access') as osaccess: @@ -44,18 +44,21 @@ class TestWhich(TestCase): # The second, iterating through $PATH, should also return False, # still checking for Linux False, + # We will now also return False once so we get a .EXE back from + # the function, see PATHEXT below. + False, # Lastly return True, this is the windows check. True ] # Let's patch os.environ to provide a custom PATH variable - with patch.dict(os.environ, {'PATH': '/bin'}): + with patch.dict(os.environ, {'PATH': '/bin', + 'PATHEXT': '.COM;.EXE;.BAT;.CMD'}): # Let's also patch is_windows to return True - with patch('salt.utils.is_windows', lambda: True): + with patch('salt.utils.platform.is_windows', lambda: True): with patch('os.path.isfile', lambda x: True): self.assertEqual( - salt.utils.which('this-binary-exists-under-windows'), - # The returned path should return the .exe suffix - '/bin/this-binary-exists-under-windows.EXE' + salt.utils.path.which('this-binary-exists-under-windows'), + os.path.join('/bin', 'this-binary-exists-under-windows.EXE') ) def test_missing_binary_in_windows(self): @@ -72,11 +75,11 @@ class TestWhich(TestCase): # Let's patch os.environ to provide a custom PATH variable with patch.dict(os.environ, {'PATH': '/bin'}): # Let's also patch is_widows to return True - with patch('salt.utils.is_windows', lambda: True): + with patch('salt.utils.platform.is_windows', lambda: True): self.assertEqual( # Since we're passing the .exe suffix, the last True above # will not matter. The result will be None - salt.utils.which('this-binary-is-missing-in-windows.exe'), + salt.utils.path.which('this-binary-is-missing-in-windows.exe'), None ) @@ -102,10 +105,9 @@ class TestWhich(TestCase): 'PATHEXT': '.COM;.EXE;.BAT;.CMD;.VBS;' '.VBE;.JS;.JSE;.WSF;.WSH;.MSC;.PY'}): # Let's also patch is_windows to return True - with patch('salt.utils.is_windows', lambda: True): + with patch('salt.utils.platform.is_windows', lambda: True): with patch('os.path.isfile', lambda x: True): self.assertEqual( - salt.utils.which('this-binary-exists-under-windows'), - # The returned path should return the .exe suffix - '/bin/this-binary-exists-under-windows.CMD' + salt.utils.path.which('this-binary-exists-under-windows'), + os.path.join('/bin', 'this-binary-exists-under-windows.CMD') ) diff --git a/tests/unit/utils/test_yamlloader.py b/tests/unit/utils/test_yamlloader.py index 69a2724f5d8..dd7870925c6 100644 --- a/tests/unit/utils/test_yamlloader.py +++ b/tests/unit/utils/test_yamlloader.py @@ -5,11 +5,12 @@ # Import python libs from __future__ import absolute_import +import textwrap # Import Salt Libs from yaml.constructor import ConstructorError from salt.utils.yamlloader import SaltYamlSafeLoader -import salt.utils +import salt.utils.files # Import Salt Testing Libs from tests.support.unit import TestCase, skipIf @@ -28,20 +29,19 @@ class YamlLoaderTestCase(TestCase): Takes a YAML string, puts it into a mock file, passes that to the YAML SaltYamlSafeLoader and then returns the rendered/parsed YAML data ''' - with patch('salt.utils.fopen', mock_open(read_data=data)) as mocked_file: - with salt.utils.fopen(mocked_file) as mocked_stream: + with patch('salt.utils.files.fopen', mock_open(read_data=data)) as mocked_file: + with salt.utils.files.fopen(mocked_file) as mocked_stream: return SaltYamlSafeLoader(mocked_stream).get_data() def test_yaml_basics(self): ''' Test parsing an ordinary path ''' - self.assertEqual( - self._render_yaml(b''' -p1: - - alpha - - beta'''), + self._render_yaml(textwrap.dedent('''\ + p1: + - alpha + - beta''')), {'p1': ['alpha', 'beta']} ) @@ -49,38 +49,37 @@ p1: ''' Test YAML anchors ''' - # Simple merge test self.assertEqual( - self._render_yaml(b''' -p1: &p1 - v1: alpha -p2: - <<: *p1 - v2: beta'''), + self._render_yaml(textwrap.dedent('''\ + p1: &p1 + v1: alpha + p2: + <<: *p1 + v2: beta''')), {'p1': {'v1': 'alpha'}, 'p2': {'v1': 'alpha', 'v2': 'beta'}} ) # Test that keys/nodes are overwritten self.assertEqual( - self._render_yaml(b''' -p1: &p1 - v1: alpha -p2: - <<: *p1 - v1: new_alpha'''), + self._render_yaml(textwrap.dedent('''\ + p1: &p1 + v1: alpha + p2: + <<: *p1 + v1: new_alpha''')), {'p1': {'v1': 'alpha'}, 'p2': {'v1': 'new_alpha'}} ) # Test merging of lists self.assertEqual( - self._render_yaml(b''' -p1: &p1 - v1: &v1 - - t1 - - t2 -p2: - v2: *v1'''), + self._render_yaml(textwrap.dedent('''\ + p1: &p1 + v1: &v1 + - t1 + - t2 + p2: + v2: *v1''')), {"p2": {"v2": ["t1", "t2"]}, "p1": {"v1": ["t1", "t2"]}} ) @@ -89,15 +88,27 @@ p2: Test that duplicates still throw an error ''' with self.assertRaises(ConstructorError): - self._render_yaml(b''' -p1: alpha -p1: beta''') + self._render_yaml(textwrap.dedent('''\ + p1: alpha + p1: beta''')) with self.assertRaises(ConstructorError): - self._render_yaml(b''' -p1: &p1 - v1: alpha -p2: - <<: *p1 - v2: beta - v2: betabeta''') + self._render_yaml(textwrap.dedent('''\ + p1: &p1 + v1: alpha + p2: + <<: *p1 + v2: beta + v2: betabeta''')) + + def test_yaml_with_unicode_literals(self): + ''' + Test proper loading of unicode literals + ''' + self.assertEqual( + self._render_yaml(textwrap.dedent('''\ + foo: + a: Д + b: {'a': u'\\u0414'}''')), + {'foo': {'a': u'\u0414', 'b': {'a': u'\u0414'}}} + ) diff --git a/tests/unit/utils/vmware_test/__init__.py b/tests/unit/utils/vmware/__init__.py similarity index 100% rename from tests/unit/utils/vmware_test/__init__.py rename to tests/unit/utils/vmware/__init__.py diff --git a/tests/unit/utils/vmware_test/test_cluster.py b/tests/unit/utils/vmware/test_cluster.py similarity index 91% rename from tests/unit/utils/vmware_test/test_cluster.py rename to tests/unit/utils/vmware/test_cluster.py index daf157709c3..9a628c9b36e 100644 --- a/tests/unit/utils/vmware_test/test_cluster.py +++ b/tests/unit/utils/vmware/test_cluster.py @@ -178,6 +178,18 @@ class CreateClusterTestCase(TestCase): self.mock_create_cluster_ex.assert_called_once_with( 'fake_cluster', self.mock_cluster_spec) + def test_create_cluster_raise_no_permission(self): + exc = vim.fault.NoPermission() + exc.privilegeId = 'Fake privilege' + self.mock_dc.hostFolder.CreateClusterEx = MagicMock( + side_effect=exc) + with self.assertRaises(VMwareApiError) as excinfo: + vmware.create_cluster(self.mock_dc, 'fake_cluster', + self.mock_cluster_spec) + self.assertEqual(excinfo.exception.strerror, + 'Not enough permissions. Required privilege: ' + 'Fake privilege') + def test_create_cluster_raise_vim_fault(self): exc = vim.fault.VimFault() exc.msg = 'VimFault msg' @@ -234,6 +246,17 @@ class UpdateClusterTestCase(TestCase): self.mock_reconfigure_compute_resource_task.assert_called_once_with( self.mock_cluster_spec, modify=True) + def test_reconfigure_compute_resource_raise_no_permission(self): + exc = vim.fault.NoPermission() + exc.privilegeId = 'Fake privilege' + self.mock_cluster.ReconfigureComputeResource_Task = \ + MagicMock(side_effect=exc) + with self.assertRaises(VMwareApiError) as excinfo: + vmware.update_cluster(self.mock_cluster, self.mock_cluster_spec) + self.assertEqual(excinfo.exception.strerror, + 'Not enough permissions. Required privilege: ' + 'Fake privilege') + def test_reconfigure_compute_resource_raise_vim_fault(self): exc = vim.fault.VimFault() exc.msg = 'VimFault msg' diff --git a/tests/unit/utils/vmware_test/test_common.py b/tests/unit/utils/vmware/test_common.py similarity index 84% rename from tests/unit/utils/vmware_test/test_common.py rename to tests/unit/utils/vmware/test_common.py index 0de56813e5e..5a946e8aa93 100644 --- a/tests/unit/utils/vmware_test/test_common.py +++ b/tests/unit/utils/vmware/test_common.py @@ -43,6 +43,19 @@ class WaitForTaskTestCase(TestCase): patcher.start() self.addCleanup(patcher.stop) + def test_first_task_info_raise_no_permission(self): + exc = vim.fault.NoPermission() + exc.privilegeId = 'Fake privilege' + mock_task = MagicMock() + type(mock_task).info = PropertyMock(side_effect=exc) + with self.assertRaises(excs.VMwareApiError) as excinfo: + salt.utils.vmware.wait_for_task(mock_task, + 'fake_instance_name', + 'task_type') + self.assertEqual(excinfo.exception.strerror, + 'Not enough permissions. Required privilege: ' + 'Fake privilege') + def test_first_task_info_raise_vim_fault(self): exc = vim.fault.VimFault() exc.msg = 'VimFault msg' @@ -65,6 +78,22 @@ class WaitForTaskTestCase(TestCase): 'task_type') self.assertEqual(excinfo.exception.strerror, 'RuntimeFault msg') + def test_inner_loop_task_info_raise_no_permission(self): + exc = vim.fault.NoPermission() + exc.privilegeId = 'Fake privilege' + mock_task = MagicMock() + mock_info1 = MagicMock() + type(mock_task).info = PropertyMock( + side_effect=[mock_info1, exc]) + type(mock_info1).state = PropertyMock(side_effect=['running', 'bad']) + with self.assertRaises(excs.VMwareApiError) as excinfo: + salt.utils.vmware.wait_for_task(mock_task, + 'fake_instance_name', + 'task_type') + self.assertEqual(excinfo.exception.strerror, + 'Not enough permissions. Required privilege: ' + 'Fake privilege') + def test_inner_loop_task_info_raise_vim_fault(self): exc = vim.fault.VimFault() exc.msg = 'VimFault msg' @@ -161,6 +190,22 @@ class WaitForTaskTestCase(TestCase): 'task_type') self.assertEqual(str(excinfo.exception), 'error exc') + def test_info_error_no_permission(self): + exc = vim.fault.NoPermission() + exc.privilegeId = 'Fake privilege' + mock_task = MagicMock() + prop_mock_state = PropertyMock(return_value='error') + prop_mock_error = PropertyMock(side_effect=exc) + type(mock_task.info).state = prop_mock_state + type(mock_task.info).error = prop_mock_error + with self.assertRaises(excs.VMwareApiError) as excinfo: + salt.utils.vmware.wait_for_task(mock_task, + 'fake_instance_name', + 'task_type') + self.assertEqual(excinfo.exception.strerror, + 'Not enough permissions. Required privilege: ' + 'Fake privilege') + def test_info_error_vim_fault(self): exc = vim.fault.VimFault() exc.msg = 'VimFault msg' @@ -502,6 +547,8 @@ class GetPropertiesOfManagedObjectTestCase(TestCase): 'retrieved', excinfo.exception.strerror) +@skipIf(NO_MOCK, NO_MOCK_REASON) +@skipIf(not HAS_PYVMOMI, 'The \'pyvmomi\' library is missing') class GetManagedObjectName(TestCase): '''Tests for salt.utils.get_managed_object_name''' @@ -658,6 +705,19 @@ class GetContentTestCase(TestCase): # check destroy is called self.assertEqual(self.destroy_mock.call_count, 1) + def test_create_container_view_raise_no_permission(self): + exc = vim.fault.NoPermission() + exc.privilegeId = 'Fake privilege' + self.si_mock.content.viewManager.CreateContainerView = \ + MagicMock(side_effect=exc) + with patch('salt.utils.vmware.get_root_folder', + self.get_root_folder_mock): + with self.assertRaises(excs.VMwareApiError) as excinfo: + salt.utils.vmware.get_content(self.si_mock, self.obj_type_mock) + self.assertEqual(excinfo.exception.strerror, + 'Not enough permissions. Required privilege: ' + 'Fake privilege') + def test_create_container_view_raise_vim_fault(self): exc = vim.fault.VimFault() exc.msg = 'VimFault msg' @@ -680,6 +740,19 @@ class GetContentTestCase(TestCase): salt.utils.vmware.get_content(self.si_mock, self.obj_type_mock) self.assertEqual(excinfo.exception.strerror, 'RuntimeFault msg') + def test_destroy_raise_no_permission(self): + exc = vim.fault.NoPermission() + exc.privilegeId = 'Fake privilege' + self.si_mock.content.viewManager.CreateContainerView = MagicMock( + return_value=MagicMock(Destroy=MagicMock(side_effect=exc))) + with patch('salt.utils.vmware.get_root_folder', + self.get_root_folder_mock): + with self.assertRaises(excs.VMwareApiError) as excinfo: + salt.utils.vmware.get_content(self.si_mock, self.obj_type_mock) + self.assertEqual(excinfo.exception.strerror, + 'Not enough permissions. Required privilege: ' + 'Fake privilege') + def test_destroy_raise_vim_fault(self): exc = vim.fault.VimFault() exc.msg = 'VimFault msg' @@ -745,6 +818,17 @@ class GetContentTestCase(TestCase): [self.filter_spec_ret_mock]) self.assertEqual(ret, self.result_mock) + def test_retrieve_contents_raise_no_permission(self): + exc = vim.fault.NoPermission() + exc.privilegeId = 'Fake privilege' + self.si_mock.content.propertyCollector.RetrieveContents = \ + MagicMock(side_effect=exc) + with self.assertRaises(excs.VMwareApiError) as excinfo: + salt.utils.vmware.get_content(self.si_mock, self.obj_type_mock) + self.assertEqual(excinfo.exception.strerror, + 'Not enough permissions. Required privilege: ' + 'Fake privilege') + def test_retrieve_contents_raise_vim_fault(self): exc = vim.fault.VimFault() exc.msg = 'VimFault msg' @@ -789,6 +873,16 @@ class GetRootFolderTestCase(TestCase): self.mock_si = MagicMock( RetrieveContent=MagicMock(return_value=self.mock_content)) + def test_raise_no_permission(self): + exc = vim.fault.NoPermission() + exc.privilegeId = 'Fake privilege' + type(self.mock_content).rootFolder = PropertyMock(side_effect=exc) + with self.assertRaises(excs.VMwareApiError) as excinfo: + salt.utils.vmware.get_root_folder(self.mock_si) + self.assertEqual(excinfo.exception.strerror, + 'Not enough permissions. Required privilege: ' + 'Fake privilege') + def test_raise_vim_fault(self): exc = vim.fault.VimFault() exc.msg = 'VimFault msg' @@ -808,3 +902,51 @@ class GetRootFolderTestCase(TestCase): def test_return(self): ret = salt.utils.vmware.get_root_folder(self.mock_si) self.assertEqual(ret, self.mock_root_folder) + + +@skipIf(NO_MOCK, NO_MOCK_REASON) +@skipIf(not HAS_PYVMOMI, 'The \'pyvmomi\' library is missing') +class GetServiceInfoTestCase(TestCase): + '''Tests for salt.utils.vmware.get_service_info''' + def setUp(self): + self.mock_about = MagicMock() + self.mock_si = MagicMock(content=MagicMock()) + type(self.mock_si.content).about = \ + PropertyMock(return_value=self.mock_about) + + def tearDown(self): + for attr in ('mock_si', 'mock_about'): + delattr(self, attr) + + def test_about_ret(self): + ret = salt.utils.vmware.get_service_info(self.mock_si) + self.assertEqual(ret, self.mock_about) + + def test_about_raises_no_permission(self): + exc = vim.fault.NoPermission() + exc.privilegeId = 'Fake privilege' + type(self.mock_si.content).about = \ + PropertyMock(side_effect=exc) + with self.assertRaises(excs.VMwareApiError) as excinfo: + salt.utils.vmware.get_service_info(self.mock_si) + self.assertEqual(excinfo.exception.strerror, + 'Not enough permissions. Required privilege: ' + 'Fake privilege') + + def test_about_raises_vim_fault(self): + exc = vim.fault.VimFault() + exc.msg = 'VimFault msg' + type(self.mock_si.content).about = \ + PropertyMock(side_effect=exc) + with self.assertRaises(excs.VMwareApiError) as excinfo: + salt.utils.vmware.get_service_info(self.mock_si) + self.assertEqual(excinfo.exception.strerror, 'VimFault msg') + + def test_about_raises_runtime_fault(self): + exc = vmodl.RuntimeFault() + exc.msg = 'RuntimeFault msg' + type(self.mock_si.content).about = \ + PropertyMock(side_effect=exc) + with self.assertRaises(excs.VMwareRuntimeError) as excinfo: + salt.utils.vmware.get_service_info(self.mock_si) + self.assertEqual(excinfo.exception.strerror, 'RuntimeFault msg') diff --git a/tests/unit/utils/vmware_test/test_connection.py b/tests/unit/utils/vmware/test_connection.py similarity index 93% rename from tests/unit/utils/vmware_test/test_connection.py rename to tests/unit/utils/vmware/test_connection.py index 812cb1231c2..4a95e9b67fc 100644 --- a/tests/unit/utils/vmware_test/test_connection.py +++ b/tests/unit/utils/vmware/test_connection.py @@ -21,7 +21,7 @@ import salt.exceptions as excs # Import Salt libraries import salt.utils.vmware # Import Third Party Libs -import salt.ext.six as six +from salt.ext import six try: from pyVmomi import vim, vmodl @@ -40,6 +40,11 @@ if sys.version_info[:3] > (2, 7, 8): else: SSL_VALIDATION = False +if hasattr(ssl, '_create_unverified_context'): + ssl_context = 'ssl._create_unverified_context' +else: + ssl_context = 'ssl._create_stdlib_context' + # Get Logging Started log = logging.getLogger(__name__) @@ -337,14 +342,14 @@ class PrivateGetServiceInstanceTestCase(TestCase): @skipIf(not SSL_VALIDATION, 'SSL validation is not enabled') def test_second_attempt_successful_connection(self): with patch('ssl.SSLContext', MagicMock()), \ - patch('ssl._create_unverified_context', MagicMock()): + patch(ssl_context, MagicMock()): exc = vim.fault.HostConnectFault() exc.msg = '[SSL: CERTIFICATE_VERIFY_FAILED]' mock_sc = MagicMock(side_effect=[exc, None]) mock_ssl = MagicMock() with patch('salt.utils.vmware.SmartConnect', mock_sc): - with patch('ssl._create_unverified_context', + with patch(ssl_context, mock_ssl): salt.utils.vmware._get_service_instance( @@ -378,7 +383,7 @@ class PrivateGetServiceInstanceTestCase(TestCase): @skipIf(not SSL_VALIDATION, 'SSL validation is not enabled') def test_third_attempt_successful_connection(self): with patch('ssl.SSLContext', MagicMock()), \ - patch('ssl._create_unverified_context', MagicMock()): + patch(ssl_context, MagicMock()): exc = vim.fault.HostConnectFault() exc.msg = '[SSL: CERTIFICATE_VERIFY_FAILED]' exc2 = Exception('certificate verify failed') @@ -387,9 +392,7 @@ class PrivateGetServiceInstanceTestCase(TestCase): mock_ssl_context = MagicMock() with patch('salt.utils.vmware.SmartConnect', mock_sc): - with patch('ssl._create_unverified_context', - mock_ssl_unverif): - + with patch(ssl_context, mock_ssl_unverif): with patch('ssl.SSLContext', mock_ssl_context): salt.utils.vmware._get_service_instance( @@ -473,7 +476,7 @@ class PrivateGetServiceInstanceTestCase(TestCase): @skipIf(not SSL_VALIDATION, 'SSL validation is not enabled') def test_second_attempt_unsuccsessful_connection_default_error(self): with patch('ssl.SSLContext', MagicMock()), \ - patch('ssl._create_unverified_context', MagicMock()): + patch(ssl_context, MagicMock()): exc = vim.fault.HostConnectFault() exc.msg = '[SSL: CERTIFICATE_VERIFY_FAILED]' exc2 = Exception('Exception') @@ -498,7 +501,7 @@ class PrivateGetServiceInstanceTestCase(TestCase): @skipIf(not SSL_VALIDATION, 'SSL validation is not enabled') def test_second_attempt_unsuccsessful_connection_vim_fault(self): with patch('ssl.SSLContext', MagicMock()), \ - patch('ssl._create_unverified_context', MagicMock()): + patch(ssl_context, MagicMock()): exc = vim.fault.HostConnectFault() exc.msg = '[SSL: CERTIFICATE_VERIFY_FAILED]' exc2 = vim.fault.VimFault() @@ -523,7 +526,7 @@ class PrivateGetServiceInstanceTestCase(TestCase): @skipIf(not SSL_VALIDATION, 'SSL validation is not enabled') def test_third_attempt_unsuccessful_connection_detault_error(self): with patch('ssl.SSLContext', MagicMock()), \ - patch('ssl._create_unverified_context', MagicMock()): + patch(ssl_context, MagicMock()): exc = vim.fault.HostConnectFault() exc.msg = '[SSL: CERTIFICATE_VERIFY_FAILED]' exc2 = Exception('certificate verify failed') @@ -548,7 +551,7 @@ class PrivateGetServiceInstanceTestCase(TestCase): @skipIf(not SSL_VALIDATION, 'SSL validation is not enabled') def test_third_attempt_unsuccessful_connection_vim_fault(self): with patch('ssl.SSLContext', MagicMock()), \ - patch('ssl._create_unverified_context', MagicMock()): + patch(ssl_context, MagicMock()): exc = vim.fault.HostConnectFault() exc.msg = '[SSL: CERTIFICATE_VERIFY_FAILED]' exc2 = Exception('certificate verify failed') @@ -597,7 +600,7 @@ class GetServiceInstanceTestCase(TestCase): None) def test_no_cached_service_instance_same_host_on_proxy(self): - with patch('salt.utils.is_proxy', MagicMock(return_value=True)): + with patch('salt.utils.platform.is_proxy', MagicMock(return_value=True)): # Service instance is uncached when using class default mock objs mock_get_si = MagicMock() with patch('salt.utils.vmware._get_service_instance', mock_get_si): @@ -688,6 +691,26 @@ class GetServiceInstanceTestCase(TestCase): self.assertEqual(mock_disconnect.call_count, 1) self.assertEqual(mock_get_si.call_count, 2) + def test_current_time_raise_no_permission(self): + exc = vim.fault.NoPermission() + exc.privilegeId = 'Fake privilege' + with patch('salt.utils.vmware._get_service_instance', + MagicMock(return_value=MagicMock( + CurrentTime=MagicMock(side_effect=exc)))): + with self.assertRaises(excs.VMwareApiError) as excinfo: + salt.utils.vmware.get_service_instance( + host='fake_host', + username='fake_username', + password='fake_password', + protocol='fake_protocol', + port=1, + mechanism='fake_mechanism', + principal='fake_principal', + domain='fake_domain') + self.assertEqual(excinfo.exception.strerror, + 'Not enough permissions. Required privilege: ' + 'Fake privilege') + def test_current_time_raise_vim_fault(self): exc = vim.fault.VimFault() exc.msg = 'VimFault msg' @@ -741,6 +764,17 @@ class DisconnectTestCase(TestCase): service_instance=self.mock_si) mock_disconnect.assert_called_once_with(self.mock_si) + def test_disconnect_raise_no_permission(self): + exc = vim.fault.NoPermission() + exc.privilegeId = 'Fake privilege' + with patch('salt.utils.vmware.Disconnect', MagicMock(side_effect=exc)): + with self.assertRaises(excs.VMwareApiError) as excinfo: + salt.utils.vmware.disconnect( + service_instance=self.mock_si) + self.assertEqual(excinfo.exception.strerror, + 'Not enough permissions. Required privilege: ' + 'Fake privilege') + def test_disconnect_raise_vim_fault(self): exc = vim.fault.VimFault() exc.msg = 'VimFault msg' @@ -765,6 +799,17 @@ class DisconnectTestCase(TestCase): class IsConnectionToAVCenterTestCase(TestCase): '''Tests for salt.utils.vmware.is_connection_to_a_vcenter''' + def test_api_type_raise_no_permission(self): + exc = vim.fault.NoPermission() + exc.privilegeId = 'Fake privilege' + mock_si = MagicMock() + type(mock_si.content.about).apiType = PropertyMock(side_effect=exc) + with self.assertRaises(excs.VMwareApiError) as excinfo: + salt.utils.vmware.is_connection_to_a_vcenter(mock_si) + self.assertEqual(excinfo.exception.strerror, + 'Not enough permissions. Required privilege: ' + 'Fake privilege') + def test_api_type_raise_vim_fault(self): exc = vim.fault.VimFault() exc.msg = 'VimFault msg' diff --git a/tests/unit/utils/vmware_test/test_datacenter.py b/tests/unit/utils/vmware/test_datacenter.py similarity index 92% rename from tests/unit/utils/vmware_test/test_datacenter.py rename to tests/unit/utils/vmware/test_datacenter.py index 86d1f0995a3..c8045fb5667 100644 --- a/tests/unit/utils/vmware_test/test_datacenter.py +++ b/tests/unit/utils/vmware/test_datacenter.py @@ -164,6 +164,19 @@ class CreateDatacenterTestCase(TestCase): vmware.create_datacenter(self.mock_si, 'fake_dc') self.mock_create_datacenter.assert_called_once_with('fake_dc') + def test_create_datacenter_raise_no_permission(self): + exc = vim.fault.NoPermission() + exc.privilegeId = 'Fake privilege' + self.mock_root_folder = MagicMock( + CreateDatacenter=MagicMock(side_effect=exc)) + with patch('salt.utils.vmware.get_root_folder', + MagicMock(return_value=self.mock_root_folder)): + with self.assertRaises(VMwareApiError) as excinfo: + vmware.create_datacenter(self.mock_si, 'fake_dc') + self.assertEqual(excinfo.exception.strerror, + 'Not enough permissions. Required privilege: ' + 'Fake privilege') + def test_create_datacenter_raise_vim_fault(self): exc = vim.VimFault() exc.msg = 'VimFault msg' diff --git a/tests/unit/utils/vmware_test/test_host.py b/tests/unit/utils/vmware/test_host.py similarity index 100% rename from tests/unit/utils/vmware_test/test_host.py rename to tests/unit/utils/vmware/test_host.py diff --git a/tests/zypp_plugin.py b/tests/zypp_plugin.py new file mode 100644 index 00000000000..ce949f4a48b --- /dev/null +++ b/tests/zypp_plugin.py @@ -0,0 +1,65 @@ +# -*- coding: utf-8 -*- +''' +Related to zypp_plugins_test.py module. +''' + + +class Plugin(object): + ''' + Bogus module for Zypp Plugins tests. + ''' + def ack(self): + ''' + Acknowledge that the plugin had finished the transaction + Returns: + + ''' + + def main(self): + ''' + Register plugin + Returns: + + ''' + + +class BogusIO(object): + ''' + Read/write logger. + ''' + + def __init__(self): + self.content = list() + self.closed = False + + def __str__(self): + return '\n'.join(self.content) + + def __call__(self, *args, **kwargs): + self.path, self.mode = args + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + + def __enter__(self): + return self + + def write(self, data): + ''' + Simulate writing data + Args: + data: + + Returns: + + ''' + self.content.append(data) + + def close(self): + ''' + Simulate closing the IO object. + Returns: + + ''' + self.closed = True