diff --git a/doc/ref/renderers/index.rst b/doc/ref/renderers/index.rst index 8399f7f6049..308145f3894 100644 --- a/doc/ref/renderers/index.rst +++ b/doc/ref/renderers/index.rst @@ -144,17 +144,14 @@ Here is a simple YAML renderer example: .. code-block:: python - import yaml + import salt.utils.yaml from salt.utils.yamlloader import SaltYamlSafeLoader from salt.ext import six def render(yaml_data, saltenv='', sls='', **kws): if not isinstance(yaml_data, six.string_types): yaml_data = yaml_data.read() - data = yaml.load( - yaml_data, - Loader=SaltYamlSafeLoader - ) + data = salt.utils.yaml.safe_load(yaml_data) return data if data else {} Full List of Renderers diff --git a/salt/cloud/__init__.py b/salt/cloud/__init__.py index 5da1db28370..5ee655849de 100644 --- a/salt/cloud/__init__.py +++ b/salt/cloud/__init__.py @@ -37,6 +37,7 @@ import salt.utils.data import salt.utils.dictupdate import salt.utils.files import salt.utils.verify +import salt.utils.yaml import salt.syspaths from salt.template import compile_template @@ -48,7 +49,6 @@ except ImportError: import Crypto.Random except ImportError: pass # pycrypto < 2.1 -import yaml from salt.ext import six from salt.ext.six.moves import input # pylint: disable=import-error,redefined-builtin @@ -1393,7 +1393,7 @@ class Cloud(object): try: with salt.utils.files.fopen(self.opts['conf_file'], 'r') as mcc: - main_cloud_config = yaml.safe_load(mcc) + main_cloud_config = salt.utils.yaml.safe_load(mcc) if not main_cloud_config: main_cloud_config = {} except KeyError: diff --git a/salt/cloud/clouds/azurearm.py b/salt/cloud/clouds/azurearm.py index 835495b420f..93851152529 100644 --- a/salt/cloud/clouds/azurearm.py +++ b/salt/cloud/clouds/azurearm.py @@ -59,13 +59,13 @@ import time import logging import pprint import base64 -import yaml import collections import salt.cache import salt.config as config import salt.utils.cloud import salt.utils.data import salt.utils.files +import salt.utils.yaml from salt.utils.versions import LooseVersion from salt.ext import six import salt.version @@ -1070,7 +1070,7 @@ def request_instance(call=None, kwargs=None): # pylint: disable=unused-argument ) if isinstance(kwargs.get('volumes'), six.string_types): - volumes = yaml.safe_load(kwargs['volumes']) + volumes = salt.utils.yaml.safe_load(kwargs['volumes']) else: volumes = kwargs.get('volumes') diff --git a/salt/cloud/clouds/ec2.py b/salt/cloud/clouds/ec2.py index acf7b2eefda..5ca57f7b30a 100644 --- a/salt/cloud/clouds/ec2.py +++ b/salt/cloud/clouds/ec2.py @@ -75,7 +75,6 @@ import time import uuid import pprint import logging -import yaml # Import libs for talking to the EC2 API import hmac @@ -92,6 +91,7 @@ import salt.utils.cloud import salt.utils.files import salt.utils.hashutils import salt.utils.json +import salt.utils.yaml from salt._compat import ElementTree as ET import salt.utils.http as http import salt.utils.aws as aws @@ -2840,7 +2840,7 @@ def create_attach_volumes(name, kwargs, call=None, wait_to_finish=True): kwargs['instance_id'] = _get_node(name)['instanceId'] if isinstance(kwargs['volumes'], six.string_types): - volumes = yaml.safe_load(kwargs['volumes']) + volumes = salt.utils.yaml.safe_load(kwargs['volumes']) else: volumes = kwargs['volumes'] @@ -4109,7 +4109,7 @@ def create_volume(kwargs=None, call=None, wait_to_finish=False): # Allow tags to be set upon creation if 'tags' in kwargs: if isinstance(kwargs['tags'], six.string_types): - tags = yaml.safe_load(kwargs['tags']) + tags = salt.utils.yaml.safe_load(kwargs['tags']) else: tags = kwargs['tags'] diff --git a/salt/cloud/clouds/joyent.py b/salt/cloud/clouds/joyent.py index d6deb94d361..c8c995fc241 100644 --- a/salt/cloud/clouds/joyent.py +++ b/salt/cloud/clouds/joyent.py @@ -57,7 +57,6 @@ import logging import base64 import pprint import inspect -import yaml import datetime from Crypto.Hash import SHA256 from Crypto.PublicKey import RSA @@ -70,6 +69,7 @@ import salt.utils.cloud import salt.utils.files import salt.utils.http import salt.utils.json +import salt.utils.yaml import salt.config as config from salt.cloud.libcloudfuncs import node_state from salt.exceptions import ( @@ -1121,6 +1121,6 @@ def query(action=None, if 'Content-Length' in result['headers']: content = result['text'] - return_content = yaml.safe_load(content) + return_content = salt.utils.yaml.safe_load(content) return [result['status'], return_content] diff --git a/salt/cloud/clouds/msazure.py b/salt/cloud/clouds/msazure.py index 31f7186177a..b58e7e0ab9e 100644 --- a/salt/cloud/clouds/msazure.py +++ b/salt/cloud/clouds/msazure.py @@ -44,12 +44,12 @@ import copy import logging import pprint import time -import yaml # Import salt libs import salt.config as config from salt.exceptions import SaltCloudSystemExit import salt.utils.cloud +import salt.utils.yaml # Import 3rd-party libs from salt.ext import six @@ -711,7 +711,7 @@ def create_attach_volumes(name, kwargs, call=None, wait_to_finish=True): kwargs = {} if isinstance(kwargs['volumes'], six.string_types): - volumes = yaml.safe_load(kwargs['volumes']) + volumes = salt.utils.yaml.safe_load(kwargs['volumes']) else: volumes = kwargs['volumes'] @@ -805,7 +805,7 @@ def create_attach_volumes(name, kwargs, call=None, wait_to_finish=True): kwargs = {} if isinstance(kwargs['volumes'], six.string_types): - volumes = yaml.safe_load(kwargs['volumes']) + volumes = salt.utils.yaml.safe_load(kwargs['volumes']) else: volumes = kwargs['volumes'] @@ -2731,7 +2731,7 @@ def set_storage_container_metadata(kwargs=None, storage_conn=None, call=None): if 'name' not in kwargs: raise SaltCloudSystemExit('An storage container name must be specified as "name"') - x_ms_meta_name_values = yaml.safe_load( + x_ms_meta_name_values = salt.utils.yaml.safe_load( kwargs.get('meta_name_values', '') ) diff --git a/salt/cloud/clouds/nova.py b/salt/cloud/clouds/nova.py index d73fa610af5..3cbbb9ea55c 100644 --- a/salt/cloud/clouds/nova.py +++ b/salt/cloud/clouds/nova.py @@ -206,13 +206,13 @@ import os import logging import socket import pprint -import yaml # Import Salt Libs from salt.ext import six import salt.utils.cloud import salt.utils.files import salt.utils.pycrypto +import salt.utils.yaml import salt.client from salt.utils.openstack import nova try: @@ -1276,7 +1276,7 @@ def volume_create_attach(name, call=None, **kwargs): ) if type(kwargs['volumes']) is str: - volumes = yaml.safe_load(kwargs['volumes']) + volumes = salt.utils.yaml.safe_load(kwargs['volumes']) else: volumes = kwargs['volumes'] diff --git a/salt/config/__init__.py b/salt/config/__init__.py index 950c3d3cbd3..545a816adb5 100644 --- a/salt/config/__init__.py +++ b/salt/config/__init__.py @@ -15,7 +15,6 @@ import time import codecs import logging import types -import yaml from copy import deepcopy # pylint: disable=import-error,no-name-in-module @@ -34,6 +33,7 @@ import salt.utils.stringutils import salt.utils.user import salt.utils.validate.path import salt.utils.xdg +import salt.utils.yaml import salt.utils.yamlloader as yamlloader import salt.utils.zeromq import salt.syspaths @@ -2098,11 +2098,8 @@ def _read_conf_file(path): log.debug('Reading configuration from {0}'.format(path)) with salt.utils.files.fopen(path, 'r') as conf_file: try: - conf_opts = yamlloader.load( - conf_file.read(), - Loader=yamlloader.SaltYamlSafeLoader, - ) or {} - except yaml.YAMLError as err: + conf_opts = salt.utils.yaml.safe_load(conf_file) or {} + except salt.utils.yaml.YAMLError as err: message = 'Error parsing configuration file: {0} - {1}'.format(path, err) log.error(message) raise salt.exceptions.SaltConfigurationError(message) diff --git a/salt/engines/slack.py b/salt/engines/slack.py index 9a36da09308..55e74cad1f2 100644 --- a/salt/engines/slack.py +++ b/salt/engines/slack.py @@ -99,7 +99,6 @@ import logging import time import re import traceback -import yaml log = logging.getLogger(__name__) @@ -119,7 +118,7 @@ import salt.utils.event import salt.utils.http import salt.utils.json import salt.utils.slack -from salt.utils.yamldumper import OrderedDumper +import salt.utils.yaml __virtualname__ = 'slack' @@ -373,7 +372,7 @@ class SlackClient(object): # Convert UTF to string _text = salt.utils.json.dumps(_text) - _text = yaml.safe_load(_text) + _text = salt.utils.yaml.safe_load(_text) if not _text: raise ValueError('_text has no value') @@ -561,7 +560,7 @@ class SlackClient(object): ''' Print out YAML using the block mode ''' - params = dict(Dumper=OrderedDumper) + params = {} if 'output_indent' not in __opts__: # default indentation params.update(default_flow_style=False) @@ -573,7 +572,7 @@ class SlackClient(object): params.update(default_flow_style=True, indent=0) try: - #return yaml.dump(data, **params).replace("\n\n", "\n") + #return salt.utils.yaml.safe_dump(data, **params).replace("\n\n", "\n") return salt.utils.json.dumps(data, sort_keys=True, indent=1) # pylint: disable=broad-except except Exception as exc: @@ -627,7 +626,7 @@ class SlackClient(object): # pylint is tripping # pylint: disable=missing-whitespace-after-comma job_data = salt.utils.json.dumps({key:val['return'] for key, val in jid_result.items()}) - results[jid] = yaml.load(job_data) + results[jid] = salt.utils.yaml.safe_load(job_data) return results @@ -692,7 +691,7 @@ class SlackClient(object): content=return_text) # Handle unicode return log.debug('Got back {} via the slack client'.format(r)) - resp = yaml.safe_load(salt.utils.json.dumps(r)) + resp = salt.utils.yaml.safe_load(salt.utils.json.dumps(r)) if 'ok' in resp and resp['ok'] is False: this_job['channel'].send_message('Error: {0}'.format(resp['error'])) del outstanding[jid] diff --git a/salt/grains/extra.py b/salt/grains/extra.py index 80c2a2468f1..3039cd0a287 100644 --- a/salt/grains/extra.py +++ b/salt/grains/extra.py @@ -6,12 +6,12 @@ from __future__ import absolute_import import os # Import third party libs -import yaml import logging # Import salt libs import salt.utils.files import salt.utils.platform +import salt.utils.yaml __proxyenabled__ = ['*'] log = logging.getLogger(__name__) @@ -69,7 +69,7 @@ def config(): log.debug('Loading static grains from %s', gfn) with salt.utils.files.fopen(gfn, 'rb') as fp_: try: - return yaml.safe_load(fp_.read()) + return salt.utils.yaml.safe_load(fp_) except Exception: log.warning("Bad syntax in grains file! Skipping.") return {} diff --git a/salt/minion.py b/salt/minion.py index 554fb5560bf..6645acaedcb 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -786,6 +786,7 @@ class SMinion(MinionBase): ''' def __init__(self, opts): # Late setup of the opts grains, so we can log from the grains module + import salt.loader opts['grains'] = salt.loader.grains(opts) super(SMinion, self).__init__(opts) @@ -803,8 +804,7 @@ class SMinion(MinionBase): # If configured, cache pillar data on the minion if self.opts['file_client'] == 'remote' and self.opts.get('minion_pillar_cache', False): - import yaml - from salt.utils.yamldumper import SafeOrderedDumper + import salt.utils.yaml pdir = os.path.join(self.opts['cachedir'], 'pillar') if not os.path.isdir(pdir): os.makedirs(pdir, 0o700) @@ -815,21 +815,11 @@ class SMinion(MinionBase): penv = 'base' cache_top = {penv: {self.opts['id']: ['cache']}} with salt.utils.files.fopen(ptop, 'wb') as fp_: - fp_.write( - yaml.dump( - cache_top, - Dumper=SafeOrderedDumper - ) - ) + salt.utils.yaml.safe_dump(cache_top, fp_) os.chmod(ptop, 0o600) cache_sls = os.path.join(pdir, 'cache.sls') with salt.utils.files.fopen(cache_sls, 'wb') as fp_: - fp_.write( - yaml.dump( - self.opts['pillar'], - Dumper=SafeOrderedDumper - ) - ) + salt.utils.yaml.safe_dump(self.opts['pillar'], fp_) os.chmod(cache_sls, 0o600) def gen_modules(self, initial_load=False): diff --git a/salt/modules/ansiblegate.py b/salt/modules/ansiblegate.py index 16687c31142..b61180cdbff 100644 --- a/salt/modules/ansiblegate.py +++ b/salt/modules/ansiblegate.py @@ -33,13 +33,13 @@ import os import sys import logging import importlib -import yaml import fnmatch import subprocess import salt.utils.json from salt.exceptions import LoaderError, CommandExecutionError -from salt.utils import timed_subprocess +import salt.utils.timed_subprocess +import salt.utils.yaml try: import ansible @@ -153,11 +153,13 @@ class AnsibleModuleCaller(object): js_args = str('{{"ANSIBLE_MODULE_ARGS": {args}}}') # future lint: disable=blacklisted-function js_args = js_args.format(args=salt.utils.json.dumps(kwargs)) - proc_out = timed_subprocess.TimedProc(["echo", "{0}".format(js_args)], - stdout=subprocess.PIPE, timeout=self.timeout) + proc_out = salt.utils.timed_subprocess.TimedProc( + ["echo", "{0}".format(js_args)], + stdout=subprocess.PIPE, timeout=self.timeout) proc_out.run() - proc_exc = timed_subprocess.TimedProc(['python', module.__file__], - stdin=proc_out.stdout, stdout=subprocess.PIPE, timeout=self.timeout) + proc_exc = salt.utils.timed_subprocess.TimedProc( + ['python', module.__file__], + stdin=proc_out.stdout, stdout=subprocess.PIPE, timeout=self.timeout) proc_exc.run() try: @@ -244,7 +246,7 @@ def help(module=None, *args): ret = {} for docset in module.DOCUMENTATION.split('---'): try: - docset = yaml.load(docset) + docset = salt.utils.yaml.safe_load(docset) if docset: doc.update(docset) except Exception as err: diff --git a/salt/modules/aptpkg.py b/salt/modules/aptpkg.py index 4224d5b55e7..8de29023c81 100644 --- a/salt/modules/aptpkg.py +++ b/salt/modules/aptpkg.py @@ -25,7 +25,6 @@ import logging import time # Import third party libs -import yaml # pylint: disable=no-name-in-module,import-error,redefined-builtin from salt.ext import six from salt.ext.six.moves.urllib.error import HTTPError @@ -47,6 +46,7 @@ import salt.utils.pkg import salt.utils.pkg.deb import salt.utils.systemd import salt.utils.versions +import salt.utils.yaml from salt.exceptions import ( CommandExecutionError, MinionError, SaltInvocationError ) @@ -2644,8 +2644,8 @@ def set_selections(path=None, selection=None, clear=False, saltenv='base'): if isinstance(selection, six.string_types): try: - selection = yaml.safe_load(selection) - except (yaml.parser.ParserError, yaml.scanner.ScannerError) as exc: + selection = salt.utils.yaml.safe_load(selection) + except (salt.utils.yaml.parser.ParserError, salt.utils.yaml.scanner.ScannerError) as exc: raise SaltInvocationError( 'Improperly-formatted selection: {0}'.format(exc) ) diff --git a/salt/modules/bamboohr.py b/salt/modules/bamboohr.py index 0fc4acd8fb2..b3a4eec6aa7 100644 --- a/salt/modules/bamboohr.py +++ b/salt/modules/bamboohr.py @@ -15,11 +15,11 @@ Requires a ``subdomain`` and an ``apikey`` in ``/etc/salt/minion``: # Import python libs from __future__ import absolute_import, print_function -import yaml import logging # Import salt libs import salt.utils.http +import salt.utils.yaml from salt.ext import six from salt._compat import ElementTree as ET @@ -167,7 +167,7 @@ def update_employee(emp_id, key=None, value=None, items=None): return {'Error': 'At least one key/value pair is required'} items = {key: value} elif isinstance(items, six.string_types): - items = yaml.safe_load(items) + items = salt.utils.yaml.safe_load(items) xml_items = '' for pair in items: diff --git a/salt/modules/beacons.py b/salt/modules/beacons.py index efd1f5c9399..cf8c7d3d249 100644 --- a/salt/modules/beacons.py +++ b/salt/modules/beacons.py @@ -11,12 +11,12 @@ from __future__ import absolute_import import difflib import logging import os -import yaml # Import Salt libs import salt.ext.six as six import salt.utils.event import salt.utils.files +import salt.utils.yaml from salt.ext.six.moves import map # Get logging started @@ -75,8 +75,7 @@ def list_(return_yaml=True, if beacons: if return_yaml: tmp = {'beacons': beacons} - yaml_out = yaml.safe_dump(tmp, default_flow_style=False) - return yaml_out + return salt.utils.yaml.safe_dump(tmp, default_flow_style=False) else: return beacons else: @@ -116,8 +115,7 @@ def list_available(return_yaml=True): if beacons: if return_yaml: tmp = {'beacons': beacons} - yaml_out = yaml.safe_dump(tmp, default_flow_style=False) - return yaml_out + return salt.utils.yaml.safe_dump(tmp, default_flow_style=False) else: return beacons else: @@ -359,7 +357,7 @@ def save(): os.path.dirname(__opts__['default_include'])) if beacons: tmp = {'beacons': beacons} - yaml_out = yaml.safe_dump(tmp, default_flow_style=False) + yaml_out = salt.utils.yaml.safe_dump(tmp, default_flow_style=False) else: yaml_out = '' diff --git a/salt/modules/boto_asg.py b/salt/modules/boto_asg.py index 3795a5a6703..78bf25626c7 100644 --- a/salt/modules/boto_asg.py +++ b/salt/modules/boto_asg.py @@ -58,7 +58,6 @@ log = logging.getLogger(__name__) DATE_FORMAT = "%Y-%m-%dT%H:%M:%SZ" # Import third party libs -import yaml from salt.ext import six try: import boto @@ -522,20 +521,13 @@ def get_cloud_init_mime(cloud_init): _cloud_init.attach(_script) if 'cloud-config' in cloud_init: cloud_config = cloud_init['cloud-config'] - _cloud_config = email.mime.text.MIMEText(_safe_dump(cloud_config), - 'cloud-config') + _cloud_config = email.mime.text.MIMEText( + salt.utils.yaml.safe_dump(cloud_config, default_flow_style=False), + 'cloud-config') _cloud_init.attach(_cloud_config) return _cloud_init.as_string() -def _safe_dump(data): - def ordered_dict_presenter(dumper, data): - return dumper.represent_dict(six.iteritems(data)) - yaml.add_representer(odict.OrderedDict, ordered_dict_presenter, - Dumper=yaml.dumper.SafeDumper) - return yaml.safe_dump(data, default_flow_style=False) - - def launch_configuration_exists(name, region=None, key=None, keyid=None, profile=None): ''' diff --git a/salt/modules/boto_cloudfront.py b/salt/modules/boto_cloudfront.py index aa932884bf3..1a9997a33f8 100644 --- a/salt/modules/boto_cloudfront.py +++ b/salt/modules/boto_cloudfront.py @@ -58,8 +58,6 @@ import logging import salt.ext.six as six from salt.utils.odict import OrderedDict -import yaml - # Import third party libs try: # pylint: disable=unused-import @@ -273,8 +271,8 @@ def export_distributions(region=None, key=None, keyid=None, profile=None): # as opposed to being called from execution or state modules raise err - dumper = __utils__['yamldumper.get_dumper']('IndentedSafeOrderedDumper') - return yaml.dump( + dumper = __utils__['yaml.get_dumper']('IndentedSafeOrderedDumper') + return __utils__['yaml.dump']( results, default_flow_style=False, Dumper=dumper, diff --git a/salt/modules/boto_cloudwatch.py b/salt/modules/boto_cloudwatch.py index f8e6f4b1bcb..e24f5ab8d32 100644 --- a/salt/modules/boto_cloudwatch.py +++ b/salt/modules/boto_cloudwatch.py @@ -48,7 +48,7 @@ from __future__ import absolute_import # Import Python libs import logging -import yaml +import yaml # pylint: disable=blacklisted-import import salt.utils.json import salt.utils.odict as odict @@ -100,31 +100,27 @@ def get_alarm(name, region=None, key=None, keyid=None, profile=None): def _safe_dump(data): - ########################################### - # this presenter magic makes yaml.safe_dump - # work with the objects returned from - # boto.describe_alarms() - ########################################### - def ordered_dict_presenter(dumper, data): - return dumper.represent_dict(list(data.items())) - - yaml.add_representer(odict.OrderedDict, ordered_dict_presenter, - Dumper=yaml.dumper.SafeDumper) + ''' + this presenter magic makes yaml.safe_dump + work with the objects returned from + boto.describe_alarms() + ''' + custom_dumper = __utils__['yaml.get_dumper']('SafeOrderedDumper') def boto_listelement_presenter(dumper, data): return dumper.represent_list(list(data)) yaml.add_representer(boto.ec2.cloudwatch.listelement.ListElement, boto_listelement_presenter, - Dumper=yaml.dumper.SafeDumper) + Dumper=custom_dumper) def dimension_presenter(dumper, data): return dumper.represent_dict(dict(data)) yaml.add_representer(boto.ec2.cloudwatch.dimension.Dimension, - dimension_presenter, Dumper=yaml.dumper.SafeDumper) + dimension_presenter, Dumper=custom_dumper) - return yaml.safe_dump(data) + return __utils__['yaml.dump'](data, Dumper=custom_dumper) def get_all_alarms(region=None, prefix=None, key=None, keyid=None, diff --git a/salt/modules/boto_iam.py b/salt/modules/boto_iam.py index ff2f93e0a31..875a3e79c02 100644 --- a/salt/modules/boto_iam.py +++ b/salt/modules/boto_iam.py @@ -40,7 +40,6 @@ Connection module for Amazon IAM # Import Python libs from __future__ import absolute_import, print_function, unicode_literals import logging -import yaml import time # Import salt libs @@ -1566,21 +1565,6 @@ def delete_server_cert(cert_name, region=None, key=None, keyid=None, profile=Non return False -def _safe_dump(data): - ########################################### - # this presenter magic makes yaml.safe_dump - # work with the objects returned from - # boto.export_users() - ########################################### - def ordered_dict_presenter(dumper, data): - return dumper.represent_dict(data.items()) - - yaml.add_representer(odict.OrderedDict, ordered_dict_presenter, - Dumper=yaml.dumper.SafeDumper) - - return yaml.safe_dump(data, default_flow_style=False, indent=2) - - def export_users(path_prefix='/', region=None, key=None, keyid=None, profile=None): ''' @@ -1614,7 +1598,10 @@ def export_users(path_prefix='/', region=None, key=None, keyid=None, user_sls.append({"policies": policies}) user_sls.append({"path": user.path}) results["manage user " + name] = {"boto_iam.user_present": user_sls} - return _safe_dump(results) + return __utils__['yaml.safe_dump']( + results, + default_flow_style=False, + indent=2) def export_roles(path_prefix='/', region=None, key=None, keyid=None, profile=None): @@ -1648,7 +1635,10 @@ def export_roles(path_prefix='/', region=None, key=None, keyid=None, profile=Non role_sls.append({'policy_document': salt.utils.json.loads(_unquote(role.assume_role_policy_document))}) role_sls.append({"path": role.path}) results["manage role " + name] = {"boto_iam_role.present": role_sls} - return _safe_dump(results) + return __utils__['yaml.safe_dump']( + results, + default_flow_style=False, + indent=2) def _get_policy_arn(name, region=None, key=None, keyid=None, profile=None): diff --git a/salt/modules/defaults.py b/salt/modules/defaults.py index da69030f8d9..7479e1f6c99 100644 --- a/salt/modules/defaults.py +++ b/salt/modules/defaults.py @@ -8,7 +8,6 @@ from __future__ import absolute_import import copy import logging import os -import yaml import salt.fileclient import salt.utils.data @@ -16,11 +15,10 @@ import salt.utils.dictupdate as dictupdate import salt.utils.files import salt.utils.json import salt.utils.url - +import salt.utils.yaml __virtualname__ = 'defaults' - log = logging.getLogger(__name__) @@ -57,7 +55,7 @@ def _load(formula): suffix = file_.rsplit('.', 1)[-1] if suffix == 'yaml': - loader = yaml.safe_load + loader = salt.utils.yaml.safe_load elif suffix == 'json': loader = salt.utils.json.load else: diff --git a/salt/modules/grains.py b/salt/modules/grains.py index b3f922ebee7..619a8785218 100644 --- a/salt/modules/grains.py +++ b/salt/modules/grains.py @@ -18,7 +18,6 @@ import logging import operator import collections import math -import yaml from functools import reduce # pylint: disable=redefined-builtin # Import Salt libs @@ -28,7 +27,7 @@ import salt.utils.data import salt.utils.files import salt.utils.json import salt.utils.platform -import salt.utils.yamldumper +import salt.utils.yaml from salt.defaults import DEFAULT_TARGET_DELIM from salt.exceptions import SaltException from salt.ext.six.moves import range @@ -273,8 +272,8 @@ def setvals(grains, destructive=False): if os.path.isfile(gfn): with salt.utils.files.fopen(gfn, 'rb') as fp_: try: - grains = yaml.safe_load(fp_.read()) - except yaml.YAMLError as exc: + grains = salt.utils.yaml.safe_load(fp_) + except salt.utils.yaml.YAMLError as exc: return 'Unable to read existing grains file: {0}'.format(exc) if not isinstance(grains, dict): grains = {} @@ -287,17 +286,16 @@ def setvals(grains, destructive=False): else: grains[key] = val __grains__[key] = val - cstr = salt.utils.yamldumper.safe_dump(grains, default_flow_style=False) try: with salt.utils.files.fopen(gfn, 'w+') as fp_: - fp_.write(cstr) + salt.utils.yaml.safe_dump(grains, fp_, default_flow_style=False) except (IOError, OSError): msg = 'Unable to write to grains file at {0}. Check permissions.' log.error(msg.format(gfn)) fn_ = os.path.join(__opts__['cachedir'], 'module_refresh') try: - with salt.utils.files.flopen(fn_, 'w+') as fp_: - fp_.write('') + with salt.utils.files.flopen(fn_, 'w+'): + pass except (IOError, OSError): msg = 'Unable to write to cache file {0}. Check permissions.' log.error(msg.format(fn_)) diff --git a/salt/modules/heat.py b/salt/modules/heat.py index 39a5019a941..e8e0d9b7515 100644 --- a/salt/modules/heat.py +++ b/salt/modules/heat.py @@ -44,19 +44,18 @@ Module for handling OpenStack Heat calls from __future__ import absolute_import import time import logging -import yaml # Import Salt libs from salt.ext import six import salt.utils.files import salt.utils.json +import salt.utils.yaml from salt.exceptions import SaltInvocationError # pylint: disable=import-error HAS_HEAT = False try: - from heatclient import client - from heatclient import exc + import heatclient HAS_HEAT = True except ImportError: pass @@ -69,40 +68,14 @@ except ImportError: pass SECTIONS = ( - PARAMETER_DEFAULTS, PARAMETERS, RESOURCE_REGISTRY, EVENT_SINKS -) = ( - 'parameter_defaults', 'parameters', 'resource_registry', 'event_sinks' -) - -if hasattr(yaml, 'CSafeLoader'): - YamlLoader = yaml.CSafeLoader -else: - YamlLoader = yaml.SafeLoader - -if hasattr(yaml, 'CSafeDumper'): - YamlDumper = yaml.CSafeDumper -else: - YamlDumper = yaml.SafeDumper - - -def _represent_yaml_str(self, node): - ''' - Represent for yaml - ''' - return self.represent_scalar(node) -YamlDumper.add_representer(u'tag:yaml.org,2002:str', - _represent_yaml_str) -YamlDumper.add_representer(u'tag:yaml.org,2002:timestamp', - _represent_yaml_str) - - -def _construct_yaml_str(self, node): - ''' - Construct for yaml - ''' - return self.construct_scalar(node) -YamlLoader.add_constructor(u'tag:yaml.org,2002:timestamp', - _construct_yaml_str) + PARAMETER_DEFAULTS, + PARAMETERS, + RESOURCE_REGISTRY, + EVENT_SINKS) = ( + 'parameter_defaults', + 'parameters', + 'resource_registry', + 'event_sinks') logging.basicConfig(level=logging.DEBUG) log = logging.getLogger(__name__) @@ -194,7 +167,7 @@ def _auth(profile=None, api_version=1, **connection_args): kwargs)) # may raise exc.HTTPUnauthorized, exc.HTTPNotFound # but we deal with those elsewhere - return client.Client(api_version, endpoint=heat_endpoint, **kwargs) + return heatclient.client.Client(api_version, endpoint=heat_endpoint, **kwargs) def _parse_template(tmpl_str): @@ -206,12 +179,9 @@ def _parse_template(tmpl_str): tpl = salt.utils.json.loads(tmpl_str) else: try: - tpl = yaml.load(tmpl_str, Loader=YamlLoader) - except yaml.YAMLError: - try: - tpl = yaml.load(tmpl_str, Loader=yaml.SafeLoader) - except yaml.YAMLError as yea: - raise ValueError(yea) + tpl = salt.utils.yaml.safe_load(tmpl_str) + except salt.utils.yaml.YAMLError as exc: + raise ValueError(six.text_type(exc)) else: if tpl is None: tpl = {} @@ -227,22 +197,20 @@ def _parse_enviroment(env_str): Parsing template ''' try: - env = yaml.load(env_str, Loader=YamlLoader) - except yaml.YAMLError: - try: - env = yaml.load(env_str, Loader=yaml.SafeLoader) - except yaml.YAMLError as yea: - raise ValueError(yea) + env = salt.utils.yaml.safe_load(env_str) + except salt.utils.yaml.YAMLError as exc: + raise ValueError(six.text_type(exc)) else: if env is None: env = {} elif not isinstance(env, dict): - raise ValueError(('The environment is not a valid ' - 'YAML mapping data type.')) + raise ValueError( + 'The environment is not a valid YAML mapping data type.' + ) for param in env: if param not in SECTIONS: - raise ValueError(('environment has wrong section "{0}"').format(param)) + raise ValueError('environment has wrong section "{0}"'.format(param)) return env @@ -255,8 +223,8 @@ def _get_stack_events(h_client, stack_id, event_args): event_args['resource_name'] = None try: events = h_client.events.list(**event_args) - except exc.HTTPNotFound as ex: - raise exc.CommandError(str(ex)) + except heatclient.exc.HTTPNotFound as ex: + raise heatclient.exc.CommandError(str(ex)) else: for event in events: event.stack_name = stack_id.split('/')[0] @@ -385,7 +353,7 @@ def show_stack(name=None, profile=None): 'links': links, } ret['result'] = True - except exc.HTTPNotFound: + except heatclient.exc.HTTPNotFound: return { 'result': False, 'comment': 'No stack {0}'.format(name) @@ -428,10 +396,10 @@ def delete_stack(name=None, poll=0, timeout=60, profile=None): return ret try: h_client.stacks.delete(name) - except exc.HTTPNotFound: + except heatclient.exc.HTTPNotFound: ret['result'] = False ret['comment'] = 'No stack {0}'.format(name) - except exc.HTTPForbidden as forbidden: + except heatclient.exc.HTTPForbidden as forbidden: log.exception(str(forbidden)) ret['result'] = False ret['comment'] = str(forbidden) @@ -442,7 +410,7 @@ def delete_stack(name=None, poll=0, timeout=60, profile=None): try: stack_status, msg = _poll_for_events(h_client, name, action='DELETE', poll_period=poll, timeout=timeout) - except exc.CommandError: + except heatclient.exc.CommandError: ret['comment'] = 'Deleted stack {0}.'.format(name) return ret except Exception as ex: # pylint: disable=W0703 @@ -858,18 +826,18 @@ def template_stack(name=None, profile=None): } try: get_template = h_client.stacks.template(name) - except exc.HTTPNotFound: + except heatclient.exc.HTTPNotFound: return { 'result': False, 'comment': 'No stack with {0}'.format(name) } - except exc.BadRequest: + except heatclient.exc.BadRequest: return { 'result': False, 'comment': 'Bad request fot stack {0}'.format(name) } if 'heat_template_version' in get_template: - template = yaml.dump(get_template, Dumper=YamlDumper) + template = salt.utils.yaml.safe_dump(get_template) else: template = jsonutils.dumps(get_template, indent=2, ensure_ascii=False) diff --git a/salt/modules/highstate_doc.py b/salt/modules/highstate_doc.py index 8aed0a361c5..fd08a8a7120 100644 --- a/salt/modules/highstate_doc.py +++ b/salt/modules/highstate_doc.py @@ -218,15 +218,14 @@ Some `replace_text_regex` values that might be helpfull. from __future__ import absolute_import import re -import yaml import logging import salt.utils.files import salt.utils.templates as tpl -from salt.utils.yamldumper import OrderedDumper - +import salt.utils.yaml __virtualname__ = 'highstate_doc' + log = logging.getLogger(__name__) @@ -519,8 +518,7 @@ def _state_data_to_yaml_string(data, whitelist=None, blacklist=None): y[k] = data[k] if len(y) == 0: return None - y = yaml.dump(y, Dumper=OrderedDumper, default_flow_style=False) - return y + return salt.utils.yaml.safe_dump(y, default_flow_style=False) def _md_fix(text): diff --git a/salt/modules/kubernetes.py b/salt/modules/kubernetes.py index a463a0bceda..c55dd253830 100644 --- a/salt/modules/kubernetes.py +++ b/salt/modules/kubernetes.py @@ -40,7 +40,6 @@ import sys import os.path import base64 import logging -import yaml import tempfile import signal from time import sleep @@ -50,6 +49,7 @@ from salt.exceptions import CommandExecutionError from salt.ext.six import iteritems import salt.utils.files import salt.utils.templates +import salt.utils.yaml from salt.exceptions import TimeoutError from salt.ext.six.moves import range # pylint: disable=import-error @@ -1475,7 +1475,7 @@ def __read_and_render_yaml_file(source, 'Unknown template specified: {0}'.format( template)) - return yaml.load(contents) + return salt.utils.yaml.safe_load(contents) def __dict_to_object_meta(name, namespace, metadata): diff --git a/salt/modules/pagerduty.py b/salt/modules/pagerduty.py index 5979ea98c90..bb10bc4e206 100644 --- a/salt/modules/pagerduty.py +++ b/salt/modules/pagerduty.py @@ -18,14 +18,13 @@ Module for Firing Events via PagerDuty ''' from __future__ import absolute_import -# Import python libs -import yaml - -# Import salt libs +# Import Salt libs import salt.utils.functools import salt.utils.json import salt.utils.pagerduty -from salt.ext.six import string_types + +# Import 3rd-party libs +from salt.ext import six def __virtual__(): @@ -176,9 +175,9 @@ def create_event(service_key=None, description=None, details=None, ''' trigger_url = 'https://events.pagerduty.com/generic/2010-04-15/create_event.json' - if isinstance(details, string_types): - details = yaml.safe_load(details) - if isinstance(details, string_types): + if isinstance(details, six.string_types): + details = salt.utils.yaml.safe_load(details) + if isinstance(details, six.string_types): details = {'details': details} ret = salt.utils.json.loads(salt.utils.pagerduty.query( diff --git a/salt/modules/parallels.py b/salt/modules/parallels.py index c996f831e28..417a001d032 100644 --- a/salt/modules/parallels.py +++ b/salt/modules/parallels.py @@ -24,11 +24,11 @@ from __future__ import absolute_import import re import logging import shlex -import yaml # Import salt libs import salt.utils.locales import salt.utils.path +import salt.utils.yaml from salt.exceptions import SaltInvocationError # Import 3rd party libs @@ -476,8 +476,8 @@ def snapshot_id_to_name(name, snap_id, strict=False, runas=None): # Try to interpret the information try: - data = yaml.safe_load(info) - except yaml.YAMLError as err: + data = salt.utils.yaml.safe_load(info) + except salt.utils.yaml.YAMLError as err: log.warning( 'Could not interpret snapshot data returned from prlctl: ' '{0}'.format(err) diff --git a/salt/modules/pillar.py b/salt/modules/pillar.py index a3dc6f57c1d..5d18208db9d 100644 --- a/salt/modules/pillar.py +++ b/salt/modules/pillar.py @@ -12,7 +12,6 @@ import copy import os import copy import logging -import yaml from salt.ext import six # Import salt libs @@ -22,6 +21,7 @@ import salt.utils.data import salt.utils.dictupdate import salt.utils.functools import salt.utils.odict +import salt.utils.yaml from salt.defaults import DEFAULT_TARGET_DELIM from salt.exceptions import CommandExecutionError @@ -429,8 +429,8 @@ def ext(external, pillar=None): .. code-block:: python - >>> import yaml - >>> ext_pillar = yaml.safe_load(""" + >>> import salt.utils.yaml + >>> ext_pillar = salt.utils.yaml.safe_load(""" ... ext_pillar: ... - git: ... - issue38440 https://github.com/terminalmage/git_pillar: @@ -463,7 +463,7 @@ def ext(external, pillar=None): salt '*' pillar.ext "{'git': [{'mybranch https://github.com/myuser/myrepo': [{'env': 'base'}]}]}" ''' if isinstance(external, six.string_types): - external = yaml.safe_load(external) + external = salt.utils.yaml.safe_load(external) pillar_obj = salt.pillar.get_pillar( __opts__, __grains__, diff --git a/salt/modules/pkg_resource.py b/salt/modules/pkg_resource.py index a59f858cb7b..8bee9e3146c 100644 --- a/salt/modules/pkg_resource.py +++ b/salt/modules/pkg_resource.py @@ -12,12 +12,12 @@ import os import pprint # Import third party libs -import yaml from salt.ext import six # Import salt libs import salt.utils.data import salt.utils.versions +import salt.utils.yaml from salt.exceptions import SaltInvocationError log = logging.getLogger(__name__) @@ -72,8 +72,8 @@ def pack_sources(sources, normalize=True): if isinstance(sources, six.string_types): try: - sources = yaml.safe_load(sources) - except yaml.parser.ParserError as err: + sources = salt.utils.yaml.safe_load(sources) + except salt.utils.yaml.parser.ParserError as err: log.error(err) return {} ret = {} diff --git a/salt/modules/puppet.py b/salt/modules/puppet.py index 0462152e037..1d65b17b4f9 100644 --- a/salt/modules/puppet.py +++ b/salt/modules/puppet.py @@ -15,10 +15,10 @@ import salt.utils.args import salt.utils.files import salt.utils.path import salt.utils.platform +import salt.utils.yaml from salt.exceptions import CommandExecutionError # Import 3rd-party libs -import yaml from salt.ext import six from salt.ext.six.moves import range log = logging.getLogger(__name__) @@ -312,7 +312,7 @@ def summary(): try: with salt.utils.files.fopen(puppet.lastrunfile, 'r') as fp_: - report = yaml.safe_load(fp_.read()) + report = salt.utils.yaml.safe_load(fp_) result = {} if 'time' in report: @@ -330,7 +330,7 @@ def summary(): if 'resources' in report: result['resources'] = report['resources'] - except yaml.YAMLError as exc: + except salt.utils.yaml.YAMLError as exc: raise CommandExecutionError( 'YAML error parsing puppet run summary: {0}'.format(exc) ) diff --git a/salt/modules/saltcheck.py b/salt/modules/saltcheck.py index d265a56ebb6..7c3d580c022 100644 --- a/salt/modules/saltcheck.py +++ b/salt/modules/saltcheck.py @@ -52,10 +52,10 @@ import logging import os import time from json import loads, dumps -import yaml try: import salt.utils.files import salt.utils.path + import salt.utils.yaml import salt.client import salt.exceptions except ImportError: @@ -557,7 +557,7 @@ class StateTestLoader(object): with __utils__['files.fopen'](filepath, 'r') as myfile: # with salt.utils.files.fopen(filepath, 'r') as myfile: # with open(filepath, 'r') as myfile: - contents_yaml = yaml.load(myfile) + contents_yaml = salt.utils.yaml.safe_load(myfile) for key, value in contents_yaml.items(): self.test_dict[key] = value except: diff --git a/salt/modules/schedule.py b/salt/modules/schedule.py index a4eeb8bf696..15ab9c9937c 100644 --- a/salt/modules/schedule.py +++ b/salt/modules/schedule.py @@ -12,12 +12,12 @@ import copy as pycopy import difflib import logging import os -import yaml # Import salt libs import salt.utils.event import salt.utils.files import salt.utils.odict +import salt.utils.yaml # Import 3rd-party libs from salt.ext import six @@ -140,8 +140,7 @@ def list_(show_all=False, if schedule: if return_yaml: tmp = {'schedule': schedule} - yaml_out = yaml.safe_dump(tmp, default_flow_style=False) - return yaml_out + return salt.utils.yaml.safe_dump(tmp, default_flow_style=False) else: return schedule else: @@ -808,8 +807,8 @@ def reload_(): if os.path.isfile(sfn): with salt.utils.files.fopen(sfn, 'rb') as fp_: try: - schedule = yaml.safe_load(fp_.read()) - except yaml.YAMLError as exc: + schedule = salt.utils.yaml.safe_load(fp_) + except salt.utils.yaml.YAMLError as exc: ret['comment'].append('Unable to read existing schedule file: {0}'.format(exc)) if schedule: diff --git a/salt/modules/splunk_search.py b/salt/modules/splunk_search.py index c78192a0555..bea4d4ac3d4 100644 --- a/salt/modules/splunk_search.py +++ b/salt/modules/splunk_search.py @@ -23,7 +23,6 @@ Module for interop with the Splunk API # Import python libs from __future__ import absolute_import import logging -import yaml import urllib # Import third party libs @@ -37,6 +36,7 @@ except ImportError: pass # Import salt libs +import salt.utils.yaml from salt.utils.odict import OrderedDict log = logging.getLogger(__name__) @@ -239,12 +239,6 @@ def list_all(prefix=None, app=None, owner=None, description_contains=None, ''' client = _get_splunk(profile) - # yaml magic to output OrderedDict - def ordered_dict_presenter(dumper, data): - return dumper.represent_dict(six.iteritems(data)) - yaml.add_representer( - OrderedDict, ordered_dict_presenter, Dumper=yaml.dumper.SafeDumper) - # splunklib doesn't provide the default settings for saved searches. # so, in order to get the defaults, we create a search with no # configuration, get that search, and then delete it. We use its contents @@ -306,4 +300,4 @@ def list_all(prefix=None, app=None, owner=None, description_contains=None, continue results["manage splunk search " + name] = {"splunk_search.present": d} - return yaml.safe_dump(results, default_flow_style=False, width=120) + return salt.utils.yaml.safe_dump(results, default_flow_style=False, width=120) diff --git a/salt/modules/virt.py b/salt/modules/virt.py index 65a89a68e28..f5b34dcc3a3 100644 --- a/salt/modules/virt.py +++ b/salt/modules/virt.py @@ -22,7 +22,6 @@ import datetime from xml.etree import ElementTree # Import third party libs -import yaml import jinja2 import jinja2.exceptions from salt.ext import six @@ -42,6 +41,7 @@ import salt.utils.path import salt.utils.stringutils import salt.utils.templates import salt.utils.validate.net +import salt.utils.yaml from salt.exceptions import CommandExecutionError, SaltInvocationError log = logging.getLogger(__name__) @@ -400,7 +400,7 @@ def _qemu_image_create(vm_name, qcow2 = False if salt.utils.path.which('qemu-img'): res = __salt__['cmd.run']('qemu-img info {}'.format(sfn)) - imageinfo = yaml.load(res) + imageinfo = salt.utils.yaml.safe_load(res) qcow2 = imageinfo['file format'] == 'qcow2' try: if enable_qcow and qcow2: @@ -1134,9 +1134,9 @@ def get_disks(vm_): continue output.append(line) output = '\n'.join(output) - disks[dev].update(yaml.safe_load(output)) + disks[dev].update(salt.utils.yaml.safe_load(output)) except TypeError: - disks[dev].update(yaml.safe_load('image: Does not exist')) + disks[dev].update({'image': 'Does not exist'}) return disks @@ -1594,7 +1594,7 @@ def seed_non_shared_migrate(disks, force=False): size = data['virtual size'].split()[1][1:] if os.path.isfile(fn_) and not force: # the target exists, check to see if it is compatible - pre = yaml.safe_load(subprocess.Popen('qemu-img info arch', + pre = salt.utils.yaml.safe_load(subprocess.Popen('qemu-img info arch', shell=True, stdout=subprocess.PIPE).communicate()[0]) if pre['file format'] != data['file format']\ diff --git a/salt/modules/x509.py b/salt/modules/x509.py index 7344bc7d6ce..21d7425f977 100644 --- a/salt/modules/x509.py +++ b/salt/modules/x509.py @@ -17,7 +17,6 @@ import glob import random import ctypes import tempfile -import yaml import re import datetime import ast @@ -177,7 +176,7 @@ def _parse_openssl_req(csr_filename): output = re.sub(r': rsaEncryption', ':', output) output = re.sub(r'[0-9a-f]{2}:', '', output) - return yaml.safe_load(output) + return salt.utils.yaml.safe_load(output) def _get_csr_extensions(csr): @@ -266,7 +265,7 @@ def _parse_openssl_crl(crl_filename): rev_sn = revoked.split('\n')[0].strip() revoked = rev_sn + ':\n' + '\n'.join(revoked.split('\n')[1:]) - rev_yaml = yaml.safe_load(revoked) + rev_yaml = salt.utils.yaml.safe_load(revoked) # pylint: disable=unused-variable for rev_item, rev_values in six.iteritems(rev_yaml): # pylint: enable=unused-variable diff --git a/salt/netapi/rest_cherrypy/app.py b/salt/netapi/rest_cherrypy/app.py index 2c9638e4efa..00f54a45345 100644 --- a/salt/netapi/rest_cherrypy/app.py +++ b/salt/netapi/rest_cherrypy/app.py @@ -606,7 +606,6 @@ except ImportError: cpstats = None logger.warn('Import of cherrypy.cpstats failed.') -import yaml # pylint: enable=import-error, 3rd-party-module-not-gated # Import Salt libs @@ -617,6 +616,7 @@ import salt.utils.event import salt.utils.json import salt.utils.stringutils import salt.utils.versions +import salt.utils.yaml from salt.ext import six from salt.ext.six import BytesIO @@ -837,7 +837,7 @@ def cors_tool(): ct_out_map = ( ('application/json', salt.utils.json.dumps), ('application/x-yaml', functools.partial( - yaml.safe_dump, default_flow_style=False)), + salt.utils.yaml.safe_dump, default_flow_style=False)), ) @@ -998,7 +998,7 @@ def yaml_processor(entity): contents.seek(0) body = salt.utils.stringutils.to_unicode(contents.read()) try: - cherrypy.serving.request.unserialized_data = yaml.safe_load(body) + cherrypy.serving.request.unserialized_data = salt.utils.yaml.safe_load(body) except ValueError: raise cherrypy.HTTPError(400, 'Invalid YAML document') diff --git a/salt/netapi/rest_tornado/saltnado.py b/salt/netapi/rest_tornado/saltnado.py index 85296687be8..22af29b0577 100644 --- a/salt/netapi/rest_tornado/saltnado.py +++ b/salt/netapi/rest_tornado/saltnado.py @@ -196,7 +196,6 @@ from collections import defaultdict # pylint: disable=import-error import cgi -import yaml import tornado.escape import tornado.httpserver import tornado.ioloop @@ -215,6 +214,7 @@ import salt.netapi import salt.utils.args import salt.utils.event import salt.utils.json +import salt.utils.yaml from salt.utils.event import tagify import salt.client import salt.runner @@ -399,7 +399,7 @@ class EventListener(object): class BaseSaltAPIHandler(tornado.web.RequestHandler, SaltClientsMixIn): # pylint: disable=W0223 ct_out_map = ( ('application/json', _json_dumps), - ('application/x-yaml', yaml.safe_dump), + ('application/x-yaml', salt.utils.yaml.safe_dump), ) def _verify_client(self, low): @@ -523,8 +523,8 @@ class BaseSaltAPIHandler(tornado.web.RequestHandler, SaltClientsMixIn): # pylin ct_in_map = { 'application/x-www-form-urlencoded': self._form_loader, 'application/json': salt.utils.json.loads, - 'application/x-yaml': yaml.safe_load, - 'text/yaml': yaml.safe_load, + 'application/x-yaml': salt.utils.yaml.safe_load, + 'text/yaml': salt.utils.yaml.safe_load, # because people are terrible and don't mean what they say 'text/plain': salt.utils.json.loads } diff --git a/salt/output/yaml_out.py b/salt/output/yaml_out.py index 4212b6c74d5..392f41f9b12 100644 --- a/salt/output/yaml_out.py +++ b/salt/output/yaml_out.py @@ -21,10 +21,9 @@ from __future__ import absolute_import, print_function, unicode_literals # Import third party libs import logging -import yaml # Import salt libs -from salt.utils.yamldumper import SafeOrderedDumper +import salt.utils.yaml # Define the module's virtual name __virtualname__ = 'yaml' @@ -41,7 +40,7 @@ def output(data, **kwargs): # pylint: disable=unused-argument Print out YAML using the block mode ''' - params = dict(Dumper=SafeOrderedDumper) + params = {} if 'output_indent' not in __opts__: # default indentation params.update(default_flow_style=False) @@ -53,7 +52,7 @@ def output(data, **kwargs): # pylint: disable=unused-argument params.update(default_flow_style=True, indent=0) try: - return yaml.dump(data, **params) + return salt.utils.yaml.safe_dump(data, **params) except Exception as exc: import pprint log.exception( diff --git a/salt/pillar/cmd_yaml.py b/salt/pillar/cmd_yaml.py index 16f92b3de55..a1316744f60 100644 --- a/salt/pillar/cmd_yaml.py +++ b/salt/pillar/cmd_yaml.py @@ -7,11 +7,11 @@ from __future__ import absolute_import # Don't "fix" the above docstring to put it on two lines, as the sphinx # autosummary pulls only the first line for its description. -# Import python libs +# Import Python libs import logging -# Import third party libs -import yaml +# Import Salt party libs +import salt.utils.yaml # Set up logging log = logging.getLogger(__name__) @@ -25,8 +25,11 @@ def ext_pillar(minion_id, # pylint: disable=W0613 ''' try: command = command.replace('%s', minion_id) - return yaml.safe_load( - __salt__['cmd.run_stdout']('{0}'.format(command), python_shell=True)) + output = __salt__['cmd.run_stdout'](command, python_shell=True) + return salt.utils.yaml.safe_load(output) except Exception: - log.critical('YAML data from {0} failed to parse'.format(command)) + log.critical( + 'YAML data from \'%s\' failed to parse. Command output:\n%s', + command, output + ) return {} diff --git a/salt/pillar/consul_pillar.py b/salt/pillar/consul_pillar.py index d661649f4f7..9e0369d0035 100644 --- a/salt/pillar/consul_pillar.py +++ b/salt/pillar/consul_pillar.py @@ -125,12 +125,11 @@ from __future__ import absolute_import # Import python libs import logging import re -import yaml from salt.exceptions import CommandExecutionError from salt.utils.dictupdate import update as dict_merge import salt.utils.minions -from salt.utils.yamlloader import SaltYamlSafeLoader +import salt.utils.yaml # Import third party libs try: @@ -253,10 +252,7 @@ def pillar_format(ret, keys, value): # If value is not None then it's a string # Use YAML to parse the data # YAML strips whitespaces unless they're surrounded by quotes - pillar_value = yaml.load( - value, - Loader=SaltYamlSafeLoader - ) + pillar_value = salt.utils.yaml.safe_load(value) keyvalue = keys.pop() pil = {keyvalue: pillar_value} diff --git a/salt/pillar/hiera.py b/salt/pillar/hiera.py index 5a1a8d4031d..392bfb941d6 100644 --- a/salt/pillar/hiera.py +++ b/salt/pillar/hiera.py @@ -6,10 +6,10 @@ Use hiera data as a Pillar source # Import Python libs from __future__ import absolute_import import logging -import yaml # Import salt libs import salt.utils.path +import salt.utils.yaml # Import 3rd-party libs from salt.ext import six @@ -37,7 +37,7 @@ def ext_pillar(minion_id, # pylint: disable=W0613 if isinstance(val, six.string_types): cmd += ' {0}=\'{1}\''.format(key, val) try: - data = yaml.safe_load(__salt__['cmd.run'](cmd)) + data = salt.utils.yaml.safe_load(__salt__['cmd.run'](cmd)) except Exception: log.critical( 'Hiera YAML data failed to parse from conf {0}'.format(conf) diff --git a/salt/pillar/makostack.py b/salt/pillar/makostack.py index 4724da2bed8..40f161b2b0d 100644 --- a/salt/pillar/makostack.py +++ b/salt/pillar/makostack.py @@ -375,12 +375,14 @@ You can also select a custom merging strategy using a ``__`` object in a list: # Import Python libs from __future__ import absolute_import +import functools import os import logging -from functools import partial -import yaml # Import Salt libs +import salt.utils.yaml + +# Import 3rd-party libs from salt.ext import six try: @@ -411,9 +413,9 @@ def ext_pillar(minion_id, pillar, *args, **kwargs): stack = {} stack_config_files = list(args) traverse = { - 'pillar': partial(salt.utils.data.traverse_dict_and_list, pillar), - 'grains': partial(salt.utils.data.traverse_dict_and_list, __grains__), - 'opts': partial(salt.utils.data.traverse_dict_and_list, __opts__), + 'pillar': functools.partial(salt.utils.data.traverse_dict_and_list, pillar), + 'grains': functools.partial(salt.utils.data.traverse_dict_and_list, __grains__), + 'opts': functools.partial(salt.utils.data.traverse_dict_and_list, __opts__), } for matcher, matchs in six.iteritems(kwargs): t, matcher = matcher.split(':', 1) @@ -456,7 +458,7 @@ def _process_stack_cfg(cfg, stack, minion_id, pillar, namespace): __grains__=__grains__, minion_id=minion_id, pillar=pillar, stack=stack) - obj = yaml.safe_load(p) + obj = salt.utils.yaml.safe_load(p) if not isinstance(obj, dict): log.info('Ignoring Stack template "{0}": Can\'t parse ' 'as a valid yaml dictionary'.format(path)) @@ -541,9 +543,11 @@ def _merge_list(stack, obj): def _parse_top_cfg(content): - """Allow top_cfg to be YAML""" + ''' + Allow top_cfg to be YAML + ''' try: - obj = yaml.safe_load(content) + obj = salt.utils.yaml.safe_load(content) if isinstance(obj, list): return obj except Exception as e: diff --git a/salt/pillar/pepa.py b/salt/pillar/pepa.py index 01fd90a559e..ec073a2d6f3 100644 --- a/salt/pillar/pepa.py +++ b/salt/pillar/pepa.py @@ -271,19 +271,18 @@ __license__ = 'Apache License, Version 2.0' __version__ = '0.6.6' # Import python libs -import logging -import sys import glob -import yaml import jinja2 +import logging +import os import re -from os.path import isfile, join +import sys # Import Salt libs from salt.ext import six from salt.ext.six.moves import input # pylint: disable=import-error,redefined-builtin import salt.utils.files -from salt.utils.yamlloader import SaltYamlSafeLoader +import salt.utils.yaml # Import 3rd-party libs try: @@ -409,9 +408,9 @@ def ext_pillar(minion_id, pillar, resource, sequence, subkey=False, subkey_only= templdir = None if info and 'base_only' in info and info['base_only']: - templdir = join(roots['base'], resource, alias) + templdir = os.path.join(roots['base'], resource, alias) else: - templdir = join(roots[inp['environment']], resource, alias) + templdir = os.path.join(roots[inp['environment']], resource, alias) entries = [] if isinstance(inp[categ], list): @@ -425,8 +424,8 @@ def ext_pillar(minion_id, pillar, resource, sequence, subkey=False, subkey_only= for entry in entries: results_jinja = None results = None - fn = join(templdir, re.sub(r'\W', '_', entry.lower()) + '.yaml') - if isfile(fn): + fn = os.path.join(templdir, re.sub(r'\W', '_', entry.lower()) + '.yaml') + if os.path.isfile(fn): log.info("Loading template: {0}".format(fn)) with salt.utils.files.fopen(fn) as fhr: template = jinja2.Template(fhr.read()) @@ -437,13 +436,10 @@ def ext_pillar(minion_id, pillar, resource, sequence, subkey=False, subkey_only= data['grains'] = __grains__.copy() data['pillar'] = pillar.copy() results_jinja = template.render(data) - results = yaml.load( - results_jinja, - Loader=SaltYamlSafeLoader - ) + results = salt.utils.yaml.safe_load(results_jinja) except jinja2.UndefinedError as err: log.error('Failed to parse JINJA template: {0}\n{1}'.format(fn, err)) - except yaml.YAMLError as err: + except salt.utils.yaml.YAMLError as err: log.error('Failed to parse YAML in template: {0}\n{1}'.format(fn, err)) else: log.info("Template doesn't exist: {0}".format(fn)) @@ -522,7 +518,7 @@ def validate(output, resource): roots = __opts__['pepa_roots'] - valdir = join(roots['base'], resource, 'validate') + valdir = os.path.join(roots['base'], resource, 'validate') all_schemas = {} pepa_schemas = [] @@ -533,10 +529,7 @@ def validate(output, resource): data = output data['grains'] = __grains__.copy() data['pillar'] = __pillar__.copy() - schema = yaml.load( - template.render(data), - Loader=SaltYamlSafeLoader - ) + schema = salt.utils.yaml.safe_load(template.render(data)) all_schemas.update(schema) pepa_schemas.append(fn) @@ -552,18 +545,13 @@ def validate(output, resource): # Only used when called from a terminal if __name__ == '__main__': # Load configuration file - if not isfile(args.config): + if not os.path.isfile(args.config): log.critical("Configuration file doesn't exist: {0}".format(args.config)) sys.exit(1) # Get configuration with salt.utils.files.fopen(args.config) as fh_: - __opts__.update( - yaml.load( - fh_.read(), - Loader=SaltYamlSafeLoader - ) - ) + __opts__.update(salt.utils.yaml.safe_load(fh_)) loc = 0 for name in [next(iter(list(e.keys()))) for e in __opts__['ext_pillar']]: @@ -576,24 +564,14 @@ if __name__ == '__main__': if 'pepa_grains' in __opts__: __grains__ = __opts__['pepa_grains'] if args.grains: - __grains__.update( - yaml.load( - args.grains, - Loader=SaltYamlSafeLoader - ) - ) + __grains__.update(salt.utils.yaml.safe_load(args.grains)) # Get pillars __pillar__ = {} if 'pepa_pillar' in __opts__: __pillar__ = __opts__['pepa_pillar'] if args.pillar: - __pillar__.update( - yaml.load( - args.pillar, - Loader=SaltYamlSafeLoader - ) - ) + __pillar__.update(salt.utils.yaml.safe_load(args.pillar)) # Validate or not if args.validate: @@ -644,19 +622,32 @@ if __name__ == '__main__': if __opts__['pepa_validate']: validate(result, __opts__['ext_pillar'][loc]['pepa']['resource']) - yaml.dumper.SafeDumper.ignore_aliases = lambda self, data: True - if not args.no_color: - try: - # pylint: disable=import-error - import pygments - import pygments.lexers - import pygments.formatters - # pylint: disable=no-member - print(pygments.highlight(yaml.safe_dump(result), - pygments.lexers.YamlLexer(), - pygments.formatters.TerminalFormatter())) - # pylint: enable=no-member, import-error - except ImportError: - print(yaml.safe_dump(result, indent=4, default_flow_style=False)) - else: - print(yaml.safe_dump(result, indent=4, default_flow_style=False)) + try: + orig_ignore = salt.utils.yaml.SafeOrderedDumper.ignore_aliases + salt.utils.yaml.SafeOrderedDumper.ignore_aliases = lambda x, y: True + + def _print_result(result): + print(salt.utils.yaml.safe_dump( + result, + indent=4, + default_flow_style=False)) + + if not args.no_color: + try: + # pylint: disable=import-error + import pygments + import pygments.lexers + import pygments.formatters + # pylint: disable=no-member + print(pygments.highlight( + salt.utils.yaml.safe_dump(result), + pygments.lexers.YamlLexer(), + pygments.formatters.TerminalFormatter())) + # pylint: enable=no-member, import-error + except ImportError: + _print_result(result) + else: + _print_result(result) + finally: + # Undo monkeypatching + salt.utils.yaml.SafeOrderedDumper.ignore_aliases = orig_ignore diff --git a/salt/pillar/pillar_ldap.py b/salt/pillar/pillar_ldap.py index c803c7348c1..80e570a6741 100644 --- a/salt/pillar/pillar_ldap.py +++ b/salt/pillar/pillar_ldap.py @@ -113,8 +113,7 @@ TODO: see also ``_result_to_dict()`` documentation ''' # Import python libs -from __future__ import print_function -from __future__ import absolute_import +from __future__ import absolute_import, print_function import os import logging @@ -122,7 +121,6 @@ import logging from salt.exceptions import SaltInvocationError # Import third party libs -import yaml from jinja2 import Environment, FileSystemLoader try: import ldap # pylint: disable=W0611 @@ -278,10 +276,11 @@ def ext_pillar(minion_id, # pylint: disable=W0613 Execute LDAP searches and return the aggregated data ''' if os.path.isfile(config_file): + import salt.utils.yaml try: #open(config_file, 'r') as raw_config: config = _render_template(config_file) or {} - opts = yaml.safe_load(config) or {} + opts = salt.utils.yaml.safe_load(config) or {} opts['conf_file'] = config_file except Exception as err: import salt.log diff --git a/salt/pillar/puppet.py b/salt/pillar/puppet.py index 9b61257ca6a..7df8704d432 100644 --- a/salt/pillar/puppet.py +++ b/salt/pillar/puppet.py @@ -7,11 +7,11 @@ from __future__ import absolute_import # Don't "fix" the above docstring to put it on two lines, as the sphinx # autosummary pulls only the first line for its description. -# Import python libs +# Import Python libs import logging -# Import third party libs -import yaml +# Import Salt libs +import salt.utils.yaml # Set up logging log = logging.getLogger(__name__) @@ -24,9 +24,8 @@ def ext_pillar(minion_id, Execute an unmodified puppet_node_classifier and read the output as YAML ''' try: - data = yaml.safe_load(__salt__['cmd.run']('{0} {1}'.format(command, minion_id))) - data = data['parameters'] - return data + data = salt.utils.yaml.safe_load(__salt__['cmd.run']('{0} {1}'.format(command, minion_id))) + return data['parameters'] except Exception: log.critical( 'YAML data from {0} failed to parse'.format(command) diff --git a/salt/pillar/stack.py b/salt/pillar/stack.py index 9c9367812d9..fafaebda533 100644 --- a/salt/pillar/stack.py +++ b/salt/pillar/stack.py @@ -376,20 +376,19 @@ You can also select a custom merging strategy using a ``__`` object in a list: # Import Python libs from __future__ import absolute_import +import functools +import glob import os import posixpath import logging -from functools import partial -from glob import glob -import yaml from jinja2 import FileSystemLoader, Environment # Import Salt libs from salt.ext import six import salt.utils.data import salt.utils.jinja - +import salt.utils.yaml log = logging.getLogger(__name__) strategies = ('overwrite', 'merge-first', 'merge-last', 'remove') @@ -399,9 +398,9 @@ def ext_pillar(minion_id, pillar, *args, **kwargs): stack = {} stack_config_files = list(args) traverse = { - 'pillar': partial(salt.utils.data.traverse_dict_and_list, pillar), - 'grains': partial(salt.utils.data.traverse_dict_and_list, __grains__), - 'opts': partial(salt.utils.data.traverse_dict_and_list, __opts__), + 'pillar': functools.partial(salt.utils.data.traverse_dict_and_list, pillar), + 'grains': functools.partial(salt.utils.data.traverse_dict_and_list, __grains__), + 'opts': functools.partial(salt.utils.data.traverse_dict_and_list, __opts__), } for matcher, matchs in six.iteritems(kwargs): t, matcher = matcher.split(':', 1) @@ -432,7 +431,6 @@ def _construct_unicode(loader, node): def _process_stack_cfg(cfg, stack, minion_id, pillar): log.debug('Config: {0}'.format(cfg)) basedir, filename = os.path.split(cfg) - yaml.SafeLoader.add_constructor("tag:yaml.org,2002:python/unicode", _construct_unicode) jenv = Environment(loader=FileSystemLoader(basedir), extensions=['jinja2.ext.do', salt.utils.jinja.SerializerExtension]) jenv.globals.update({ "__opts__": __opts__, @@ -448,7 +446,7 @@ def _process_stack_cfg(cfg, stack, minion_id, pillar): jenv.get_template(filename).render(stack=stack)): if not item.strip(): continue # silently ignore whitespace or empty lines - paths = glob(os.path.join(basedir, item)) + paths = glob.glob(os.path.join(basedir, item)) if not paths: log.warning('Ignoring pillar stack template "{0}": can\'t find from ' 'root dir "{1}"'.format(item, basedir)) @@ -457,7 +455,7 @@ def _process_stack_cfg(cfg, stack, minion_id, pillar): log.debug('YAML: basedir={0}, path={1}'.format(basedir, path)) # FileSystemLoader always expects unix-style paths unix_path = _to_unix_slashes(os.path.relpath(path, basedir)) - obj = yaml.safe_load(jenv.get_template(unix_path).render(stack=stack)) + obj = salt.utils.yaml.safe_load(jenv.get_template(unix_path).render(stack=stack)) if not isinstance(obj, dict): log.info('Ignoring pillar stack template "{0}": Can\'t parse ' 'as a valid yaml dictionary'.format(path)) @@ -535,7 +533,7 @@ def _parse_stack_cfg(content): Allow top level cfg to be YAML ''' try: - obj = yaml.safe_load(content) + obj = salt.utils.yaml.safe_load(content) if isinstance(obj, list): return obj except Exception as e: diff --git a/salt/returners/highstate_return.py b/salt/returners/highstate_return.py index 46e9b43c89a..32a0e623220 100644 --- a/salt/returners/highstate_return.py +++ b/salt/returners/highstate_return.py @@ -84,7 +84,6 @@ import smtplib import cgi from email.mime.text import MIMEText -import yaml from salt.ext.six.moves import range from salt.ext.six.moves import StringIO from salt.ext import six @@ -92,6 +91,7 @@ from salt.ext import six import salt.utils.files import salt.utils.json import salt.utils.stringutils +import salt.utils.yaml import salt.returners log = logging.getLogger(__name__) @@ -427,7 +427,7 @@ def _produce_output(report, failed, setup): report_text = salt.utils.json.dumps(report) elif report_format == 'yaml': string_file = StringIO() - yaml.safe_dump(report, string_file, default_flow_style=False) + salt.utils.yaml.safe_dump(report, string_file, default_flow_style=False) string_file.seek(0) report_text = string_file.read() else: @@ -494,7 +494,7 @@ def __test_html(): ''' with salt.utils.files.fopen('test.rpt', 'r') as input_file: data_text = salt.utils.stringutils.to_unicode(input_file.read()) - data = yaml.safe_load(data_text) + data = salt.utils.yaml.safe_load(data_text) string_file = StringIO() _generate_html(data, string_file) diff --git a/salt/returners/slack_returner.py b/salt/returners/slack_returner.py index 9a5e8d7cd1c..7bd5faf91b8 100644 --- a/salt/returners/slack_returner.py +++ b/salt/returners/slack_returner.py @@ -79,7 +79,6 @@ To override individual configuration items, append --return_kwargs '{"key:": "va from __future__ import absolute_import, print_function, unicode_literals # Import Python libs -import yaml import pprint import logging @@ -91,7 +90,7 @@ from salt.ext.six.moves.urllib.parse import urlencode as _urlencode # Import Salt Libs import salt.returners import salt.utils.slack -from salt.utils.yamldumper import SafeOrderedDumper +import salt.utils.yaml log = logging.getLogger(__name__) @@ -212,7 +211,7 @@ def returner(ret): returns = dict((key, value) for key, value in returns.items() if value['result'] is not True or value['changes']) if yaml_format is True: - returns = yaml.dump(returns, Dumper=SafeOrderedDumper) + returns = salt.utils.yaml.safe_dump(returns) else: returns = pprint.pformat(returns) diff --git a/salt/runners/pagerduty.py b/salt/runners/pagerduty.py index 7095c9836c5..a8d4e08eaa2 100644 --- a/salt/runners/pagerduty.py +++ b/salt/runners/pagerduty.py @@ -17,13 +17,11 @@ Runner Module for Firing Events via PagerDuty ''' from __future__ import absolute_import, print_function, unicode_literals -# Import python libs -import yaml - # Import salt libs import salt.utils.functools import salt.utils.json import salt.utils.pagerduty +import salt.utils.yaml from salt.ext import six @@ -176,7 +174,7 @@ def create_event(service_key=None, description=None, details=None, trigger_url = 'https://events.pagerduty.com/generic/2010-04-15/create_event.json' if isinstance(details, six.string_types): - details = yaml.safe_load(details) + details = salt.utils.yaml.safe_load(details) if isinstance(details, six.string_types): details = {'details': details} diff --git a/salt/spm/__init__.py b/salt/spm/__init__.py index 8a27e2c7feb..565dae5b893 100644 --- a/salt/spm/__init__.py +++ b/salt/spm/__init__.py @@ -8,7 +8,6 @@ This module provides the point of entry to SPM, the Salt Package Manager # Import Python libs from __future__ import absolute_import, print_function, unicode_literals import os -import yaml import tarfile import shutil import hashlib @@ -36,7 +35,7 @@ import salt.utils.http as http import salt.utils.path import salt.utils.platform import salt.utils.win_functions -from salt.utils.yamldumper import SafeOrderedDumper +import salt.utils.yaml # Get logging started log = logging.getLogger(__name__) @@ -255,7 +254,7 @@ class SPMClient(object): formula_tar = tarfile.open(pkg, 'r:bz2') formula_ref = formula_tar.extractfile('{0}/FORMULA'.format(pkg_name)) - formula_def = yaml.safe_load(formula_ref) + formula_def = salt.utils.yaml.safe_load(formula_ref) file_map[pkg_name] = pkg to_, op_, re_ = self._check_all_deps( @@ -459,7 +458,7 @@ class SPMClient(object): self.ui.status('... installing {0}'.format(pkg_name)) formula_tar = tarfile.open(pkg_file, 'r:bz2') formula_ref = formula_tar.extractfile('{0}/FORMULA'.format(pkg_name)) - formula_def = yaml.safe_load(formula_ref) + formula_def = salt.utils.yaml.safe_load(formula_ref) for field in ('version', 'release', 'summary', 'description'): if field not in formula_def: @@ -625,7 +624,7 @@ class SPMClient(object): for repo_file in repo_files: repo_path = '{0}.d/{1}'.format(self.opts['spm_repos_config'], repo_file) with salt.utils.files.fopen(repo_path) as rph: - repo_data = yaml.safe_load(rph) + repo_data = salt.utils.yaml.safe_load(rph) for repo in repo_data: if repo_data[repo].get('enabled', True) is False: continue @@ -662,7 +661,7 @@ class SPMClient(object): try: if query: if 'SPM-METADATA' in dl_path: - response = yaml.safe_load(query.get('text', '{}')) + response = salt.utils.yaml.safe_load(query.get('text', '{}')) else: response = query.get('text') else: @@ -683,7 +682,7 @@ class SPMClient(object): if dl_path.startswith('file://'): dl_path = dl_path.replace('file://', '') with salt.utils.files.fopen(dl_path, 'r') as rpm: - metadata = yaml.safe_load(rpm) + metadata = salt.utils.yaml.safe_load(rpm) else: metadata = self._query_http(dl_path, repo_info) @@ -738,7 +737,7 @@ class SPMClient(object): spm_name = '-'.join(comps[:-2]) spm_fh = tarfile.open(spm_path, 'r:bz2') formula_handle = spm_fh.extractfile('{0}/FORMULA'.format(spm_name)) - formula_conf = yaml.safe_load(formula_handle.read()) + formula_conf = salt.utils.yaml.safe_load(formula_handle.read()) use_formula = True if spm_name in repo_metadata: @@ -784,13 +783,12 @@ class SPMClient(object): metadata_filename = '{0}/SPM-METADATA'.format(repo_path) with salt.utils.files.fopen(metadata_filename, 'w') as mfh: - yaml.dump( + salt.utils.yaml.safe_dump( repo_metadata, mfh, indent=4, canonical=False, default_flow_style=False, - Dumper=SafeOrderedDumper ) log.debug('Wrote %s', metadata_filename) @@ -900,7 +898,7 @@ class SPMClient(object): formula_tar = tarfile.open(pkg_file, 'r:bz2') formula_ref = formula_tar.extractfile('{0}/FORMULA'.format(name)) - formula_def = yaml.safe_load(formula_ref) + formula_def = salt.utils.yaml.safe_load(formula_ref) self.ui.status(self._get_info(formula_def)) @@ -1017,7 +1015,7 @@ class SPMClient(object): if not os.path.exists(formula_path): raise SPMPackageError('Formula file {0} not found'.format(formula_path)) with salt.utils.files.fopen(formula_path) as fp_: - formula_conf = yaml.safe_load(fp_) + formula_conf = salt.utils.yaml.safe_load(fp_) for field in ('name', 'version', 'release', 'summary', 'description'): if field not in formula_conf: diff --git a/salt/states/boto_apigateway.py b/salt/states/boto_apigateway.py index ee0bcc6c154..2113a3d89a6 100644 --- a/salt/states/boto_apigateway.py +++ b/salt/states/boto_apigateway.py @@ -52,12 +52,11 @@ import hashlib import logging import os import re -import yaml # Import Salt Libs import salt.utils.files import salt.utils.json -from salt.utils.yamlloader import SaltYamlSafeLoader +import salt.utils.yaml # Import 3rd-party libs from salt.ext import six @@ -718,10 +717,7 @@ class _Swagger(object): error_response_template, response_template) with salt.utils.files.fopen(self._swagger_file, 'rb') as sf: - self._cfg = yaml.load( - sf, - Loader=SaltYamlSafeLoader - ) + self._cfg = salt.utils.yaml.safe_load(sf) self._swagger_version = '' else: raise IOError('Invalid swagger file path, {0}'.format(swagger_file_path)) diff --git a/salt/states/boto_cloudfront.py b/salt/states/boto_cloudfront.py index eb4d2ab940d..24a53c57eb8 100644 --- a/salt/states/boto_cloudfront.py +++ b/salt/states/boto_cloudfront.py @@ -49,8 +49,6 @@ from __future__ import absolute_import import difflib import logging -import yaml - log = logging.getLogger(__name__) @@ -176,14 +174,16 @@ def present( ) def _yaml_safe_dump(attrs): - '''Safely dump YAML using a readable flow style''' + ''' + Safely dump YAML using a readable flow style + ''' dumper_name = 'IndentedSafeOrderedDumper' - dumper = __utils__['yamldumper.get_dumper'](dumper_name) - return yaml.dump( + dumper = __utils__['yaml.get_dumper'](dumper_name) + return __utils__['yaml.dump']( attrs, default_flow_style=False, - Dumper=dumper, - ) + Dumper=dumper) + changes_diff = ''.join(difflib.unified_diff( _yaml_safe_dump(full_config_old).splitlines(True), _yaml_safe_dump(full_config_new).splitlines(True), diff --git a/salt/states/boto_s3.py b/salt/states/boto_s3.py index b4e994aa57f..7093a5b2e2a 100644 --- a/salt/states/boto_s3.py +++ b/salt/states/boto_s3.py @@ -55,8 +55,6 @@ import copy import difflib import logging -import yaml - # Import Salt libs import salt.ext.six as six import salt.utils.hashutils @@ -265,14 +263,16 @@ def object_present( action = 'create' def _yaml_safe_dump(attrs): - '''Safely dump YAML using a readable flow style''' + ''' + Safely dump YAML using a readable flow style + ''' dumper_name = 'IndentedSafeOrderedDumper' - dumper = __utils__['yamldumper.get_dumper'](dumper_name) - return yaml.dump( + dumper = __utils__['yaml.get_dumper'](dumper_name) + return __utils__['yaml.dump']( attrs, default_flow_style=False, - Dumper=dumper, - ) + Dumper=dumper) + changes_diff = ''.join(difflib.unified_diff( _yaml_safe_dump(s3_metadata).splitlines(True), _yaml_safe_dump(desired_metadata).splitlines(True), diff --git a/salt/states/boto_sqs.py b/salt/states/boto_sqs.py index 1c385a09b95..e2700ed9cd4 100644 --- a/salt/states/boto_sqs.py +++ b/salt/states/boto_sqs.py @@ -62,7 +62,6 @@ from __future__ import absolute_import # Import Python libs import difflib import logging -import yaml # Import Salt libs import salt.utils.json @@ -204,14 +203,15 @@ def present( final_attributes.update(attrs_to_set) def _yaml_safe_dump(attrs): - '''Safely dump YAML using a readable flow style''' - dumper_name = 'IndentedSafeOrderedDumper' - dumper = __utils__['yamldumper.get_dumper'](dumper_name) - return yaml.dump( + ''' + Safely dump YAML using a readable flow style + ''' + dumper = __utils__['yaml.get_dumper']('IndentedSafeOrderedDumper') + return __utils__['yaml.dump']( attrs, default_flow_style=False, - Dumper=dumper, - ) + Dumper=dumper) + attributes_diff = ''.join(difflib.unified_diff( _yaml_safe_dump(current_attributes).splitlines(True), _yaml_safe_dump(final_attributes).splitlines(True), diff --git a/salt/states/heat.py b/salt/states/heat.py index 0bc32d6a2f4..ed600c897eb 100644 --- a/salt/states/heat.py +++ b/salt/states/heat.py @@ -37,11 +37,11 @@ mysql: # Import Python libs from __future__ import absolute_import import logging -import yaml # Import Salt libs import salt.utils.files import salt.utils.json +import salt.utils.yaml import salt.exceptions # Import 3rd-party libs @@ -55,36 +55,6 @@ try: except ImportError: pass -if hasattr(yaml, 'CSafeLoader'): - YamlLoader = yaml.CSafeLoader -else: - YamlLoader = yaml.SafeLoader - -if hasattr(yaml, 'CSafeDumper'): - YamlDumper = yaml.CSafeDumper -else: - YamlDumper = yaml.SafeDumper - - -def _represent_yaml_str(self, node): - ''' - Represent for yaml - ''' - return self.represent_scalar(node) -YamlDumper.add_representer('tag:yaml.org,2002:str', - _represent_yaml_str) -YamlDumper.add_representer('tag:yaml.org,2002:timestamp', - _represent_yaml_str) - - -def _construct_yaml_str(self, node): - ''' - Construct for yaml - ''' - return self.construct_scalar(node) -YamlLoader.add_constructor('tag:yaml.org,2002:timestamp', - _construct_yaml_str) - logging.basicConfig(level=logging.DEBUG) log = logging.getLogger(__name__) @@ -108,12 +78,9 @@ def _parse_template(tmpl_str): tpl = salt.utils.json.loads(tmpl_str) else: try: - tpl = yaml.load(tmpl_str, Loader=YamlLoader) - except yaml.YAMLError: - try: - tpl = yaml.load(tmpl_str, Loader=yaml.SafeLoader) - except yaml.YAMLError as yea: - raise ValueError(yea) + tpl = salt.utils.yaml.safe_load(tmpl_str) + except salt.utils.yaml.YAMLError as exc: + raise ValueError(six.text_type(exc)) else: if tpl is None: tpl = {} @@ -220,7 +187,7 @@ def deployed(name, template=None, enviroment=None, params=None, poll=5, tpl = tpl.decode('utf-8') template_parse = _parse_template(tpl) if 'heat_template_version' in template_parse: - template_new = yaml.dump(template_parse, Dumper=YamlDumper) + template_new = salt.utils.yaml.safe_dump(template_parse) else: template_new = jsonutils.dumps(template_parse, indent=2, ensure_ascii=False) salt.utils.files.safe_rm(template_tmp_file) diff --git a/salt/states/netyang.py b/salt/states/netyang.py index 5295062743d..dbbc7adeb32 100644 --- a/salt/states/netyang.py +++ b/salt/states/netyang.py @@ -28,11 +28,10 @@ log = logging.getLogger(__file__) # Import third party libs try: - import yaml - # pylint: disable=W0611 + # pylint: disable=unused-import import napalm_yang HAS_NAPALM_YANG = True - # pylint: enable=W0611 + # pylint: enable=unused-import except ImportError: HAS_NAPALM_YANG = False @@ -40,6 +39,7 @@ except ImportError: import salt.utils.files import salt.utils.json import salt.utils.napalm +import salt.utils.yaml # ------------------------------------------------------------------------------ # state properties @@ -157,7 +157,7 @@ def managed(name, data = {'to_dict': data} data = [data] with salt.utils.files.fopen(temp_file, 'w') as file_handle: - yaml.safe_dump(salt.utils.json.loads(salt.utils.json.dumps(data)), file_handle, encoding='utf-8', allow_unicode=True) + salt.utils.yaml.safe_dump(salt.utils.json.loads(salt.utils.json.dumps(data)), file_handle, encoding='utf-8') device_config = __salt__['napalm_yang.parse'](*models, config=True, profiles=profiles) diff --git a/salt/tops/ext_nodes.py b/salt/tops/ext_nodes.py index c71668f1f33..d12fdb93f89 100644 --- a/salt/tops/ext_nodes.py +++ b/salt/tops/ext_nodes.py @@ -47,12 +47,12 @@ The above essentially is the same as a top.sls containing the following: ''' from __future__ import absolute_import, print_function, unicode_literals -# Import python libs +# Import Python libs import logging import subprocess -# Import third party libs -import yaml +# Import Salt libs +import salt.utils.yaml log = logging.getLogger(__name__) @@ -76,12 +76,12 @@ def top(**kwargs): __opts__['master_tops']['ext_nodes'], kwargs['opts']['id'] ) - ndata = yaml.safe_load( - subprocess.Popen( - cmd, - shell=True, - stdout=subprocess.PIPE - ).communicate()[0]) + ndata = salt.utils.yaml.safe_load( + subprocess.Popen( + cmd, + shell=True, + stdout=subprocess.PIPE.communicate()[0]) + ) if not ndata: log.info('master_tops ext_nodes call did not return any data') ret = {} diff --git a/salt/utils/args.py b/salt/utils/args.py index 51db5d23624..6f5e687bd7d 100644 --- a/salt/utils/args.py +++ b/salt/utils/args.py @@ -18,6 +18,7 @@ from salt.ext.six.moves import zip # pylint: disable=import-error,redefined-bui import salt.utils.data import salt.utils.jid import salt.utils.versions +import salt.utils.yaml if six.PY3: @@ -154,19 +155,19 @@ def yamlify_arg(arg): try: # Explicit late import to avoid circular import. DO NOT MOVE THIS. - import salt.utils.yamlloader as yamlloader + import salt.utils.yaml original_arg = arg if '#' in arg: # Only yamlify if it parses into a non-string type, to prevent # loss of content due to # as comment character - parsed_arg = yamlloader.load(arg, Loader=yamlloader.SaltYamlSafeLoader) + parsed_arg = salt.utils.yaml.safe_load(arg) if isinstance(parsed_arg, six.string_types) or parsed_arg is None: return arg return parsed_arg if arg == 'None': arg = None else: - arg = yamlloader.load(arg, Loader=yamlloader.SaltYamlSafeLoader) + arg = salt.utils.yaml.safe_load(arg) if isinstance(arg, dict): # dicts must be wrapped in curly braces diff --git a/salt/utils/cloud.py b/salt/utils/cloud.py index e1a59369910..bfc9902773c 100644 --- a/salt/utils/cloud.py +++ b/salt/utils/cloud.py @@ -53,8 +53,8 @@ import salt.utils.files import salt.utils.platform import salt.utils.versions import salt.utils.vt +import salt.utils.yaml from salt.utils.nb_popen import NonBlockingPopen -from salt.utils.yamldumper import SafeOrderedDumper from salt.utils.validate.path import is_writeable # Import salt cloud libs @@ -72,7 +72,6 @@ from salt.exceptions import ( from salt.ext import six from salt.ext.six.moves import range # pylint: disable=import-error,redefined-builtin,W0611 from jinja2 import Template -import yaml # Let's import pwd and catch the ImportError. We'll raise it if this is not # Windows. This import has to be below where we import salt.utils.platform! @@ -287,10 +286,10 @@ def salt_config_to_yaml(configuration, line_break='\n'): ''' Return a salt configuration dictionary, master or minion, as a yaml dump ''' - return yaml.dump(configuration, - line_break=line_break, - default_flow_style=False, - Dumper=SafeOrderedDumper) + return salt.utils.yaml.safe_dump( + configuration, + line_break=line_break, + default_flow_style=False) def bootstrap(vm_, opts=None): diff --git a/salt/utils/color.py b/salt/utils/color.py index 21efb315dc2..573ff512822 100644 --- a/salt/utils/color.py +++ b/salt/utils/color.py @@ -11,7 +11,6 @@ import os # Import Salt libs from salt.ext import six from salt.textformat import TextFormat -import salt.utils.files log = logging.getLogger(__name__) @@ -21,13 +20,14 @@ def get_color_theme(theme): Return the color theme to use ''' # Keep the heavy lifting out of the module space - import yaml + import salt.utils.files + import salt.utils.yaml if not os.path.isfile(theme): log.warning('The named theme {0} if not available'.format(theme)) try: with salt.utils.files.fopen(theme, 'rb') as fp_: - colors = yaml.safe_load(fp_.read()) + colors = salt.utils.yaml.safe_load(fp_) ret = {} for color in colors: ret[color] = '\033[{0}m'.format(colors[color]) diff --git a/salt/utils/data.py b/salt/utils/data.py index 84ed7ec4508..34178ad106d 100644 --- a/salt/utils/data.py +++ b/salt/utils/data.py @@ -12,11 +12,11 @@ import copy import fnmatch import logging import re -import yaml # Import Salt libs import salt.utils.dictupdate import salt.utils.stringutils +import salt.utils.yaml from salt.defaults import DEFAULT_TARGET_DELIM from salt.exceptions import SaltException from salt.utils.decorators.jinja import jinja_filter @@ -473,8 +473,8 @@ def repack_dictlist(data, ''' if isinstance(data, six.string_types): try: - data = yaml.safe_load(data) - except yaml.parser.ParserError as err: + data = salt.utils.yaml.safe_load(data) + except salt.utils.yaml.parser.ParserError as err: log.error(err) return {} diff --git a/salt/utils/http.py b/salt/utils/http.py index edaf86f2bc5..a70dbbc2919 100644 --- a/salt/utils/http.py +++ b/salt/utils/http.py @@ -13,7 +13,6 @@ import logging import os import pprint import socket -import yaml import io import zlib import gzip @@ -47,6 +46,7 @@ import salt.utils.json import salt.utils.network import salt.utils.platform import salt.utils.stringutils +import salt.utils.yaml import salt.version import salt.utils.xmlutil as xml from salt._compat import ElementTree as ET @@ -657,7 +657,7 @@ def query(url, for item in items: ret['dict'].append(xml.to_dict(item)) elif decode_type == 'yaml': - ret['dict'] = yaml.safe_load(result_text) + ret['dict'] = salt.utils.yaml.safe_load(result_text) else: text = True diff --git a/salt/utils/jinja.py b/salt/utils/jinja.py index f1fcb10df52..01e98d30ef6 100644 --- a/salt/utils/jinja.py +++ b/salt/utils/jinja.py @@ -19,7 +19,6 @@ from xml.etree.ElementTree import Element, SubElement, tostring # Import third party libs import jinja2 from salt.ext import six -import yaml from jinja2 import BaseLoader, Markup, TemplateNotFound, nodes from jinja2.environment import TemplateModule from jinja2.exceptions import TemplateRuntimeError @@ -32,7 +31,7 @@ import salt.utils.data import salt.utils.files import salt.utils.json import salt.utils.url -import salt.utils.yamldumper +import salt.utils.yaml from salt.utils.decorators.jinja import jinja_filter, jinja_test, jinja_global from salt.utils.odict import OrderedDict @@ -785,7 +784,7 @@ class SerializerExtension(Extension, object): return Markup(salt.utils.json.dumps(value, sort_keys=sort_keys, indent=indent).strip()) def format_yaml(self, value, flow_style=True): - yaml_txt = salt.utils.yamldumper.safe_dump( + yaml_txt = salt.utils.yaml.safe_dump( value, default_flow_style=flow_style).strip() if yaml_txt.endswith('\n...'): yaml_txt = yaml_txt[:len(yaml_txt)-4] @@ -844,7 +843,7 @@ class SerializerExtension(Extension, object): if isinstance(value, TemplateModule): value = str(value) try: - return yaml.safe_load(value) + return salt.utils.yaml.safe_load(value) except AttributeError: raise TemplateRuntimeError( 'Unable to load yaml from {0}'.format(value)) diff --git a/salt/utils/kickstart.py b/salt/utils/kickstart.py index f8df726c086..9f9376c780e 100644 --- a/salt/utils/kickstart.py +++ b/salt/utils/kickstart.py @@ -5,10 +5,10 @@ Utilities for managing kickstart .. versionadded:: Beryllium ''' from __future__ import absolute_import -import yaml import shlex import argparse # pylint: disable=minimum-python-version import salt.utils.files +import salt.utils.yaml from salt.ext.six.moves import range @@ -1176,6 +1176,6 @@ def mksls(src, dst=None): if dst: with salt.utils.files.fopen(dst, 'w') as fp_: - fp_.write(yaml.safe_dump(sls, default_flow_style=False)) + salt.utils.yaml.safe_dump(sls, fp_, default_flow_style=False) else: - return yaml.safe_dump(sls, default_flow_style=False) + return salt.utils.yaml.safe_dump(sls, default_flow_style=False) diff --git a/salt/utils/mount.py b/salt/utils/mount.py index 09dbc8bfc19..67f69eb23f5 100644 --- a/salt/utils/mount.py +++ b/salt/utils/mount.py @@ -7,15 +7,13 @@ Common functions for managing mounts from __future__ import absolute_import import logging import os -import yaml # Import Salt libs import salt.utils.files import salt.utils.stringutils +import salt.utils.yaml import salt.utils.versions -from salt.utils.yamldumper import SafeOrderedDumper - log = logging.getLogger(__name__) @@ -25,7 +23,7 @@ def _read_file(path): ''' try: with salt.utils.files.fopen(path, 'rb') as contents: - return yaml.safe_load(contents.read()) + return salt.utils.yaml.safe_load(contents) except (OSError, IOError): return {} @@ -53,11 +51,8 @@ def write_cache(cache, opts): try: _cache = salt.utils.stringutils.to_bytes( - yaml.dump( - cache, - Dumper=SafeOrderedDumper - ) - ) + salt.utils.yaml.safe_dump(cache) + ) with salt.utils.files.fopen(cache_file, 'wb+') as fp_: fp_.write(_cache) return True diff --git a/salt/utils/parsers.py b/salt/utils/parsers.py index fcb11d8761b..8d89932b140 100644 --- a/salt/utils/parsers.py +++ b/salt/utils/parsers.py @@ -20,7 +20,6 @@ import getpass import logging import optparse import traceback -import yaml from functools import partial @@ -39,13 +38,13 @@ import salt.utils.process import salt.utils.stringutils import salt.utils.user import salt.utils.xdg +import salt.utils.yaml from salt.defaults import DEFAULT_TARGET_DELIM from salt.utils.validate.path import is_writeable from salt.utils.verify import verify_files import salt.exceptions from salt.ext import six from salt.ext.six.moves import range # pylint: disable=import-error,redefined-builtin -from salt.utils.yamldumper import SafeOrderedDumper def _sorted(mixins_or_funcs): @@ -2091,11 +2090,9 @@ class SaltCMDOptionParser(six.with_metaclass(OptionParserMeta, if self.options.config_dump: cfg = config.master_config(self.get_config_file_path()) sys.stdout.write( - yaml.dump( + salt.utils.yaml.safe_dump( cfg, - default_flow_style=False, - Dumper=SafeOrderedDumper - ) + default_flow_style=False) ) sys.exit(salt.defaults.exitcodes.EX_OK) diff --git a/salt/utils/preseed.py b/salt/utils/preseed.py index acebe16dea5..1406271a8f6 100644 --- a/salt/utils/preseed.py +++ b/salt/utils/preseed.py @@ -5,9 +5,9 @@ Utilities for managing Debian preseed .. versionadded:: Beryllium ''' from __future__ import absolute_import -import yaml import shlex import salt.utils.files +import salt.utils.yaml def mksls(src, dst=None): @@ -73,6 +73,6 @@ def mksls(src, dst=None): if dst is not None: with salt.utils.files.fopen(dst, 'w') as fh_: - fh_.write(yaml.safe_dump(sls, default_flow_style=False)) + salt.utils.yaml.safe_dump(sls, fh_, default_flow_style=False) else: - return yaml.safe_dump(sls, default_flow_style=False) + return salt.utils.yaml.safe_dump(sls, default_flow_style=False) diff --git a/salt/utils/reactor.py b/salt/utils/reactor.py index debb23598e7..0d95db5854e 100644 --- a/salt/utils/reactor.py +++ b/salt/utils/reactor.py @@ -20,11 +20,11 @@ import salt.utils.data import salt.utils.event import salt.utils.files import salt.utils.process +import salt.utils.yaml import salt.wheel import salt.defaults.exitcodes # Import 3rd-party libs -import yaml from salt.ext import six log = logging.getLogger(__name__) @@ -111,7 +111,7 @@ class Reactor(salt.utils.process.SignalHandlingMultiprocessingProcess, salt.stat if isinstance(self.opts['reactor'], six.string_types): try: with salt.utils.files.fopen(self.opts['reactor']) as fp_: - react_map = yaml.safe_load(fp_.read()) + react_map = salt.utils.yaml.safe_load(fp_) except (OSError, IOError): log.error( 'Failed to read reactor map: "{0}"'.format( @@ -148,7 +148,7 @@ class Reactor(salt.utils.process.SignalHandlingMultiprocessingProcess, salt.stat log.debug('Reading reactors from yaml {0}'.format(self.opts['reactor'])) try: with salt.utils.files.fopen(self.opts['reactor']) as fp_: - react_map = yaml.safe_load(fp_.read()) + react_map = salt.utils.yaml.safe_load(fp_) except (OSError, IOError): log.error( 'Failed to read reactor map: "{0}"'.format( diff --git a/salt/utils/saltclass.py b/salt/utils/saltclass.py index c12ff57f5eb..06810547b3e 100644 --- a/salt/utils/saltclass.py +++ b/salt/utils/saltclass.py @@ -4,11 +4,11 @@ from __future__ import absolute_import import os import re import logging -import yaml from jinja2 import FileSystemLoader, Environment # Import Salt libs import salt.utils.path +import salt.utils.yaml # Import 3rd-party libs from salt.ext.six import iteritems @@ -32,7 +32,7 @@ def render_jinja(_file, salt_data): # Renders yaml from rendered jinja def render_yaml(_file, salt_data): - return yaml.safe_load(render_jinja(_file, salt_data)) + return salt.utils.yaml.safe_load(render_jinja(_file, salt_data)) # Returns a dict from a class yaml definition diff --git a/salt/utils/schedule.py b/salt/utils/schedule.py index ac9a61d26e5..a5061baec9f 100644 --- a/salt/utils/schedule.py +++ b/salt/utils/schedule.py @@ -24,7 +24,6 @@ import logging import errno import random import weakref -import yaml # Import Salt libs import salt.config @@ -38,6 +37,7 @@ import salt.utils.platform import salt.utils.process import salt.utils.stringutils import salt.utils.user +import salt.utils.yaml import salt.loader import salt.minion import salt.payload @@ -46,7 +46,6 @@ import salt.exceptions import salt.log.setup as log_setup import salt.defaults.exitcodes from salt.utils.odict import OrderedDict -from salt.utils.yamldumper import SafeOrderedDumper # Import 3rd-party libs from salt.ext import six @@ -176,9 +175,8 @@ class Schedule(object): with salt.utils.files.fopen(schedule_conf, 'wb+') as fp_: fp_.write( salt.utils.stringutils.to_bytes( - yaml.dump( - {'schedule': self._get_schedule(include_pillar=False)}, - Dumper=SafeOrderedDumper + salt.utils.yaml.safe_dump( + {'schedule': self._get_schedule(include_pillar=False)} ) ) ) diff --git a/salt/utils/schema.py b/salt/utils/schema.py index 2d8984cd458..4de3374596c 100644 --- a/salt/utils/schema.py +++ b/salt/utils/schema.py @@ -328,10 +328,10 @@ import functools # Import salt libs import salt.utils.args +#import salt.utils.yaml from salt.utils.odict import OrderedDict # Import 3rd-party libs -#import yaml from salt.ext import six BASE_SCHEMA_URL = 'https://non-existing.saltstack.com/schemas' @@ -871,7 +871,7 @@ class BaseSchemaItem(SchemaItem): # width=RENDER_COMMENT_YAML_MAX_LINE_LENGTH, # initial_indent='# ')) # output += '\n' - # yamled_default_value = yaml.dump(self.default, default_flow_style=False).split('\n...', 1)[0] + # yamled_default_value = salt.utils.yaml.safe_dump(self.default, default_flow_style=False).split('\n...', 1)[0] # output += '# Default: {0}\n'.format(yamled_default_value) # output += '#{0}: {1}\n'.format(name, yamled_default_value) # output += '# <---- ' diff --git a/salt/utils/thin.py b/salt/utils/thin.py index aa8176d7f5a..017b8486065 100644 --- a/salt/utils/thin.py +++ b/salt/utils/thin.py @@ -512,6 +512,8 @@ def gen_min(cachedir, extra_mods='', overwrite=False, so_mods='', 'salt/utils/vt.py', 'salt/utils/templates.py', 'salt/utils/aggregation.py', + 'salt/utils/yaml.py', + 'salt/utils/yamldumper.py', 'salt/utils/yamlloader.py', 'salt/utils/event.py', 'salt/serializers', diff --git a/salt/utils/yaml.py b/salt/utils/yaml.py new file mode 100644 index 00000000000..7fbfbe6035c --- /dev/null +++ b/salt/utils/yaml.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +''' +Convenience module that provides our custom loader and dumper in a single module +''' +from __future__ import absolute_import, print_function, unicode_literals + +# pylint: disable=wildcard-import,unused-wildcard-import,unused-import +from yaml import YAMLError, parser, scanner +from salt.utils.yamldumper import * +from salt.utils.yamlloader import * diff --git a/salt/utils/yamldumper.py b/salt/utils/yamldumper.py index 514aae9613c..d8d0aaac442 100644 --- a/salt/utils/yamldumper.py +++ b/salt/utils/yamldumper.py @@ -15,7 +15,7 @@ except ImportError: from yaml import Dumper from yaml import SafeDumper -import yaml +import yaml # pylint: disable=blacklisted-import import collections from salt.utils.odict import OrderedDict @@ -27,6 +27,9 @@ except ImportError: odict = None HAS_IOFLO = False +__all__ = ['OrderedDumper', 'SafeOrderedDumper', 'IndentedSafeOrderedDumper', + 'get_dumper', 'dump', 'safe_dump'] + class IndentMixin(Dumper): ''' @@ -75,6 +78,13 @@ SafeOrderedDumper.add_representer( yaml.representer.SafeRepresenter.represent_dict ) +OrderedDumper.add_representer( + 'tag:yaml.org,2002:timestamp', + OrderedDumper.represent_scalar) +SafeOrderedDumper.add_representer( + 'tag:yaml.org,2002:timestamp', + SafeOrderedDumper.represent_scalar) + if HAS_IOFLO: OrderedDumper.add_representer(odict, represent_ordereddict) SafeOrderedDumper.add_representer(odict, represent_ordereddict) @@ -88,9 +98,24 @@ def get_dumper(dumper_name): }.get(dumper_name) +def dump(data, stream=None, **kwargs): + ''' + .. versionadded:: Oxygen + + Helper that wraps yaml.dump and ensures that we encode unicode strings + unless explicitly told not to. + ''' + if 'allow_unicode' not in kwargs: + kwargs['allow_unicode'] = True + return yaml.dump(data, stream, **kwargs) + + def safe_dump(data, stream=None, **kwargs): ''' Use a custom dumper to ensure that defaultdict and OrderedDict are - represented properly + represented properly. Ensure that unicode strings are encoded unless + explicitly told not to. ''' + if 'allow_unicode' not in kwargs: + kwargs['allow_unicode'] = True return yaml.dump(data, stream, Dumper=SafeOrderedDumper, **kwargs) diff --git a/salt/utils/yamlencoding.py b/salt/utils/yamlencoding.py index 2f8ac8fb32e..2b21d776864 100644 --- a/salt/utils/yamlencoding.py +++ b/salt/utils/yamlencoding.py @@ -8,7 +8,7 @@ from __future__ import absolute_import import io # Import 3rd-party libs -import yaml +import yaml # pylint: disable=blacklisted-import from salt.ext import six # Import salt libs diff --git a/salt/utils/yamlloader.py b/salt/utils/yamlloader.py index 032db42c780..b0cac25593b 100644 --- a/salt/utils/yamlloader.py +++ b/salt/utils/yamlloader.py @@ -9,7 +9,7 @@ import warnings # Import third party libs import re -import yaml +import yaml # pylint: disable=blacklisted-import from yaml.nodes import MappingNode, SequenceNode from yaml.constructor import ConstructorError try: @@ -18,11 +18,7 @@ try: except Exception: pass -# This function is safe and needs to stay as yaml.load. The load function -# accepts a custom loader, and every time this function is used in Salt -# the custom loader defined below is used. This should be altered though to -# not require the custom loader to be explicitly added. -load = yaml.load # pylint: disable=C0103 +__all__ = ['SaltYamlSafeLoader', 'load', 'safe_load'] class DuplicateKeyWarning(RuntimeWarning): @@ -53,6 +49,9 @@ class SaltYamlSafeLoader(yaml.SafeLoader, object): self.add_constructor( 'tag:yaml.org,2002:python/unicode', type(self).construct_unicode) + self.add_constructor( + 'tag:yaml.org,2002:timestamp', + type(self).construct_scalar) self.dictclass = dictclass def construct_yaml_map(self, node): @@ -153,3 +152,16 @@ class SaltYamlSafeLoader(yaml.SafeLoader, object): mergeable_items = [x for x in merge if x[0].value not in existing_nodes] node.value = mergeable_items + node.value + + +def load(stream, Loader=SaltYamlSafeLoader): + return yaml.load(stream, Loader=Loader) + + +def safe_load(stream, Loader=SaltYamlSafeLoader): + ''' + .. versionadded:: Oxygen + + Helper function which automagically uses our custom loader. + ''' + return yaml.load(stream, Loader=Loader) diff --git a/salt/utils/yast.py b/salt/utils/yast.py index 481e1dcf07a..de0c85ff025 100644 --- a/salt/utils/yast.py +++ b/salt/utils/yast.py @@ -8,7 +8,7 @@ from __future__ import absolute_import from salt._compat import ElementTree as ET import salt.utils.xmlutil as xml import salt.utils.files -import yaml +import salt.utils.yaml def mksls(src, dst=None): @@ -20,6 +20,6 @@ def mksls(src, dst=None): if dst is not None: with salt.utils.files.fopen(dst, 'w') as fh_: - fh_.write(yaml.safe_dump(ps_opts, default_flow_style=False)) + salt.utils.yaml.safe_dump(ps_opts, fh_, default_flow_style=False) else: - return yaml.safe_dump(ps_opts, default_flow_style=False) + return salt.utils.yaml.safe_dump(ps_opts, default_flow_style=False) diff --git a/salt/wheel/config.py b/salt/wheel/config.py index 26da2725975..cf6d0bc0ad9 100644 --- a/salt/wheel/config.py +++ b/salt/wheel/config.py @@ -8,13 +8,10 @@ from __future__ import absolute_import, print_function, unicode_literals import logging import os -# Import third party libs -import yaml - # Import salt libs import salt.config import salt.utils.files -from salt.utils.yamldumper import SafeOrderedDumper +import salt.utils.yaml # Import 3rd-party libs from salt.ext import six @@ -44,13 +41,7 @@ def apply(key, value): data = values() data[key] = value with salt.utils.files.fopen(path, 'w+') as fp_: - fp_.write( - yaml.dump( - data, - default_flow_style=False, - Dumper=SafeOrderedDumper - ) - ) + salt.utils.yaml.safe_dump(data, default_flow_style=False) def update_config(file_name, yaml_contents): @@ -84,7 +75,7 @@ def update_config(file_name, yaml_contents): dir_path = os.path.join(__opts__['config_dir'], os.path.dirname(__opts__['default_include'])) try: - yaml_out = yaml.safe_dump(yaml_contents, default_flow_style=False) + yaml_out = salt.utils.yaml.safe_dump(yaml_contents, default_flow_style=False) if not os.path.exists(dir_path): log.debug('Creating directory %s', dir_path) @@ -95,5 +86,5 @@ def update_config(file_name, yaml_contents): fp_.write(yaml_out) return 'Wrote {0}'.format(file_name) - except (IOError, OSError, yaml.YAMLError, ValueError) as err: + except (IOError, OSError, salt.utils.yaml.YAMLError, ValueError) as err: return six.text_type(err) diff --git a/tests/consist.py b/tests/consist.py index 28c59c6503d..6b3890ac2ec 100644 --- a/tests/consist.py +++ b/tests/consist.py @@ -11,9 +11,9 @@ import optparse # Import Salt libs import salt.utils.color import salt.utils.files +import salt.utils.yaml # Import 3rd-party libs -import yaml from salt.ext import six colors = salt.utils.color.get_colors() @@ -47,7 +47,7 @@ def run(command): cmd = r'salt \* {0} --yaml-out -t 500 > high'.format(command) subprocess.call(cmd, shell=True) with salt.utils.files.fopen('high') as fp_: - data = yaml.load(fp_) + data = salt.utils.yaml.safe_load(fp_) hashes = set() for key, val in six.iteritems(data): has = hashlib.md5(str(val)).hexdigest() diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index 9c9ccb5714b..f6209725763 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -54,6 +54,7 @@ import salt.utils.path import salt.utils.platform import salt.utils.process import salt.utils.stringutils +import salt.utils.yaml import salt.log.setup as salt_log_setup from salt.utils.verify import verify_env from salt.utils.immutabletypes import freeze @@ -67,7 +68,6 @@ except ImportError: pass # Import 3rd-party libs -import yaml import msgpack from salt.ext import six from salt.ext.six.moves import cStringIO @@ -933,24 +933,18 @@ class TestDaemon(object): for entry in ('master', 'minion', 'sub_minion', 'syndic', 'syndic_master', 'proxy'): computed_config = copy.deepcopy(locals()['{0}_opts'.format(entry)]) with salt.utils.files.fopen(os.path.join(RUNTIME_VARS.TMP_CONF_DIR, entry), 'w') as fp_: - fp_.write(yaml.dump(computed_config, default_flow_style=False)) + salt.utils.yaml.safe_dump(computed_config, fp_, default_flow_style=False) sub_minion_computed_config = copy.deepcopy(sub_minion_opts) with salt.utils.files.fopen(os.path.join(RUNTIME_VARS.TMP_SUB_MINION_CONF_DIR, 'minion'), 'w') as wfh: - wfh.write( - yaml.dump(sub_minion_computed_config, default_flow_style=False) - ) + salt.utils.yaml.safe_dump(sub_minion_computed_config, wfh, default_flow_style=False) shutil.copyfile(os.path.join(RUNTIME_VARS.TMP_CONF_DIR, 'master'), os.path.join(RUNTIME_VARS.TMP_SUB_MINION_CONF_DIR, 'master')) syndic_master_computed_config = copy.deepcopy(syndic_master_opts) with salt.utils.files.fopen(os.path.join(RUNTIME_VARS.TMP_SYNDIC_MASTER_CONF_DIR, 'master'), 'w') as wfh: - wfh.write( - yaml.dump(syndic_master_computed_config, default_flow_style=False) - ) + salt.utils.yaml.safe_dump(syndic_master_computed_config, wfh, default_flow_style=False) syndic_computed_config = copy.deepcopy(syndic_opts) with salt.utils.files.fopen(os.path.join(RUNTIME_VARS.TMP_SYNDIC_MINION_CONF_DIR, 'minion'), 'w') as wfh: - wfh.write( - yaml.dump(syndic_computed_config, default_flow_style=False) - ) + salt.utils.yaml.safe_dump(syndic_computed_config, wfh, default_flow_style=False) shutil.copyfile(os.path.join(RUNTIME_VARS.TMP_CONF_DIR, 'master'), os.path.join(RUNTIME_VARS.TMP_SYNDIC_MINION_CONF_DIR, 'master')) # <---- Transcribe Configuration ----------------------------------------------------------------------------- diff --git a/tests/integration/minion/test_pillar.py b/tests/integration/minion/test_pillar.py index a2fa5879140..e1806aa3d0e 100644 --- a/tests/integration/minion/test_pillar.py +++ b/tests/integration/minion/test_pillar.py @@ -20,12 +20,12 @@ from tests.support.unit import skipIf from tests.support.helpers import requires_system_grains # Import 3rd-party libs -import yaml from salt.ext import six # Import salt libs import salt.utils.files import salt.utils.path +import salt.utils.yaml import salt.pillar as pillar log = logging.getLogger(__name__) @@ -272,7 +272,7 @@ class DecryptGPGPillarTest(ModuleCase): Test recursive decryption of secrets:vault as well as the fallback to default decryption renderer. ''' - decrypt_pillar_opts = yaml.safe_load(textwrap.dedent('''\ + decrypt_pillar_opts = salt.utils.yaml.safe_load(textwrap.dedent('''\ decrypt_pillar: - 'secrets:vault' ''')) @@ -287,7 +287,7 @@ class DecryptGPGPillarTest(ModuleCase): Test recursive decryption of secrets:vault using a pipe instead of a colon as the nesting delimiter. ''' - decrypt_pillar_opts = yaml.safe_load(textwrap.dedent('''\ + decrypt_pillar_opts = salt.utils.yaml.safe_load(textwrap.dedent('''\ decrypt_pillar_delimiter: '|' decrypt_pillar: - 'secrets|vault' @@ -303,7 +303,7 @@ class DecryptGPGPillarTest(ModuleCase): Test recursive decryption, only with a more deeply-nested target. This should leave the other keys in secrets:vault encrypted. ''' - decrypt_pillar_opts = yaml.safe_load(textwrap.dedent('''\ + decrypt_pillar_opts = salt.utils.yaml.safe_load(textwrap.dedent('''\ decrypt_pillar: - 'secrets:vault:qux' ''')) @@ -322,7 +322,7 @@ class DecryptGPGPillarTest(ModuleCase): explicitly defined, overriding the default. Setting the default to a nonexistant renderer so we can be sure that the override happened. ''' - decrypt_pillar_opts = yaml.safe_load(textwrap.dedent('''\ + decrypt_pillar_opts = salt.utils.yaml.safe_load(textwrap.dedent('''\ decrypt_pillar_default: asdf decrypt_pillar_renderers: - asdf @@ -341,7 +341,7 @@ class DecryptGPGPillarTest(ModuleCase): Test decryption using a missing renderer. It should fail, leaving the encrypted keys intact, and add an error to the pillar dictionary. ''' - decrypt_pillar_opts = yaml.safe_load(textwrap.dedent('''\ + decrypt_pillar_opts = salt.utils.yaml.safe_load(textwrap.dedent('''\ decrypt_pillar_default: asdf decrypt_pillar_renderers: - asdf @@ -365,7 +365,7 @@ class DecryptGPGPillarTest(ModuleCase): fail, leaving the encrypted keys intact, and add an error to the pillar dictionary. ''' - decrypt_pillar_opts = yaml.safe_load(textwrap.dedent('''\ + decrypt_pillar_opts = salt.utils.yaml.safe_load(textwrap.dedent('''\ decrypt_pillar_default: foo decrypt_pillar_renderers: - foo diff --git a/tests/integration/modules/test_state.py b/tests/integration/modules/test_state.py index 1a0293bac86..63649cbad65 100644 --- a/tests/integration/modules/test_state.py +++ b/tests/integration/modules/test_state.py @@ -1650,10 +1650,10 @@ class StateModuleTest(ModuleCase, SaltReturnAssertsMixin): ''' helper class to add pillar data at runtime ''' - import yaml + import salt.utils.yaml with salt.utils.files.fopen(os.path.join(TMP_PILLAR_TREE, 'pillar.sls'), 'w') as fp: - fp.write(yaml.dump(pillar)) + salt.utils.yaml.safe_dump(pillar, fp) with salt.utils.files.fopen(os.path.join(TMP_PILLAR_TREE, 'top.sls'), 'w') as fp: fp.write(textwrap.dedent('''\ diff --git a/tests/integration/runners/test_runner_returns.py b/tests/integration/runners/test_runner_returns.py index 63e607773f2..d998bb971d6 100644 --- a/tests/integration/runners/test_runner_returns.py +++ b/tests/integration/runners/test_runner_returns.py @@ -7,7 +7,6 @@ from __future__ import absolute_import, print_function, unicode_literals import errno import os import tempfile -import yaml # Import Salt Testing libs from tests.support.case import ShellCase @@ -18,6 +17,7 @@ import salt.payload import salt.utils.args import salt.utils.files import salt.utils.jid +import salt.utils.yaml class RunnerReturnsTest(ShellCase): @@ -73,7 +73,7 @@ class RunnerReturnsTest(ShellCase): ''' Dump the config dict to the conf file ''' - self.conf.write(yaml.dump(data, default_flow_style=False)) + self.conf.write(salt.utils.yaml.safe_dump(data, default_flow_style=False)) self.conf.flush() def test_runner_returns_disabled(self): diff --git a/tests/integration/runners/test_state.py b/tests/integration/runners/test_state.py index 5283ff90c05..82e3ebf29c6 100644 --- a/tests/integration/runners/test_state.py +++ b/tests/integration/runners/test_state.py @@ -11,7 +11,6 @@ import shutil import signal import tempfile import textwrap -import yaml import threading from salt.ext.six.moves import queue @@ -26,6 +25,7 @@ import salt.utils.event import salt.utils.files import salt.utils.json import salt.utils.stringutils +import salt.utils.yaml # Import 3rd-party libs from salt.ext import six @@ -275,7 +275,7 @@ class OrchEventTest(ShellCase): ''' Dump the config dict to the conf file ''' - self.conf.write(yaml.dump(data, default_flow_style=False)) + self.conf.write(salt.utils.yaml.safe_dump(data, default_flow_style=False)) self.conf.flush() def test_jid_in_ret_event(self): diff --git a/tests/integration/shell/test_call.py b/tests/integration/shell/test_call.py index 2d5b74684ce..3dff8fea5e5 100644 --- a/tests/integration/shell/test_call.py +++ b/tests/integration/shell/test_call.py @@ -17,9 +17,6 @@ import shutil from datetime import datetime import logging -# Import 3rd-party libs -import yaml - # Import Salt Testing libs from tests.support.case import ShellCase from tests.support.unit import skipIf @@ -30,6 +27,7 @@ from tests.integration.utils import testprogram # Import salt libs import salt.utils.files +import salt.utils.yaml from salt.ext import six log = logging.getLogger(__name__) @@ -175,7 +173,7 @@ class CallTest(ShellCase, testprogram.TestProgramCase, ShellCaseCommonTestsMixin os.makedirs(config_dir) with salt.utils.files.fopen(self.get_config_file_path('master')) as fhr: - master_config = yaml.load(fhr.read()) + master_config = salt.utils.yaml.safe_load(fhr) master_root_dir = master_config['root_dir'] this_minion_key = os.path.join( @@ -205,9 +203,7 @@ class CallTest(ShellCase, testprogram.TestProgramCase, ShellCaseCommonTestsMixin # Let's first test with a master running with salt.utils.files.fopen(minion_config_file, 'w') as fh_: - fh_.write( - yaml.dump(minion_config, default_flow_style=False) - ) + salt.utils.yaml.safe_dump(minion_config, fh_, default_flow_style=False) ret = self.run_script( 'salt-call', '--config-dir {0} cmd.run "echo foo"'.format( @@ -235,9 +231,7 @@ class CallTest(ShellCase, testprogram.TestProgramCase, ShellCaseCommonTestsMixin minion_config.pop('master') minion_config.pop('master_port') with salt.utils.files.fopen(minion_config_file, 'w') as fh_: - fh_.write( - yaml.dump(minion_config, default_flow_style=False) - ) + salt.utils.yaml.safe_dump(minion_config, fh_, default_flow_style=False) out = self.run_script( 'salt-call', @@ -283,9 +277,7 @@ class CallTest(ShellCase, testprogram.TestProgramCase, ShellCaseCommonTestsMixin # Should work with local file client minion_config['file_client'] = 'local' with salt.utils.files.fopen(minion_config_file, 'w') as fh_: - fh_.write( - yaml.dump(minion_config, default_flow_style=False) - ) + salt.utils.yaml.safe_dump(minion_config, fh_, default_flow_style=False) ret = self.run_script( 'salt-call', '--config-dir {0} cmd.run "echo foo"'.format( @@ -310,12 +302,10 @@ class CallTest(ShellCase, testprogram.TestProgramCase, ShellCaseCommonTestsMixin os.chdir(config_dir) with salt.utils.files.fopen(self.get_config_file_path('minion'), 'r') as fh_: - minion_config = yaml.load(fh_.read()) + minion_config = salt.utils.yaml.safe_load(fh_) minion_config['log_file'] = 'file:///dev/log/LOG_LOCAL3' with salt.utils.files.fopen(os.path.join(config_dir, 'minion'), 'w') as fh_: - fh_.write( - yaml.dump(minion_config, default_flow_style=False) - ) + salt.utils.yaml.safe_dump(minion_config, fh_, default_flow_style=False) ret = self.run_script( 'salt-call', '--config-dir {0} cmd.run "echo foo"'.format( diff --git a/tests/integration/shell/test_cp.py b/tests/integration/shell/test_cp.py index 1e7a5978e1b..523b19a176a 100644 --- a/tests/integration/shell/test_cp.py +++ b/tests/integration/shell/test_cp.py @@ -15,9 +15,6 @@ import pipes import shutil import tempfile -# Import 3rd-party libs -import yaml - # Import Salt Testing libs from tests.support.case import ShellCase from tests.support.paths import TMP @@ -25,6 +22,7 @@ from tests.support.mixins import ShellCaseCommonTestsMixin # Import salt libs import salt.utils.files +import salt.utils.yaml # Import 3rd-party libs from salt.ext import six @@ -42,7 +40,7 @@ class CopyTest(ShellCase, ShellCaseCommonTestsMixin): for line in self.run_salt('--out yaml "*" test.ping'): if not line: continue - data = yaml.load(line) + data = salt.utils.yaml.safe_load(line) minions.extend(data.keys()) self.assertNotEqual(minions, []) @@ -62,7 +60,7 @@ class CopyTest(ShellCase, ShellCaseCommonTestsMixin): pipes.quote(minion), TMP ) ) - data = yaml.load('\n'.join(ret)) + data = salt.utils.yaml.safe_load('\n'.join(ret)) if data[minion] is False: ret = self.run_salt( '--out yaml {0} file.makedirs {1}'.format( @@ -71,7 +69,7 @@ class CopyTest(ShellCase, ShellCaseCommonTestsMixin): ) ) - data = yaml.load('\n'.join(ret)) + data = salt.utils.yaml.safe_load('\n'.join(ret)) self.assertTrue(data[minion]) minion_testfile = os.path.join( @@ -84,7 +82,7 @@ class CopyTest(ShellCase, ShellCaseCommonTestsMixin): pipes.quote(minion_testfile) )) - data = yaml.load('\n'.join(ret)) + data = salt.utils.yaml.safe_load('\n'.join(ret)) for part in six.itervalues(data): self.assertTrue(part[minion_testfile]) @@ -94,7 +92,7 @@ class CopyTest(ShellCase, ShellCaseCommonTestsMixin): pipes.quote(minion_testfile) ) ) - data = yaml.load('\n'.join(ret)) + data = salt.utils.yaml.safe_load('\n'.join(ret)) self.assertTrue(data[minion]) ret = self.run_salt( @@ -104,7 +102,7 @@ class CopyTest(ShellCase, ShellCaseCommonTestsMixin): pipes.quote(testfile_contents) ) ) - data = yaml.load('\n'.join(ret)) + data = salt.utils.yaml.safe_load('\n'.join(ret)) self.assertTrue(data[minion]) ret = self.run_salt( '--out yaml {0} file.remove {1}'.format( @@ -112,7 +110,7 @@ class CopyTest(ShellCase, ShellCaseCommonTestsMixin): pipes.quote(minion_testfile) ) ) - data = yaml.load('\n'.join(ret)) + data = salt.utils.yaml.safe_load('\n'.join(ret)) self.assertTrue(data[minion]) def test_issue_7754(self): @@ -126,12 +124,10 @@ class CopyTest(ShellCase, ShellCaseCommonTestsMixin): config_file_name = 'master' with salt.utils.files.fopen(self.get_config_file_path(config_file_name), 'r') as fhr: - config = yaml.load(fhr.read()) + config = salt.utils.yaml.safe_load(fhr) config['log_file'] = 'file:///dev/log/LOG_LOCAL3' with salt.utils.files.fopen(os.path.join(config_dir, config_file_name), 'w') as fhw: - fhw.write( - yaml.dump(config, default_flow_style=False) - ) + salt.utils.yaml.safe_dump(config, fhw, default_flow_style=False) try: fd_, fn_ = tempfile.mkstemp() diff --git a/tests/integration/shell/test_key.py b/tests/integration/shell/test_key.py index 9600c5d1849..b22b66eab83 100644 --- a/tests/integration/shell/test_key.py +++ b/tests/integration/shell/test_key.py @@ -13,12 +13,12 @@ from tests.support.paths import TMP from tests.support.mixins import ShellCaseCommonTestsMixin # Import 3rd-party libs -import yaml from salt.ext import six # Import Salt libs import salt.utils.files import salt.utils.platform +import salt.utils.yaml USERA = 'saltdev' USERA_PWD = 'saltdev' @@ -156,8 +156,8 @@ class KeyTest(ShellCase, ShellCaseCommonTestsMixin): data = self.run_key('-L --out yaml') ret = {} try: - import yaml - ret = yaml.load('\n'.join(data)) + import salt.utils.yaml + ret = salt.utils.yaml.safe_load('\n'.join(data)) except Exception: pass @@ -289,12 +289,10 @@ class KeyTest(ShellCase, ShellCaseCommonTestsMixin): config_file_name = 'master' with salt.utils.files.fopen(self.get_config_file_path(config_file_name), 'r') as fhr: - config = yaml.load(fhr.read()) + config = salt.utils.yaml.safe_load(fhr) config['log_file'] = 'file:///dev/log/LOG_LOCAL3' with salt.utils.files.fopen(os.path.join(config_dir, config_file_name), 'w') as fhw: - fhw.write( - yaml.dump(config, default_flow_style=False) - ) + salt.utils.yaml.safe_dump(config, fhw, default_flow_style=False) ret = self.run_script( self._call_binary_, '--config-dir {0} -L'.format( diff --git a/tests/integration/shell/test_master.py b/tests/integration/shell/test_master.py index 929c9a9653a..39acaec9662 100644 --- a/tests/integration/shell/test_master.py +++ b/tests/integration/shell/test_master.py @@ -13,11 +13,9 @@ import os import signal import shutil -# Import 3rd-party libs -import yaml - # Import salt libs import salt.utils.files +import salt.utils.yaml # Import salt test libs import tests.integration.utils @@ -42,16 +40,14 @@ class MasterTest(ShellCase, testprogram.TestProgramCase, ShellCaseCommonTestsMix config_file_name = 'master' pid_path = os.path.join(config_dir, '{0}.pid'.format(config_file_name)) with salt.utils.files.fopen(self.get_config_file_path(config_file_name), 'r') as fhr: - config = yaml.load(fhr.read()) + config = salt.utils.yaml.safe_load(fhr) config['root_dir'] = config_dir config['log_file'] = 'file:///tmp/log/LOG_LOCAL3' config['ret_port'] = config['ret_port'] + 10 config['publish_port'] = config['publish_port'] + 10 with salt.utils.files.fopen(os.path.join(config_dir, config_file_name), 'w') as fhw: - fhw.write( - yaml.dump(config, default_flow_style=False) - ) + salt.utils.yaml.safe_dump(config, fhw, default_flow_style=False) ret = self.run_script( self._call_binary_, diff --git a/tests/integration/shell/test_matcher.py b/tests/integration/shell/test_matcher.py index f6b371e2bb7..b1a809021d1 100644 --- a/tests/integration/shell/test_matcher.py +++ b/tests/integration/shell/test_matcher.py @@ -6,9 +6,6 @@ import os import shutil import time -# Import 3rd-party libs -import yaml - # Import Salt Testing libs from tests.support.case import ShellCase from tests.support.paths import TMP @@ -16,6 +13,7 @@ from tests.support.mixins import ShellCaseCommonTestsMixin # Import salt libs import salt.utils.files +import salt.utils.yaml def minion_in_returns(minion, lines): @@ -285,7 +283,7 @@ class MatchTest(ShellCase, ShellCaseCommonTestsMixin): def test_ipcidr(self): subnets_data = self.run_salt('--out yaml \'*\' network.subnets') - yaml_data = yaml.load('\n'.join(subnets_data)) + yaml_data = salt.utils.yaml.safe_load('\n'.join(subnets_data)) # We're just after the first defined subnet from 'minion' subnet = yaml_data['minion'][0] @@ -349,14 +347,10 @@ class MatchTest(ShellCase, ShellCaseCommonTestsMixin): config_file_name = 'master' with salt.utils.files.fopen(self.get_config_file_path(config_file_name), 'r') as fhr: - config = yaml.load( - fhr.read() - ) + config = salt.utils.yaml.safe_load(fhr) config['log_file'] = 'file:///dev/log/LOG_LOCAL3' with salt.utils.files.fopen(os.path.join(config_dir, config_file_name), 'w') as fhw: - fhw.write( - yaml.dump(config, default_flow_style=False) - ) + salt.utils.yaml.safe_dump(config, fhw, default_flow_style=False) ret = self.run_script( self._call_binary_, '--config-dir {0} minion test.ping'.format( diff --git a/tests/integration/shell/test_minion.py b/tests/integration/shell/test_minion.py index 6f440ae6831..fc6bda2d03a 100644 --- a/tests/integration/shell/test_minion.py +++ b/tests/integration/shell/test_minion.py @@ -17,9 +17,6 @@ import signal import shutil import logging -# Import 3rd-party libs -import yaml - # Import Salt Testing libs import tests.integration.utils from tests.support.case import ShellCase @@ -33,6 +30,7 @@ from salt.ext import six # Import salt libs import salt.utils.files +import salt.utils.yaml log = logging.getLogger(__name__) @@ -61,13 +59,11 @@ class MinionTest(ShellCase, testprogram.TestProgramCase, ShellCaseCommonTestsMix config_file_name = 'minion' pid_path = os.path.join(config_dir, '{0}.pid'.format(config_file_name)) with salt.utils.files.fopen(self.get_config_file_path(config_file_name), 'r') as fhr: - config = yaml.load(fhr.read()) + config = salt.utils.yaml.safe_load(fhr) config['log_file'] = 'file:///tmp/log/LOG_LOCAL3' with salt.utils.files.fopen(os.path.join(config_dir, config_file_name), 'w') as fhw: - fhw.write( - yaml.dump(config, default_flow_style=False) - ) + salt.utils.yaml.safe_dump(config, fhw, default_flow_style=False) ret = self.run_script( self._call_binary_, diff --git a/tests/integration/shell/test_runner.py b/tests/integration/shell/test_runner.py index 3cdaa7cffb5..410a614c421 100644 --- a/tests/integration/shell/test_runner.py +++ b/tests/integration/shell/test_runner.py @@ -16,12 +16,10 @@ from tests.support.paths import TMP from tests.support.mixins import ShellCaseCommonTestsMixin from tests.support.helpers import skip_if_not_root -# Import 3rd-party libs -import yaml - # Import Salt libs import salt.utils.files import salt.utils.platform +import salt.utils.yaml USERA = 'saltdev' USERA_PWD = 'saltdev' @@ -102,12 +100,10 @@ class RunTest(ShellCase, testprogram.TestProgramCase, ShellCaseCommonTestsMixin) config_file_name = 'master' with salt.utils.files.fopen(self.get_config_file_path(config_file_name), 'r') as fhr: - config = yaml.load(fhr.read()) + config = salt.utils.yaml.safe_load(fhr) config['log_file'] = 'file:///dev/log/LOG_LOCAL3' with salt.utils.files.fopen(os.path.join(config_dir, config_file_name), 'w') as fhw: - fhw.write( - yaml.dump(config, default_flow_style=False) - ) + salt.utils.yaml.safe_dump(config, fhw, default_flow_style=False) ret = self.run_script( self._call_binary_, '--config-dir {0} -d'.format( diff --git a/tests/integration/shell/test_syndic.py b/tests/integration/shell/test_syndic.py index 239b848b73a..87eb055ebe0 100644 --- a/tests/integration/shell/test_syndic.py +++ b/tests/integration/shell/test_syndic.py @@ -14,9 +14,6 @@ import signal import shutil import logging -# Import 3rd-party libs -import yaml - # Import Salt Testing libs from tests.support.case import ShellCase from tests.support.paths import TMP @@ -25,7 +22,7 @@ from tests.integration.utils import testprogram # Import salt libs import salt.utils.files - +import salt.utils.yaml log = logging.getLogger(__name__) @@ -48,7 +45,7 @@ class SyndicTest(ShellCase, testprogram.TestProgramCase, ShellCaseCommonTestsMix for fname in ('master', 'minion'): pid_path = os.path.join(config_dir, '{0}.pid'.format(fname)) with salt.utils.files.fopen(self.get_config_file_path(fname), 'r') as fhr: - config = yaml.load(fhr.read()) + config = salt.utils.yaml.safe_load(fhr) config['log_file'] = config['syndic_log_file'] = 'file:///tmp/log/LOG_LOCAL3' config['root_dir'] = config_dir if 'ret_port' in config: @@ -56,9 +53,7 @@ class SyndicTest(ShellCase, testprogram.TestProgramCase, ShellCaseCommonTestsMix config['publish_port'] = int(config['publish_port']) + 10 with salt.utils.files.fopen(os.path.join(config_dir, fname), 'w') as fhw: - fhw.write( - yaml.dump(config, default_flow_style=False) - ) + salt.utils.yaml.safe_dump(config, fhw, default_flow_style=False) ret = self.run_script( self._call_binary_, diff --git a/tests/integration/utils/testprogram.py b/tests/integration/utils/testprogram.py index 80d2b9128d6..bcf6c2c6e39 100644 --- a/tests/integration/utils/testprogram.py +++ b/tests/integration/utils/testprogram.py @@ -19,11 +19,10 @@ import sys import tempfile import time -import yaml - import salt.utils.files import salt.utils.process import salt.utils.psutil_compat as psutils +import salt.utils.yaml import salt.defaults.exitcodes as exitcodes from salt.ext import six from salt.ext.six.moves import range # pylint: disable=import-error,redefined-builtin @@ -600,7 +599,7 @@ class TestSaltProgram(six.with_metaclass(TestSaltProgramMeta, TestProgram)): @staticmethod def config_caster(cfg): - return yaml.safe_load(cfg) + return salt.utils.yaml.safe_load(cfg) def __init__(self, *args, **kwargs): if len(args) < 2 and 'program' not in kwargs: @@ -649,8 +648,7 @@ class TestSaltProgram(six.with_metaclass(TestSaltProgramMeta, TestProgram)): cfg[key] = val.format(**subs) else: cfg[key] = val - scfg = yaml.safe_dump(cfg, default_flow_style=False) - return scfg + return salt.utils.yaml.safe_dump(cfg, default_flow_style=False) def setup(self, *args, **kwargs): super(TestSaltProgram, self).setup(*args, **kwargs) diff --git a/tests/jenkins.py b/tests/jenkins.py index 0bbb0f89b7f..9ca0cf840ce 100644 --- a/tests/jenkins.py +++ b/tests/jenkins.py @@ -23,6 +23,7 @@ import random import salt.utils.files import salt.utils.json import salt.utils.stringutils +import salt.utils.yaml try: from salt.utils.nb_popen import NonBlockingPopen except ImportError: @@ -44,7 +45,6 @@ except ImportError: from nb_popen import NonBlockingPopen # Import 3rd-party libs -import yaml try: import requests HAS_REQUESTS = True @@ -77,7 +77,7 @@ def build_pillar_data(options): pillar['package_artifact_dir'] = options.package_artifact_dir if options.pillar: pillar.update(dict(options.pillar)) - return yaml.dump(pillar, default_flow_style=True, indent=0, width=sys.maxint).rstrip() + return salt.utils.yaml.safe_dump(pillar, default_flow_style=True, indent=0, width=sys.maxint).rstrip() def build_minion_target(options, vm_name): diff --git a/tests/minionswarm.py b/tests/minionswarm.py index aacdb6d21ce..2b09cd6e964 100644 --- a/tests/minionswarm.py +++ b/tests/minionswarm.py @@ -24,9 +24,9 @@ import uuid # Import salt libs import salt import salt.utils.files +import salt.utils.yaml # Import third party libs -import yaml from salt.ext import six from salt.ext.six.moves import range # pylint: disable=import-error,redefined-builtin @@ -301,7 +301,7 @@ class MinionSwarm(Swarm): if self.opts['config_dir']: spath = os.path.join(self.opts['config_dir'], 'minion') with salt.utils.files.fopen(spath) as conf: - data = yaml.load(conf) or {} + data = salt.utils.yaml.safe_load(conf) or {} minion_id = '{0}-{1}'.format( self.opts['name'], str(idx).zfill(self.zfill) @@ -360,7 +360,7 @@ class MinionSwarm(Swarm): data['grains']['uuid'] = str(uuid.uuid4()) with salt.utils.files.fopen(path, 'w+') as fp_: - yaml.dump(data, fp_) + salt.utils.yaml.safe_dump(data, fp_) self.confs.add(dpath) def prep_configs(self): @@ -414,7 +414,7 @@ class MasterSwarm(Swarm): if self.opts['config_dir']: spath = os.path.join(self.opts['config_dir'], 'master') with salt.utils.files.fopen(spath) as conf: - data = yaml.load(conf) + data = salt.utils.yaml.safe_load(conf) data.update({ 'log_file': os.path.join(self.conf, 'master.log'), 'open_mode': True # TODO Pre-seed keys @@ -424,7 +424,7 @@ class MasterSwarm(Swarm): path = os.path.join(self.conf, 'master') with salt.utils.files.fopen(path, 'w+') as fp_: - yaml.dump(data, fp_) + salt.utils.yaml.safe_dump(data, fp_) def shutdown(self): print('Killing master') diff --git a/tests/modparser.py b/tests/modparser.py index cc8b2d7cac0..dbdc6d7ce3d 100644 --- a/tests/modparser.py +++ b/tests/modparser.py @@ -12,8 +12,8 @@ import sys import os import modulefinder import pprint -import yaml import salt.utils.json +import salt.utils.yaml def parse(): @@ -97,7 +97,7 @@ if __name__ == '__main__': try: scand = scan(opts) if opts['format'] == 'yaml': - print(yaml.dump(scand)) + print(salt.utils.yaml.safe_dump(scand)) if opts['format'] == 'json': print(salt.utils.json.dumps(scand)) else: diff --git a/tests/saltsh.py b/tests/saltsh.py index 6b70f87fcb6..d3404f1fc47 100644 --- a/tests/saltsh.py +++ b/tests/saltsh.py @@ -42,14 +42,11 @@ import salt.runner import jinja2 from salt.ext.six.moves import builtins # pylint: disable=import-error - -# pylint: disable=W0611 +# pylint: disable=unused-import # These are imported to be available in the spawned shell - - -import yaml +import salt.utils.yaml import pprint - +# pylint: enable=unused-import HISTFILE = '{HOME}/.saltsh_history'.format(**os.environ) diff --git a/tests/support/case.py b/tests/support/case.py index aee748c9a5b..e9e07a76d70 100644 --- a/tests/support/case.py +++ b/tests/support/case.py @@ -655,13 +655,13 @@ class SPMCase(TestCase, AdaptedConfigurationTestCaseMixin): }) import salt.utils.files - import yaml + import salt.utils.yaml if not os.path.isdir(config['formula_path']): os.makedirs(config['formula_path']) with salt.utils.files.fopen(os.path.join(self._tmp_spm, 'spm'), 'w') as fp: - fp.write(yaml.dump(config)) + salt.utils.yaml.safe_dump(config, fp) return config diff --git a/tests/support/copyartifacts.py b/tests/support/copyartifacts.py index c9eae272338..b4ad7057979 100644 --- a/tests/support/copyartifacts.py +++ b/tests/support/copyartifacts.py @@ -7,7 +7,8 @@ import argparse # pylint: disable=minimum-python-version import os import paramiko import subprocess -import yaml + +import salt.utils.yaml class DownloadArtifacts(object): @@ -19,7 +20,7 @@ class DownloadArtifacts(object): def setup_transport(self): # pylint: disable=minimum-python-version - config = yaml.load(subprocess.check_output(['bundle', 'exec', 'kitchen', 'diagnose', self.instance])) + config = salt.utils.yaml.safe_load(subprocess.check_output(['bundle', 'exec', 'kitchen', 'diagnose', self.instance])) # pylint: enable=minimum-python-version state = config['instances'][self.instance]['state_file'] tport = config['instances'][self.instance]['transport'] diff --git a/tests/support/gitfs.py b/tests/support/gitfs.py index 72871476014..3046082244f 100644 --- a/tests/support/gitfs.py +++ b/tests/support/gitfs.py @@ -15,11 +15,11 @@ import signal import tempfile import textwrap import time -import yaml # Import Salt libs import salt.utils.files import salt.utils.path +import salt.utils.yaml from salt.fileserver import gitfs from salt.pillar import git_pillar from salt.ext.six.moves import range # pylint: disable=redefined-builtin @@ -331,7 +331,7 @@ class GitPillarTestBase(GitTestBase, LoaderModuleMockMixin): ''' cachedir = tempfile.mkdtemp(dir=TMP) self.addCleanup(shutil.rmtree, cachedir, ignore_errors=True) - ext_pillar_opts = yaml.safe_load( + ext_pillar_opts = salt.utils.yaml.safe_load( ext_pillar_conf.format( cachedir=cachedir, extmods=os.path.join(cachedir, 'extmods'), diff --git a/tests/support/mixins.py b/tests/support/mixins.py index 22d490ea74d..0804818b06b 100644 --- a/tests/support/mixins.py +++ b/tests/support/mixins.py @@ -36,6 +36,7 @@ import salt.utils.files import salt.utils.functools import salt.utils.path import salt.utils.stringutils +import salt.utils.yaml import salt.version import salt.exceptions from salt.utils.verify import verify_env @@ -43,7 +44,6 @@ from salt.utils.immutabletypes import freeze from salt._compat import ElementTree as etree # Import 3rd-party libs -import yaml from salt.ext import six from salt.ext.six.moves import zip # pylint: disable=import-error,redefined-builtin @@ -118,7 +118,7 @@ class AdaptedConfigurationTestCaseMixin(object): rdict['config_dir'] = conf_dir rdict['conf_file'] = os.path.join(conf_dir, config_for) with salt.utils.files.fopen(rdict['conf_file'], 'w') as wfh: - wfh.write(yaml.dump(rdict, default_flow_style=False)) + salt.utils.yaml.safe_dump(rdict, wfh, default_flow_style=False) return rdict @staticmethod diff --git a/tests/unit/config/test_config.py b/tests/unit/config/test_config.py index 72c7eca5bd5..c01a5c7a2e8 100644 --- a/tests/unit/config/test_config.py +++ b/tests/unit/config/test_config.py @@ -25,6 +25,7 @@ import salt.minion import salt.utils.files import salt.utils.network import salt.utils.platform +import salt.utils.yaml from salt.syspaths import CONFIG_DIR from salt import config as sconfig from salt.exceptions import ( @@ -33,9 +34,6 @@ from salt.exceptions import ( SaltCloudConfigError ) -# Import Third-Party Libs -import yaml - log = logging.getLogger(__name__) # mock hostname should be more complex than the systems FQDN @@ -1133,7 +1131,7 @@ class ConfigTestCase(TestCase, AdaptedConfigurationTestCaseMixin): default_config = sconfig.cloud_config(config_file_path) default_config['deploy_scripts_search_path'] = deploy_dir_path with salt.utils.files.fopen(config_file_path, 'w') as cfd: - cfd.write(yaml.dump(default_config, default_flow_style=False)) + salt.utils.yaml.safe_dump(default_config, cfd, default_flow_style=False) default_config = sconfig.cloud_config(config_file_path) diff --git a/tests/unit/fileserver/test_gitfs.py b/tests/unit/fileserver/test_gitfs.py index bda0182eecb..bc0e0aca6df 100644 --- a/tests/unit/fileserver/test_gitfs.py +++ b/tests/unit/fileserver/test_gitfs.py @@ -19,7 +19,6 @@ except ImportError: pass # Import 3rd-party libs -import yaml try: import git # pylint: disable=unused-import HAS_GITPYTHON = True @@ -39,6 +38,7 @@ import salt.fileserver.gitfs as gitfs import salt.utils.gitfs import salt.utils.platform import salt.utils.win_functions +import salt.utils.yaml log = logging.getLogger(__name__) @@ -126,7 +126,7 @@ class GitfsConfigTestCase(TestCase, LoaderModuleMockMixin): - baz: - mountpoint: abc ''') - with patch.dict(gitfs.__opts__, yaml.safe_load(opts_override)): + with patch.dict(gitfs.__opts__, salt.utils.yaml.safe_load(opts_override)): git_fs = salt.utils.gitfs.GitFS( gitfs.__opts__, gitfs.__opts__['gitfs_remotes'], diff --git a/tests/unit/loader/test_globals.py b/tests/unit/loader/test_globals.py index a61dafe7b05..118e1741c19 100644 --- a/tests/unit/loader/test_globals.py +++ b/tests/unit/loader/test_globals.py @@ -5,17 +5,16 @@ Test Salt's loader regarding globals that it should pack in ''' - # Import Python libs from __future__ import absolute_import, print_function, unicode_literals +import inspect # Import Salt Testing libs from tests.support.case import ModuleCase # Import salt libs import salt.loader -import inspect -import yaml +import salt.utils.yaml # Import 3rd-party libs from salt.ext import six @@ -46,7 +45,7 @@ class LoaderGlobalsTest(ModuleCase): # get the names of the globals you should have func_name = inspect.stack()[1][3] - names = next(six.itervalues(yaml.load(getattr(self, func_name).__doc__))) + names = next(six.itervalues(salt.utils.yaml.safe_load(getattr(self, func_name).__doc__))) # Now, test each module! for item in global_vars: diff --git a/tests/unit/modules/test_pkg_resource.py b/tests/unit/modules/test_pkg_resource.py index 3ac5a6f7d8c..16420272708 100644 --- a/tests/unit/modules/test_pkg_resource.py +++ b/tests/unit/modules/test_pkg_resource.py @@ -19,6 +19,7 @@ from tests.support.mock import ( # Import Salt Libs import salt.utils.data +import salt.utils.yaml import salt.modules.pkg_resource as pkg_resource from salt.ext import six @@ -36,7 +37,7 @@ class PkgresTestCase(TestCase, LoaderModuleMockMixin): Test to accepts list of dicts (or a string representing a list of dicts) and packs the key/value pairs into a single dict. ''' - with patch.object(yaml, + with patch.object(salt.utils.yaml, 'safe_load', MagicMock(side_effect=yaml.parser.ParserError('f'))): with patch.dict(pkg_resource.__salt__, diff --git a/tests/unit/modules/test_virt.py b/tests/unit/modules/test_virt.py index 029bff098e1..2d0cd6a46d8 100644 --- a/tests/unit/modules/test_virt.py +++ b/tests/unit/modules/test_virt.py @@ -11,13 +11,13 @@ from tests.support.unit import skipIf, TestCase from tests.support.mock import NO_MOCK, NO_MOCK_REASON, MagicMock, patch # Import salt libs +import salt.utils.yaml import salt.modules.virt as virt import salt.modules.config as config from salt._compat import ElementTree as ET import salt.config # Import third party libs -import yaml from salt.ext import six @@ -329,8 +329,8 @@ class VirtTestCase(TestCase, LoaderModuleMockMixin): ''' with patch('salt.modules.virt._nic_profile') as nic_profile, \ patch('salt.modules.virt._disk_profile') as disk_profile: - disk_profile.return_value = yaml.load(diskp_yaml) - nic_profile.return_value = yaml.load(nicp_yaml) + disk_profile.return_value = salt.utils.yaml.safe_load(diskp_yaml) + nic_profile.return_value = salt.utils.yaml.safe_load(nicp_yaml) diskp = virt._disk_profile('noeffect', 'esxi') nicp = virt._nic_profile('noeffect', 'esxi') xml_data = virt._gen_xml( @@ -376,8 +376,8 @@ class VirtTestCase(TestCase, LoaderModuleMockMixin): ''' with patch('salt.modules.virt._nic_profile') as nic_profile, \ patch('salt.modules.virt._disk_profile') as disk_profile: - disk_profile.return_value = yaml.load(diskp_yaml) - nic_profile.return_value = yaml.load(nicp_yaml) + disk_profile.return_value = salt.utils.yaml.safe_load(diskp_yaml) + nic_profile.return_value = salt.utils.yaml.safe_load(nicp_yaml) diskp = virt._disk_profile('noeffect', 'kvm') nicp = virt._nic_profile('noeffect', 'kvm') xml_data = virt._gen_xml( @@ -455,7 +455,7 @@ class VirtTestCase(TestCase, LoaderModuleMockMixin): bridge: br1 model: virtio ''' - mock_config = yaml.load(yaml_config) + mock_config = salt.utils.yaml.safe_load(yaml_config) with patch.dict(salt.modules.config.__opts__, mock_config): for name in six.iterkeys(mock_config['virt.nic']): diff --git a/tests/unit/netapi/rest_cherrypy/test_tools.py b/tests/unit/netapi/rest_cherrypy/test_tools.py index 1d1b9291ae8..4fbded4cb9e 100644 --- a/tests/unit/netapi/rest_cherrypy/test_tools.py +++ b/tests/unit/netapi/rest_cherrypy/test_tools.py @@ -2,15 +2,17 @@ # Import Python libs from __future__ import absolute_import -import yaml # Import Salt libs import salt.utils.json -from tests.support.cherrypy_testclasses import BaseToolsTest +import salt.utils.yaml # Import 3rd-party libs from salt.ext.six.moves.urllib.parse import urlencode # pylint: disable=no-name-in-module,import-error +# Import Salt libs +from tests.support.cherrypy_testclasses import BaseToolsTest + class TestOutFormats(BaseToolsTest): def __get_cp_config__(self): @@ -80,7 +82,7 @@ class TestInFormats(BaseToolsTest): def test_yaml_ctype(self): data = {'valid': 'stuff'} request, response = self.request('/', method='POST', - body=yaml.dump(data), headers=( + body=salt.utils.yaml.safe_dump(data), headers=( ('Content-type', 'application/x-yaml'), )) self.assertEqual(response.status, '200 OK') diff --git a/tests/unit/netapi/rest_tornado/test_handlers.py b/tests/unit/netapi/rest_tornado/test_handlers.py index 3a5329f7275..1e9040059a3 100644 --- a/tests/unit/netapi/rest_tornado/test_handlers.py +++ b/tests/unit/netapi/rest_tornado/test_handlers.py @@ -13,6 +13,7 @@ from tests.support.unit import TestCase, skipIf # Import Salt libs import salt.auth import salt.utils.json +import salt.utils.yaml from salt.ext.six.moves import map # pylint: disable=import-error try: import salt.netapi.rest_tornado as rest_tornado @@ -22,7 +23,6 @@ except ImportError: HAS_TORNADO = False # Import 3rd-party libs -import yaml # pylint: disable=import-error try: import tornado.escape @@ -187,7 +187,7 @@ class TestBaseSaltAPIHandler(SaltnadoTestCase): # Request application/x-yaml response = self.fetch('/', headers={'Accept': self.content_type_map['yaml']}) self.assertEqual(response.headers['Content-Type'], self.content_type_map['yaml']) - self.assertEqual(type(yaml.load(response.body)), dict) + self.assertEqual(type(salt.utils.yaml.safe_load(response.body)), dict) # Request not supported content-type response = self.fetch('/', headers={'Accept': self.content_type_map['xml']}) @@ -203,7 +203,7 @@ class TestBaseSaltAPIHandler(SaltnadoTestCase): accept_header = self.content_type_map['real-accept-header-yaml'] response = self.fetch('/', headers={'Accept': accept_header}) self.assertEqual(response.headers['Content-Type'], self.content_type_map['yaml']) - self.assertEqual(type(yaml.load(response.body)), dict) + self.assertEqual(type(salt.utils.yaml.safe_load(response.body)), dict) def test_token(self): ''' @@ -256,14 +256,14 @@ class TestBaseSaltAPIHandler(SaltnadoTestCase): # send yaml as json (should break) response = self.fetch('/', method='POST', - body=yaml.dump(valid_lowstate), + body=salt.utils.yaml.safe_dump(valid_lowstate), headers={'Content-Type': self.content_type_map['json']}) self.assertEqual(response.code, 400) # send as yaml response = self.fetch('/', method='POST', - body=yaml.dump(valid_lowstate), + body=salt.utils.yaml.safe_dump(valid_lowstate), headers={'Content-Type': self.content_type_map['yaml']}) self.assertEqual(valid_lowstate, salt.utils.json.loads(response.body)['lowstate']) @@ -369,7 +369,7 @@ class TestBaseSaltAPIHandler(SaltnadoTestCase): # send as yaml response = self.fetch('/', method='POST', - body=yaml.dump(request_lowstate), + body=salt.utils.yaml.safe_dump(request_lowstate), headers={'Content-Type': self.content_type_map['yaml']}) self.assertEqual(valid_lowstate, salt.utils.json.loads(response.body)['lowstate']) @@ -552,7 +552,7 @@ class TestSaltAuthHandler(SaltnadoTestCase): # Test in YAML response = self.fetch('/login', method='POST', - body=yaml.dump(self.auth_creds_dict), + body=salt.utils.yaml.safe_dump(self.auth_creds_dict), headers={'Content-Type': self.content_type_map['yaml']}) self.assertEqual(response.code, 200) diff --git a/tests/unit/pillar/test_hg.py b/tests/unit/pillar/test_hg.py index 66da2465fc8..9f2172c3806 100644 --- a/tests/unit/pillar/test_hg.py +++ b/tests/unit/pillar/test_hg.py @@ -8,7 +8,6 @@ import os import tempfile import shutil import subprocess -import yaml # Import Salt Testing libs from tests.integration import AdaptedConfigurationTestCaseMixin @@ -28,6 +27,7 @@ FILE_DATA = { # Import Salt Libs import salt.utils.files +import salt.utils.yaml import salt.pillar.hg_pillar as hg_pillar HGLIB = hg_pillar.hglib @@ -67,7 +67,7 @@ class HgPillarTestCase(TestCase, AdaptedConfigurationTestCaseMixin, LoaderModule subprocess.check_call(['hg', 'init', hg_repo]) for filename in FILE_DATA: with salt.utils.files.fopen(os.path.join(hg_repo, filename), 'w') as data_file: - yaml.dump(FILE_DATA[filename], data_file) + salt.utils.yaml.safe_dump(FILE_DATA[filename], data_file) subprocess.check_call(['hg', 'ci', '-A', '-R', hg_repo, '-m', 'first commit', '-u', COMMIT_USER_NAME]) return hg_repo diff --git a/tests/unit/ssh/test_roster_defaults.py b/tests/unit/ssh/test_roster_defaults.py index 3f0d991040e..ce8ad85ab50 100644 --- a/tests/unit/ssh/test_roster_defaults.py +++ b/tests/unit/ssh/test_roster_defaults.py @@ -9,7 +9,6 @@ from __future__ import absolute_import import os import shutil import tempfile -import yaml # Import Salt Testing libs from tests.support.unit import TestCase @@ -20,6 +19,7 @@ from tests.support.paths import TMP import salt.roster import salt.config import salt.utils.files +import salt.utils.yaml ROSTER = ''' localhost: @@ -62,7 +62,7 @@ class SSHRosterDefaults(TestCase): ) opts = salt.config.master_config(fpath) with patch('salt.roster.get_roster_file', MagicMock(return_value=ROSTER)): - with patch('salt.template.compile_template', MagicMock(return_value=yaml.load(ROSTER))): + with patch('salt.template.compile_template', MagicMock(return_value=salt.utils.yaml.safe_load(ROSTER))): roster = salt.roster.Roster(opts=opts) self.assertEqual(roster.targets('*', 'glob'), expected) finally: diff --git a/tests/unit/states/test_boto_apigateway.py b/tests/unit/states/test_boto_apigateway.py index 7614ed5556c..fad38f88846 100644 --- a/tests/unit/states/test_boto_apigateway.py +++ b/tests/unit/states/test_boto_apigateway.py @@ -17,13 +17,13 @@ from tests.support.mock import NO_MOCK, NO_MOCK_REASON, MagicMock, patch import salt.config import salt.loader import salt.utils.files +import salt.utils.yaml from salt.utils.versions import LooseVersion # pylint: disable=import-error,no-name-in-module from tests.unit.modules.test_boto_apigateway import BotoApiGatewayTestCaseMixin # Import 3rd-party libs -import yaml from salt.ext.six.moves import range try: import boto3 @@ -57,99 +57,99 @@ error_content = { } } -api_ret = dict(description=u'{\n "context": "See deployment or stage description",\n "provisioned_by": "Salt boto_apigateway.present State"\n}', +api_ret = dict(description='{\n "context": "See deployment or stage description",\n "provisioned_by": "Salt boto_apigateway.present State"\n}', createdDate=datetime.datetime(2015, 11, 17, 16, 33, 50), - id=u'vni0vq8wzi', - name=u'unit test api') + id='vni0vq8wzi', + name='unit test api') -no_apis_ret = {u'items': []} +no_apis_ret = {'items': []} -apis_ret = {u'items': [api_ret]} +apis_ret = {'items': [api_ret]} -mock_model_ret = dict(contentType=u'application/json', - description=u'mock model', - id=u'123abc', - name=u'mock model', - schema=(u'{\n' - u' "$schema": "http://json-schema.org/draft-04/schema#",\n' - u' "properties": {\n' - u' "field": {\n' - u' "type": "string"\n' - u' }\n' - u' }\n' - u'}')) +mock_model_ret = dict(contentType='application/json', + description='mock model', + id='123abc', + name='mock model', + schema=('{\n' + ' "$schema": "http://json-schema.org/draft-04/schema#",\n' + ' "properties": {\n' + ' "field": {\n' + ' "type": "string"\n' + ' }\n' + ' }\n' + '}')) -models_ret = {u'items': [dict(contentType=u'application/json', - description=u'Error', - id=u'50nw8r', - name=u'Error', - schema=(u'{\n' - u' "$schema": "http://json-schema.org/draft-04/schema#",\n' - u' "properties": {\n' - u' "code": {\n' - u' "format": "int32",\n' - u' "type": "integer"\n' - u' },\n' - u' "fields": {\n' - u' "type": "string"\n' - u' },\n' - u' "message": {\n' - u' "type": "string"\n' - u' }\n' - u' },\n' - u' "title": "Error Schema",\n' - u' "type": "object"\n' - u'}')), - dict(contentType=u'application/json', - description=u'User', - id=u'terlnw', - name=u'User', - schema=(u'{\n' - u' "$schema": "http://json-schema.org/draft-04/schema#",\n' - u' "properties": {\n' - u' "password": {\n' - u' "description": "A password for the new user",\n' - u' "type": "string"\n' - u' },\n' - u' "username": {\n' - u' "description": "A unique username for the user",\n' - u' "type": "string"\n' - u' }\n' - u' },\n' - u' "title": "User Schema",\n' - u' "type": "object"\n' - u'}'))]} +models_ret = {'items': [dict(contentType='application/json', + description='Error', + id='50nw8r', + name='Error', + schema=('{\n' + ' "$schema": "http://json-schema.org/draft-04/schema#",\n' + ' "properties": {\n' + ' "code": {\n' + ' "format": "int32",\n' + ' "type": "integer"\n' + ' },\n' + ' "fields": {\n' + ' "type": "string"\n' + ' },\n' + ' "message": {\n' + ' "type": "string"\n' + ' }\n' + ' },\n' + ' "title": "Error Schema",\n' + ' "type": "object"\n' + '}')), + dict(contentType='application/json', + description='User', + id='terlnw', + name='User', + schema=('{\n' + ' "$schema": "http://json-schema.org/draft-04/schema#",\n' + ' "properties": {\n' + ' "password": {\n' + ' "description": "A password for the new user",\n' + ' "type": "string"\n' + ' },\n' + ' "username": {\n' + ' "description": "A unique username for the user",\n' + ' "type": "string"\n' + ' }\n' + ' },\n' + ' "title": "User Schema",\n' + ' "type": "object"\n' + '}'))]} -root_resources_ret = {u'items': [dict(id=u'bgk0rk8rqb', - path=u'/')]} +root_resources_ret = {'items': [dict(id='bgk0rk8rqb', + path='/')]} -resources_ret = {u'items': [dict(id=u'bgk0rk8rqb', - path=u'/'), - dict(id=u'9waiaz', - parentId=u'bgk0rk8rqb', - path=u'/users', - pathPart=u'users', - resourceMethods={u'POST': {}})]} +resources_ret = {'items': [dict(id='bgk0rk8rqb', + path='/'), + dict(id='9waiaz', + parentId='bgk0rk8rqb', + path='/users', + pathPart='users', + resourceMethods={'POST': {}})]} -no_resources_ret = {u'items': []} +no_resources_ret = {'items': []} stage1_deployment1_ret = dict(cacheClusterEnabled=False, cacheClusterSize=0.5, cacheClusterStatus='NOT_AVAILABLE', createdDate=datetime.datetime(2015, 11, 17, 16, 33, 50), - deploymentId=u'kobnrb', - description=(u'{\n' - u' "current_deployment_label": {\n' - u' "api_name": "unit test api",\n' - u' "swagger_file": "temp-swagger-sample.yaml",\n' - u' "swagger_file_md5sum": "4fb17e43bab3a96e7f2410a1597cd0a5",\n' - u' "swagger_info_object": {\n' - u' "description": "salt boto apigateway unit test service",\n' - u' "title": "salt boto apigateway unit test service",\n' - u' "version": "0.0.0"\n' - u' }\n' - u' }\n' - u'}'), + deploymentId='kobnrb', + description=('{\n' + ' "current_deployment_label": {\n' + ' "api_name": "unit test api",\n' + ' "swagger_file": "temp-swagger-sample.yaml",\n' + ' "swagger_file_md5sum": "4fb17e43bab3a96e7f2410a1597cd0a5",\n' + ' "swagger_info_object": {\n' + ' "description": "salt boto apigateway unit test service",\n' + ' "title": "salt boto apigateway unit test service",\n' + ' "version": "0.0.0"\n' + ' }\n' + ' }\n' + '}'), lastUpdatedDate=datetime.datetime(2015, 11, 17, 16, 33, 50), methodSettings=dict(), stageName='test', @@ -159,19 +159,19 @@ stage1_deployment1_vars_ret = dict(cacheClusterEnabled=False, cacheClusterSize=0.5, cacheClusterStatus='NOT_AVAILABLE', createdDate=datetime.datetime(2015, 11, 17, 16, 33, 50), - deploymentId=u'kobnrb', - description=(u'{\n' - u' "current_deployment_label": {\n' - u' "api_name": "unit test api",\n' - u' "swagger_file": "temp-swagger-sample.yaml",\n' - u' "swagger_file_md5sum": "4fb17e43bab3a96e7f2410a1597cd0a5",\n' - u' "swagger_info_object": {\n' - u' "description": "salt boto apigateway unit test service",\n' - u' "title": "salt boto apigateway unit test service",\n' - u' "version": "0.0.0"\n' - u' }\n' - u' }\n' - u'}'), + deploymentId='kobnrb', + description=('{\n' + ' "current_deployment_label": {\n' + ' "api_name": "unit test api",\n' + ' "swagger_file": "temp-swagger-sample.yaml",\n' + ' "swagger_file_md5sum": "4fb17e43bab3a96e7f2410a1597cd0a5",\n' + ' "swagger_info_object": {\n' + ' "description": "salt boto apigateway unit test service",\n' + ' "title": "salt boto apigateway unit test service",\n' + ' "version": "0.0.0"\n' + ' }\n' + ' }\n' + '}'), lastUpdatedDate=datetime.datetime(2015, 11, 17, 16, 33, 50), methodSettings=dict(), stageName='test', @@ -181,19 +181,19 @@ stage1_deployment2_ret = dict(cacheClusterEnabled=False, cacheClusterSize=0.5, cacheClusterStatus='NOT_AVAILABLE', createdDate=datetime.datetime(2015, 11, 17, 16, 33, 50), - deploymentId=u'kobnrc', - description=(u'{\n' - u' "current_deployment_label": {\n' - u' "api_name": "unit test api",\n' - u' "swagger_file": "temp-swagger-sample.yaml",\n' - u' "swagger_file_md5sum": "5fd538c4336ed5c54b4bf39ddf97c661",\n' - u' "swagger_info_object": {\n' - u' "description": "salt boto apigateway unit test service",\n' - u' "title": "salt boto apigateway unit test service",\n' - u' "version": "0.0.2"\n' - u' }\n' - u' }\n' - u'}'), + deploymentId='kobnrc', + description=('{\n' + ' "current_deployment_label": {\n' + ' "api_name": "unit test api",\n' + ' "swagger_file": "temp-swagger-sample.yaml",\n' + ' "swagger_file_md5sum": "5fd538c4336ed5c54b4bf39ddf97c661",\n' + ' "swagger_info_object": {\n' + ' "description": "salt boto apigateway unit test service",\n' + ' "title": "salt boto apigateway unit test service",\n' + ' "version": "0.0.2"\n' + ' }\n' + ' }\n' + '}'), lastUpdatedDate=datetime.datetime(2015, 11, 17, 16, 33, 50), methodSettings=dict(), stageName='test', @@ -203,55 +203,55 @@ stage2_ret = dict(cacheClusterEnabled=False, cacheClusterSize=0.5, cacheClusterStatus='NOT_AVAILABLE', createdDate=datetime.datetime(2015, 11, 17, 16, 33, 50), - deploymentId=u'kobnrb', - description=(u'{\n' - u' "current_deployment_label": {\n' - u' "api_name": "unit test api",\n' - u' "swagger_file": "temp-swagger-sample.yaml",\n' - u' "swagger_file_md5sum": "4fb17e43bab3a96e7f2410a1597cd0a5",\n' - u' "swagger_info_object": {\n' - u' "description": "salt boto apigateway unit test service",\n' - u' "title": "salt boto apigateway unit test service",\n' - u' "version": "0.0.0"\n' - u' }\n' - u' }\n' - u'}'), + deploymentId='kobnrb', + description=('{\n' + ' "current_deployment_label": {\n' + ' "api_name": "unit test api",\n' + ' "swagger_file": "temp-swagger-sample.yaml",\n' + ' "swagger_file_md5sum": "4fb17e43bab3a96e7f2410a1597cd0a5",\n' + ' "swagger_info_object": {\n' + ' "description": "salt boto apigateway unit test service",\n' + ' "title": "salt boto apigateway unit test service",\n' + ' "version": "0.0.0"\n' + ' }\n' + ' }\n' + '}'), lastUpdatedDate=datetime.datetime(2015, 11, 17, 16, 33, 50), methodSettings=dict(), stageName='dev', variables=dict()) -stages_stage2_ret = {u'item': [stage2_ret]} +stages_stage2_ret = {'item': [stage2_ret]} -no_stages_ret = {u'item': []} +no_stages_ret = {'item': []} deployment1_ret = dict(createdDate=datetime.datetime(2015, 11, 17, 16, 33, 50), - description=(u'{\n' - u' "api_name": "unit test api",\n' - u' "swagger_file": "temp-swagger-sample.yaml",\n' - u' "swagger_file_md5sum": "99cd44f491f25a8e8bb468b0a8e23a0a",\n' - u' "swagger_info_object": {\n' - u' "description": "salt boto apigateway unit test service",\n' - u' "title": "salt boto apigateway unit test service",\n' - u' "version": "0.0.0"\n' - u' }\n' - u'}'), - id=u'kobnrb') + description=('{\n' + ' "api_name": "unit test api",\n' + ' "swagger_file": "temp-swagger-sample.yaml",\n' + ' "swagger_file_md5sum": "693c57997a12a2446bb5c08c793d943c",\n' + ' "swagger_info_object": {\n' + ' "description": "salt boto apigateway unit test service",\n' + ' "title": "salt boto apigateway unit test service",\n' + ' "version": "0.0.0"\n' + ' }\n' + '}'), + id='kobnrb') deployment2_ret = dict(createdDate=datetime.datetime(2015, 11, 17, 16, 33, 50), - description=(u'{\n' - u' "api_name": "unit test api",\n' - u' "swagger_file": "temp-swagger-sample.yaml",\n' - u' "swagger_file_md5sum": "5fd538c4336ed5c54b4bf39ddf97c661",\n' - u' "swagger_info_object": {\n' - u' "description": "salt boto apigateway unit test service",\n' - u' "title": "salt boto apigateway unit test service",\n' - u' "version": "0.0.2"\n' - u' }\n' - u'}'), - id=u'kobnrc') + description=('{\n' + ' "api_name": "unit test api",\n' + ' "swagger_file": "temp-swagger-sample.yaml",\n' + ' "swagger_file_md5sum": "5fd538c4336ed5c54b4bf39ddf97c661",\n' + ' "swagger_info_object": {\n' + ' "description": "salt boto apigateway unit test service",\n' + ' "title": "salt boto apigateway unit test service",\n' + ' "version": "0.0.2"\n' + ' }\n' + '}'), + id='kobnrc') -deployments_ret = {u'items': [deployment1_ret, deployment2_ret]} +deployments_ret = {'items': [deployment1_ret, deployment2_ret]} function_ret = dict(FunctionName='unit_test_api_users_post', Runtime='python2.7', @@ -346,7 +346,7 @@ class TempSwaggerFile(object): 'name': 'NewUser', 'schema': {'$ref': '#/definitions/User'}}], 'produces': ['application/json'], - 'description': 'Creates a new user.\n', + 'description': 'Creates a new user.', 'tags': ['Auth'], 'consumes': ['application/json'], 'summary': 'Registers a new user'}}}, @@ -370,8 +370,8 @@ class TempSwaggerFile(object): def __enter__(self): self.swaggerfile = 'temp-swagger-sample.yaml' - with salt.utils.files.fopen(self.swaggerfile, 'w') as f: - f.write(yaml.dump(self.swaggerdict)) + with salt.utils.files.fopen(self.swaggerfile, 'w') as fp_: + salt.utils.yaml.safe_dump(self.swaggerdict, fp_) return self.swaggerfile def __exit__(self, objtype, value, traceback): diff --git a/tests/unit/states/test_boto_cloudfront.py b/tests/unit/states/test_boto_cloudfront.py index dddb78d384b..567867a6e21 100644 --- a/tests/unit/states/test_boto_cloudfront.py +++ b/tests/unit/states/test_boto_cloudfront.py @@ -29,6 +29,9 @@ class BotoCloudfrontTestCase(TestCase, LoaderModuleMockMixin): whitelist=['boto3', 'dictdiffer', 'yamldumper'], context={}, ) + # Force the LazyDict to populate its references. Otherwise the lookup + # will fail inside the unit tests. + list(utils) return { boto_cloudfront: { '__utils__': utils, diff --git a/tests/unit/states/test_boto_sqs.py b/tests/unit/states/test_boto_sqs.py index 80672e87f4b..0a8260a42e7 100644 --- a/tests/unit/states/test_boto_sqs.py +++ b/tests/unit/states/test_boto_sqs.py @@ -28,6 +28,9 @@ class BotoSqsTestCase(TestCase, LoaderModuleMockMixin): whitelist=['boto3', 'yamldumper'], context={} ) + # Force the LazyDict to populate its references. Otherwise the lookup + # will fail inside the unit tests. + list(utils) return { boto_sqs: { '__utils__': utils, diff --git a/tests/unit/states/test_file.py b/tests/unit/states/test_file.py index 2444d83c636..6e3b4196208 100644 --- a/tests/unit/states/test_file.py +++ b/tests/unit/states/test_file.py @@ -26,13 +26,11 @@ from tests.support.mock import ( mock_open, patch) -# Import third party libs -import yaml - # Import salt libs import salt.utils.files import salt.utils.json import salt.utils.platform +import salt.utils.yaml import salt.states.file as filestate import salt.serializers.yaml as yamlserializer import salt.serializers.json as jsonserializer @@ -75,10 +73,10 @@ class TestFileState(TestCase, LoaderModuleMockMixin): } filestate.serialize('/tmp', dataset) - self.assertEqual(yaml.load(returner.returned), dataset) + self.assertEqual(salt.utils.yaml.safe_load(returner.returned), dataset) filestate.serialize('/tmp', dataset, formatter="yaml") - self.assertEqual(yaml.load(returner.returned), dataset) + self.assertEqual(salt.utils.yaml.safe_load(returner.returned), dataset) filestate.serialize('/tmp', dataset, formatter="json") self.assertEqual(salt.utils.json.loads(returner.returned), dataset) diff --git a/tests/unit/states/test_grains.py b/tests/unit/states/test_grains.py index 836aed83268..c81368081c2 100644 --- a/tests/unit/states/test_grains.py +++ b/tests/unit/states/test_grains.py @@ -7,7 +7,6 @@ from __future__ import absolute_import # Import Python libs import os -import yaml import contextlib # Import Salt Testing libs @@ -18,6 +17,7 @@ from tests.support.mock import NO_MOCK, NO_MOCK_REASON, MagicMock, patch # Import salt libs import salt.utils.files +import salt.utils.yaml import salt.modules.grains as grainsmod import salt.states.grains as grains @@ -77,9 +77,8 @@ class GrainsTestCase(TestCase, LoaderModuleMockMixin): else: grains_file = os.path.join( os.path.dirname(grains.__opts__['conf_file']), 'grains') - cstr = yaml.safe_dump(grains_data, default_flow_style=False) with salt.utils.files.fopen(grains_file, "w+") as grf: - grf.write(cstr) + salt.utils.yaml.safe_dump(grains_data, grf, default_flow_style=False) yield # 'exists' function tests: 2 diff --git a/tests/unit/states/test_syslog_ng.py b/tests/unit/states/test_syslog_ng.py index a2984dfec57..7c4c06f94cc 100644 --- a/tests/unit/states/test_syslog_ng.py +++ b/tests/unit/states/test_syslog_ng.py @@ -8,13 +8,13 @@ from __future__ import absolute_import import os import re import tempfile -import yaml from tests.support.mixins import LoaderModuleMockMixin from tests.support.unit import skipIf, TestCase from tests.support.mock import NO_MOCK, NO_MOCK_REASON, MagicMock, patch import salt.utils.files +import salt.utils.yaml import salt.states.syslog_ng as syslog_ng import salt.modules.syslog_ng as syslog_ng_module @@ -331,7 +331,7 @@ class SyslogNGTestCase(TestCase, LoaderModuleMockMixin): self._config_generator_template(GIVEN_CONFIG, SHORT_FORM_EXPECTED) def _config_generator_template(self, yaml_input, expected): - parsed_yaml_config = yaml.load(yaml_input["config"]) + parsed_yaml_config = salt.utils.yaml.safe_load(yaml_input["config"]) id = yaml_input["id"] with patch.dict(syslog_ng.__salt__, _SALT_VAR_WITH_MODULE_METHODS): @@ -357,7 +357,7 @@ class SyslogNGTestCase(TestCase, LoaderModuleMockMixin): syslog_ng_module.write_config(config='@include "scl.conf"') for i in yaml_inputs: - parsed_yaml_config = yaml.load(i["config"]) + parsed_yaml_config = salt.utils.yaml.safe_load(i["config"]) id = i["id"] got = syslog_ng.config(id, config=parsed_yaml_config, write=True) diff --git a/tests/unit/templates/test_jinja.py b/tests/unit/templates/test_jinja.py index 1b6914802a7..550351dc411 100644 --- a/tests/unit/templates/test_jinja.py +++ b/tests/unit/templates/test_jinja.py @@ -10,7 +10,6 @@ import os import pprint import re import tempfile -import yaml # Import Salt Testing libs from tests.support.unit import skipIf, TestCase @@ -21,6 +20,7 @@ from tests.support.paths import TMP_CONF_DIR # Import Salt libs import salt.config import salt.loader +import salt.utils.yaml from salt.exceptions import SaltRenderError from salt.ext import six @@ -643,7 +643,7 @@ class TestCustomExtensions(TestCase): } env = Environment(extensions=[SerializerExtension]) rendered = env.from_string('{{ dataset|yaml }}').render(dataset=dataset) - self.assertEqual(dataset, yaml.load(rendered)) + self.assertEqual(dataset, salt.utils.yaml.safe_load(rendered)) def test_serialize_yaml_str(self): dataset = "str value" diff --git a/tests/unit/test_pydsl.py b/tests/unit/test_pydsl.py index f14123ffba2..e2f6ca59295 100644 --- a/tests/unit/test_pydsl.py +++ b/tests/unit/test_pydsl.py @@ -242,8 +242,8 @@ class PyDSLRendererTestCase(CommonTestCaseBoilerplate): def test_load_highstate(self): result = self.render_sls(textwrap.dedent(''' - import yaml - __pydsl__.load_highstate(yaml.load(""" + import salt.utils.yaml + __pydsl__.load_highstate(salt.utils.yaml.safe_load(""" A: cmd.run: - name: echo hello diff --git a/tests/unit/utils/test_reactor.py b/tests/unit/utils/test_reactor.py index 564c2f2b6eb..0b97438ee49 100644 --- a/tests/unit/utils/test_reactor.py +++ b/tests/unit/utils/test_reactor.py @@ -6,11 +6,11 @@ import glob import logging import os import textwrap -import yaml import salt.loader import salt.utils.data import salt.utils.reactor as reactor +import salt.utils.yaml from tests.support.unit import TestCase, skipIf from tests.support.mixins import AdaptedConfigurationTestCaseMixin @@ -390,7 +390,7 @@ class TestReactor(TestCase, AdaptedConfigurationTestCaseMixin): Load the reactor config for mocking ''' cls.opts = cls.get_temp_config('master') - reactor_config = yaml.safe_load(REACTOR_CONFIG) + reactor_config = salt.utils.yaml.safe_load(REACTOR_CONFIG) cls.opts.update(reactor_config) cls.reactor = reactor.Reactor(cls.opts) cls.reaction_map = salt.utils.data.repack_dictlist(reactor_config['reactor']) diff --git a/tests/unit/utils/test_schema.py b/tests/unit/utils/test_schema.py index dd5500f5fd3..0d1f305e575 100644 --- a/tests/unit/utils/test_schema.py +++ b/tests/unit/utils/test_schema.py @@ -6,13 +6,13 @@ # Import python libs from __future__ import absolute_import import copy -import yaml # Import Salt Testing Libs from tests.support.unit import TestCase, skipIf # Import Salt Libs import salt.utils.json +import salt.utils.yaml import salt.utils.schema as schema from salt.utils.versions import LooseVersion as _LooseVersion @@ -2112,7 +2112,7 @@ class ComplexSchemaTestCase(TestCase): self.assertDictEqual(complex_obj.get_definition(), expected_def) def test_complex_definition_schema(self): - serialized = yaml.safe_load(salt.utils.json.dumps(self.schema.serialize())) + serialized = salt.utils.yaml.safe_load(salt.utils.json.dumps(self.schema.serialize())) expected = { '$schema': 'http://json-schema.org/draft-04/schema#', 'title': 'Test Complex Definition Schema', @@ -2134,7 +2134,7 @@ class ComplexSchemaTestCase(TestCase): self.assertDictEqual(serialized, expected) def test_one_of_complex_definition_schema(self): - serialized = yaml.safe_load(salt.utils.json.dumps(self.one_of_schema.serialize())) + serialized = salt.utils.yaml.safe_load(salt.utils.json.dumps(self.one_of_schema.serialize())) expected = { '$schema': 'http://json-schema.org/draft-04/schema#', 'title': 'Test OneOf Complex Definitions Schema', @@ -2157,7 +2157,7 @@ class ComplexSchemaTestCase(TestCase): self.assertDictEqual(serialized, expected) def test_array_complex_definition_schema(self): - serialized = yaml.safe_load(salt.utils.json.dumps(self.array_schema.serialize())) + serialized = salt.utils.yaml.safe_load(salt.utils.json.dumps(self.array_schema.serialize())) expected = { '$schema': 'http://json-schema.org/draft-04/schema#', 'title': 'Test Array Complex Definitions Schema', @@ -2181,7 +2181,7 @@ class ComplexSchemaTestCase(TestCase): self.assertDictEqual(serialized, expected) def test_dict_complex_definition_schema(self): - serialized = yaml.safe_load(salt.utils.json.dumps(self.dict_schema.serialize())) + serialized = salt.utils.yaml.safe_load(salt.utils.json.dumps(self.dict_schema.serialize())) expected = { '$schema': 'http://json-schema.org/draft-04/schema#', 'title': 'Test Dict Complex Definitions Schema', @@ -2210,8 +2210,9 @@ class ComplexSchemaTestCase(TestCase): self.assertDictEqual(serialized, expected) def test_complex_complex_definition_schema(self): - serialized = yaml.safe_load(salt.utils.json.dumps( - self.complex_schema.serialize())) + serialized = salt.utils.yaml.safe_load(salt.utils.json.dumps( + self.complex_schema.serialize() + )) expected = { '$schema': 'http://json-schema.org/draft-04/schema#', 'title': 'Test Complex Complex Definition Schema', diff --git a/tests/unit/utils/test_yamlencoding.py b/tests/unit/utils/test_yamlencoding.py index 984c5377bee..a1f8570ba1f 100644 --- a/tests/unit/utils/test_yamlencoding.py +++ b/tests/unit/utils/test_yamlencoding.py @@ -5,9 +5,9 @@ Tests for salt.utils.yamlencoding # Import Python libs from __future__ import absolute_import -import yaml # Import Salt libs +import salt.utils.yaml import salt.utils.yamlencoding from tests.support.unit import TestCase @@ -16,7 +16,7 @@ class YamlEncodingTestCase(TestCase): def test_yaml_dquote(self): for teststr in (r'"\ []{}"',): - self.assertEqual(teststr, yaml.safe_load(salt.utils.yamlencoding.yaml_dquote(teststr))) + self.assertEqual(teststr, salt.utils.yaml.safe_load(salt.utils.yamlencoding.yaml_dquote(teststr))) def test_yaml_dquote_doesNotAddNewLines(self): teststr = '"' * 100 @@ -32,7 +32,7 @@ class YamlEncodingTestCase(TestCase): def test_yaml_encode(self): for testobj in (None, True, False, '[7, 5]', '"monkey"', 5, 7.5, "2014-06-02 15:30:29.7"): - self.assertEqual(testobj, yaml.safe_load(salt.utils.yamlencoding.yaml_encode(testobj))) + self.assertEqual(testobj, salt.utils.yaml.safe_load(salt.utils.yamlencoding.yaml_encode(testobj))) for testobj in ({}, [], set()): self.assertRaises(TypeError, salt.utils.yamlencoding.yaml_encode, testobj)