From 7b13a7df8b95119373606f8ed3af9cf3efc2a7b5 Mon Sep 17 00:00:00 2001 From: Erik Johnson Date: Fri, 22 Dec 2017 13:54:53 -0600 Subject: [PATCH] Replace json module usage with a helper to ensure unicode content is handled properly This adds wrappers for json.dump{,s} which disable `ensure_ascii` by default. --- doc/ref/returners/index.rst | 19 +- doc/topics/proxyminion/index.rst | 4 +- salt/client/ssh/__init__.py | 29 +- salt/client/ssh/shell.py | 6 +- salt/client/ssh/ssh_py_shim.py | 3 +- salt/client/ssh/state.py | 10 +- salt/client/ssh/wrapper/__init__.py | 14 +- salt/client/ssh/wrapper/config.py | 2 +- salt/client/ssh/wrapper/cp.py | 2 +- salt/client/ssh/wrapper/grains.py | 6 +- salt/client/ssh/wrapper/mine.py | 2 +- salt/client/ssh/wrapper/pillar.py | 2 +- salt/client/ssh/wrapper/publish.py | 2 +- salt/client/ssh/wrapper/state.py | 17 +- salt/cloud/clouds/aliyun.py | 4 +- salt/cloud/clouds/clc.py | 24 +- salt/cloud/clouds/digitalocean.py | 14 +- salt/cloud/clouds/ec2.py | 4 +- salt/cloud/clouds/joyent.py | 10 +- salt/cloud/clouds/lxc.py | 8 +- salt/cloud/clouds/openstack.py | 4 +- salt/cloud/clouds/proxmox.py | 12 +- salt/cloud/clouds/qingcloud.py | 7 +- salt/cloud/clouds/scaleway.py | 4 +- salt/daemons/flo/jobber.py | 4 +- salt/engines/docker_events.py | 4 +- salt/engines/hipchat.py | 14 +- salt/engines/http_logstash.py | 4 +- salt/engines/logentries.py | 61 +-- salt/engines/slack.py | 10 +- salt/engines/sqs_events.py | 4 +- salt/engines/test.py | 4 +- salt/executors/sudo.py | 4 +- salt/fileserver/azurefs.py | 7 +- salt/grains/mdata.py | 4 +- salt/grains/metadata.py | 4 +- salt/grains/smartos.py | 4 +- salt/key.py | 6 +- salt/log/handlers/logstash_mod.py | 6 +- salt/log/setup.py | 29 +- salt/modules/ansiblegate.py | 7 +- salt/modules/aptly.py | 4 +- salt/modules/aptpkg.py | 4 +- salt/modules/aws_sqs.py | 4 +- salt/modules/bigip.py | 80 +++- salt/modules/boto_apigateway.py | 6 +- salt/modules/boto_asg.py | 36 +- salt/modules/boto_cloudwatch.py | 14 +- salt/modules/boto_cloudwatch_event.py | 12 +- salt/modules/boto_ec2.py | 4 +- salt/modules/boto_elasticsearch_domain.py | 10 +- salt/modules/boto_elb.py | 39 +- salt/modules/boto_iam.py | 376 +++++++----------- salt/modules/boto_iot.py | 6 +- salt/modules/boto_kms.py | 8 +- salt/modules/boto_lambda.py | 6 +- salt/modules/boto_s3_bucket.py | 24 +- salt/modules/boto_sqs.py | 8 +- salt/modules/bower.py | 10 +- salt/modules/cassandra_cql.py | 8 +- salt/modules/chronos.py | 4 +- salt/modules/cmdmod.py | 6 +- salt/modules/consul.py | 15 +- salt/modules/ddns.py | 4 +- salt/modules/defaults.py | 8 +- salt/modules/dockermod.py | 14 +- salt/modules/glassfish.py | 6 +- salt/modules/grains.py | 4 +- salt/modules/heat.py | 4 +- salt/modules/hipchat.py | 4 +- salt/modules/ifttt.py | 4 +- salt/modules/influx.py | 4 +- salt/modules/ini_manage.py | 15 +- salt/modules/junos.py | 4 +- salt/modules/k8s.py | 26 +- salt/modules/kapacitor.py | 5 +- salt/modules/mac_brew.py | 11 +- salt/modules/mandrill.py | 8 +- salt/modules/marathon.py | 4 +- salt/modules/mattermost.py | 17 +- salt/modules/mongodb.py | 4 +- salt/modules/msteams.py | 17 +- salt/modules/npm.py | 8 +- salt/modules/opsgenie.py | 8 +- salt/modules/osquery.py | 8 +- salt/modules/pagerduty.py | 4 +- salt/modules/pagerduty_util.py | 6 +- salt/modules/pip.py | 4 +- salt/modules/rabbitmq.py | 17 +- salt/modules/rallydev.py | 4 +- salt/modules/random_org.py | 7 +- salt/modules/saltcloudmod.py | 7 +- salt/modules/serverdensity_device.py | 17 +- salt/modules/slack_notify.py | 12 +- salt/modules/smartos_imgadm.py | 10 +- salt/modules/smartos_vmadm.py | 27 +- salt/modules/solr.py | 10 +- salt/modules/state.py | 24 +- salt/modules/swarm.py | 10 +- salt/modules/telemetry.py | 47 ++- salt/modules/travisci.py | 5 +- salt/modules/uwsgi.py | 7 +- salt/modules/victorops.py | 4 +- salt/modules/win_dsc.py | 4 +- salt/modules/win_iis.py | 25 +- salt/modules/win_pki.py | 4 +- salt/modules/win_psget.py | 4 +- salt/modules/win_servermanager.py | 4 +- salt/modules/zabbix.py | 4 +- salt/modules/zenoss.py | 6 +- salt/netapi/rest_cherrypy/app.py | 20 +- salt/netapi/rest_cherrypy/event_processor.py | 6 +- salt/netapi/rest_tornado/event_processor.py | 9 +- salt/netapi/rest_tornado/saltnado.py | 25 +- .../rest_tornado/saltnado_websockets.py | 7 +- salt/netapi/rest_wsgi.py | 8 +- salt/output/json_out.py | 15 +- salt/pillar/cmd_json.py | 8 +- salt/pillar/redismod.py | 7 +- salt/proxy/philips_hue.py | 6 +- salt/proxy/ssh_sample.py | 6 +- salt/queues/pgjsonb_queue.py | 36 +- salt/queues/sqlite_queue.py | 26 +- salt/returners/cassandra_cql_return.py | 20 +- salt/returners/couchbase_return.py | 14 +- salt/returners/couchdb_return.py | 8 +- salt/returners/elasticsearch_return.py | 10 +- salt/returners/etcd_return.py | 10 +- salt/returners/highstate_return.py | 4 +- salt/returners/hipchat_return.py | 4 +- salt/returners/influxdb_return.py | 15 +- salt/returners/kafka_return.py | 4 +- salt/returners/mattermost_returner.py | 9 +- salt/returners/memcache_return.py | 14 +- salt/returners/mysql.py | 36 +- salt/returners/odbc.py | 21 +- salt/returners/postgres.py | 20 +- salt/returners/postgres_local_cache.py | 12 +- salt/returners/rawfile_json.py | 9 +- salt/returners/redis_return.py | 15 +- salt/returners/splunk.py | 46 ++- salt/returners/sqlite3_return.py | 14 +- salt/returners/syslog_return.py | 4 +- salt/roster/ansible.py | 4 +- salt/runners/ddns.py | 4 +- salt/runners/digicertapi.py | 4 +- salt/runners/mattermost.py | 12 +- salt/runners/pagerduty.py | 4 +- salt/runners/venafiapi.py | 6 +- salt/runners/vistara.py | 7 +- salt/sdb/tism.py | 10 +- salt/serializers/json.py | 14 +- salt/serializers/python.py | 18 +- salt/states/bigip.py | 5 +- salt/states/boto3_sns.py | 15 +- salt/states/boto_apigateway.py | 12 +- salt/states/boto_cfn.py | 6 +- salt/states/boto_cloudwatch_event.py | 11 +- salt/states/boto_datapipeline.py | 8 +- salt/states/boto_elasticsearch_domain.py | 13 +- salt/states/boto_iam.py | 16 +- salt/states/boto_iot.py | 28 +- salt/states/boto_lambda.py | 7 +- salt/states/boto_route53.py | 11 +- salt/states/boto_s3_bucket.py | 24 +- salt/states/boto_sqs.py | 10 +- salt/states/ceph.py | 4 +- salt/states/cmd.py | 4 +- salt/states/elasticsearch.py | 11 +- salt/states/glassfish.py | 6 +- salt/states/grafana.py | 17 +- salt/states/grafana4_dashboard.py | 24 +- salt/states/grafana_dashboard.py | 24 +- salt/states/heat.py | 19 +- salt/states/netyang.py | 6 +- salt/states/serverdensity_device.py | 11 +- salt/states/solrcloud.py | 17 +- salt/states/win_lgpo.py | 17 +- salt/template.py | 2 +- salt/thorium/file.py | 8 +- salt/utils/data.py | 127 ++++-- salt/utils/github.py | 9 +- salt/utils/http.py | 6 +- salt/utils/jinja.py | 6 +- salt/utils/json.py | 82 +++- salt/utils/pagerduty.py | 6 +- salt/utils/pkg/win.py | 4 +- salt/utils/schema.py | 6 +- salt/utils/ssdp.py | 23 +- salt/utils/thin.py | 26 +- tests/integration/__init__.py | 1 - tests/integration/cloud/helpers/virtualbox.py | 4 +- .../netapi/rest_cherrypy/test_app.py | 8 +- .../netapi/rest_tornado/test_app.py | 76 ++-- tests/integration/runners/test_state.py | 4 +- tests/integration/shell/test_key.py | 4 +- tests/integration/states/test_bower.py | 4 +- tests/jenkins.py | 8 +- tests/modparser.py | 4 +- tests/support/case.py | 4 +- tests/support/parser/cover.py | 8 +- tests/support/runtests.py | 5 +- tests/unit/config/schemas/test_ssh.py | 8 +- tests/unit/modules/test_ddns.py | 4 +- tests/unit/modules/test_junos.py | 2 +- tests/unit/modules/test_k8s.py | 40 +- tests/unit/modules/test_kapacitor.py | 4 +- tests/unit/modules/test_npm.py | 8 +- tests/unit/modules/test_pagerduty.py | 4 +- tests/unit/modules/test_saltcloudmod.py | 46 +-- .../unit/modules/test_serverdensity_device.py | 62 +-- tests/unit/modules/test_state.py | 36 +- tests/unit/modules/test_win_iis.py | 12 +- tests/unit/netapi/rest_cherrypy/test_tools.py | 13 +- .../unit/netapi/rest_tornado/test_handlers.py | 84 ++-- tests/unit/output/test_json_out.py | 20 +- tests/unit/states/test_boto_lambda.py | 4 +- tests/unit/states/test_file.py | 4 +- tests/unit/states/test_grafana.py | 8 +- tests/unit/templates/test_jinja.py | 4 +- tests/unit/test_context.py | 6 +- tests/unit/utils/test_schema.py | 12 +- 222 files changed, 1676 insertions(+), 1546 deletions(-) diff --git a/doc/ref/returners/index.rst b/doc/ref/returners/index.rst index 39b1c2b0fc6..04bd482434f 100644 --- a/doc/ref/returners/index.rst +++ b/doc/ref/returners/index.rst @@ -72,7 +72,7 @@ Other optional functions can be included to add support for .. code-block:: python import redis - import json + import salt.utils.json def returner(ret): ''' @@ -84,7 +84,7 @@ Other optional functions can be included to add support for port=6379, db='0') serv.sadd("%(id)s:jobs" % ret, ret['jid']) - serv.set("%(jid)s:%(id)s" % ret, json.dumps(ret['return'])) + serv.set("%(jid)s:%(id)s" % ret, salt.utils.json.dumps(ret['return'])) serv.sadd('jobs', ret['jid']) serv.sadd(ret['jid'], ret['id']) @@ -168,6 +168,8 @@ must implement the following functions: .. code-block:: python + import salt.utils.json + def save_load(jid, load): ''' Save the load to the specified jid id @@ -176,7 +178,7 @@ must implement the following functions: jid, load ) VALUES ( '{0}', '{1}' - );'''.format(jid, json.dumps(load)) + );'''.format(jid, salt.utils.json.dumps(load)) # cassandra_cql.cql_query may raise a CommandExecutionError try: @@ -185,8 +187,9 @@ must implement the following functions: log.critical('Could not save load in jids table.') raise except Exception as e: - log.critical('''Unexpected error while inserting into - jids: {0}'''.format(str(e))) + log.critical( + 'Unexpected error while inserting into jids: {0}'.format(e) + ) raise @@ -316,6 +319,8 @@ contains the jid and therefore is guaranteed to be unique. .. code-block:: python + import salt.utils.json + def event_return(events): ''' Return event to mysql server @@ -329,7 +334,7 @@ contains the jid and therefore is guaranteed to be unique. data = event.get('data', '') sql = '''INSERT INTO `salt_events` (`tag`, `data`, `master_id` ) VALUES (%s, %s, %s)''' - cur.execute(sql, (tag, json.dumps(data), __opts__['id'])) + cur.execute(sql, (tag, salt.utils.json.dumps(data), __opts__['id'])) Testing the Returner @@ -358,7 +363,7 @@ infrastructure, all events seen by a salt master may be logged to one or more returners. To enable event logging, set the ``event_return`` configuration option in the -master config to the returner(s) which should be designated as the handler +master config to the returner(s) which should be designated as the handler for event returns. .. note:: diff --git a/doc/topics/proxyminion/index.rst b/doc/topics/proxyminion/index.rst index bfa5383155f..c398020f72e 100644 --- a/doc/topics/proxyminion/index.rst +++ b/doc/topics/proxyminion/index.rst @@ -753,7 +753,7 @@ This proxymodule enables "package" installation. from __future__ import absolute_import # Import python libs - import json + import salt.utils.json import logging # Import Salt's libs @@ -821,7 +821,7 @@ This proxymodule enables "package" installation. jsonret.append(ln_) if '}' in ln_: in_json = False - return json.loads('\n'.join(jsonret)) + return salt.utils.json.loads('\n'.join(jsonret)) def package_list(): diff --git a/salt/client/ssh/__init__.py b/salt/client/ssh/__init__.py index 3c580f84fe3..c8cd09ba152 100644 --- a/salt/client/ssh/__init__.py +++ b/salt/client/ssh/__init__.py @@ -3,11 +3,10 @@ Create ssh executor system ''' # Import python libs -from __future__ import absolute_import, print_function, unicode_literals +from __future__ import absolute_import, print_function import base64 import copy import getpass -import json import logging import multiprocessing import subprocess @@ -1017,7 +1016,7 @@ class Single(object): if '_error' in opts_pkg: #Refresh failed retcode = opts_pkg['retcode'] - ret = json.dumps({'local': opts_pkg}) + ret = salt.utils.json.dumps({'local': opts_pkg}) return ret, retcode opts_pkg['file_roots'] = self.opts['file_roots'] @@ -1139,9 +1138,9 @@ class Single(object): # Mimic the json data-structure that "salt-call --local" will # emit (as seen in ssh_py_shim.py) if isinstance(result, dict) and 'local' in result: - ret = json.dumps({'local': result['local']}) + ret = salt.utils.json.dumps({'local': result['local']}) else: - ret = json.dumps({'local': {'return': result}}) + ret = salt.utils.json.dumps({'local': {'return': result}}) return ret, retcode def _cmd_str(self): @@ -1176,16 +1175,16 @@ OPTIONS.wipe = {7} OPTIONS.tty = {8} OPTIONS.cmd_umask = {9} ARGS = {10}\n'''.format(self.minion_config, - RSTR, - self.thin_dir, - thin_sum, - 'sha1', - salt.version.__version__, - self.mods.get('version', ''), - self.wipe, - self.tty, - self.cmd_umask, - self.argv) + RSTR, + self.thin_dir, + thin_sum, + 'sha1', + salt.version.__version__, + self.mods.get('version', ''), + self.wipe, + self.tty, + self.cmd_umask, + self.argv) py_code = SSH_PY_SHIM.replace('#%%OPTS', arg_str) if six.PY2: py_code_enc = py_code.encode('base64') diff --git a/salt/client/ssh/shell.py b/salt/client/ssh/shell.py index 97d02a96825..427c0a8a94f 100644 --- a/salt/client/ssh/shell.py +++ b/salt/client/ssh/shell.py @@ -2,19 +2,19 @@ ''' Manage transport commands via ssh ''' -from __future__ import absolute_import, print_function, unicode_literals +from __future__ import absolute_import, print_function # Import python libs import re import os import sys -import json import time import logging import subprocess # Import salt libs import salt.defaults.exitcodes +import salt.utils.json import salt.utils.nb_popen import salt.utils.vt @@ -420,7 +420,7 @@ class Shell(object): 'flag:\n{0}').format(stdout) return ret_stdout, '', 254 elif buff and buff.endswith('_||ext_mods||_'): - mods_raw = json.dumps(self.mods, separators=(',', ':')) + '|_E|0|' + mods_raw = salt.utils.json.dumps(self.mods, separators=(',', ':')) + '|_E|0|' term.sendline(mods_raw) if stdout: old_stdout = stdout diff --git a/salt/client/ssh/ssh_py_shim.py b/salt/client/ssh/ssh_py_shim.py index 47ebc383772..6a5df938c86 100644 --- a/salt/client/ssh/ssh_py_shim.py +++ b/salt/client/ssh/ssh_py_shim.py @@ -7,8 +7,7 @@ This is not intended to be instantiated as a module, rather it is a helper script used by salt.client.ssh.Single. It is here, in a separate file, for convenience of development. ''' - -from __future__ import absolute_import, print_function, unicode_literals +from __future__ import absolute_import, print_function import hashlib import tarfile diff --git a/salt/client/ssh/state.py b/salt/client/ssh/state.py index 40eb4fd2e35..b912e8ab128 100644 --- a/salt/client/ssh/state.py +++ b/salt/client/ssh/state.py @@ -2,13 +2,12 @@ ''' Create ssh executor system ''' -from __future__ import absolute_import, print_function, unicode_literals +from __future__ import absolute_import, print_function # Import python libs import logging import os import tarfile import tempfile -import json import shutil from contextlib import closing @@ -16,6 +15,7 @@ from contextlib import closing import salt.client.ssh.shell import salt.client.ssh import salt.utils.files +import salt.utils.json import salt.utils.path import salt.utils.stringutils import salt.utils.thin @@ -181,13 +181,13 @@ def prep_trans_tar(opts, file_client, chunks, file_refs, pillar=None, id_=None, [salt.utils.url.create('_utils')], ] with salt.utils.files.fopen(lowfn, 'w+') as fp_: - fp_.write(salt.utils.stringutils.to_str(json.dumps(chunks))) + salt.utils.json.dump(chunks, fp_) if pillar: with salt.utils.files.fopen(pillarfn, 'w+') as fp_: - fp_.write(salt.utils.stringutils.to_str(json.dumps(pillar))) + salt.utils.json.dump(pillar, fp_) if roster_grains: with salt.utils.files.fopen(roster_grainsfn, 'w+') as fp_: - fp_.write(salt.utils.stringutils.to_str(json.dumps(roster_grains))) + salt.utils.json.dump(roster_grains, fp_) if id_ is None: id_ = '' diff --git a/salt/client/ssh/wrapper/__init__.py b/salt/client/ssh/wrapper/__init__.py index 6bda039a318..c5d1b5708f5 100644 --- a/salt/client/ssh/wrapper/__init__.py +++ b/salt/client/ssh/wrapper/__init__.py @@ -7,13 +7,13 @@ as ZeroMQ salt, but via ssh. ''' # Import python libs -from __future__ import absolute_import, print_function, unicode_literals -import json +from __future__ import absolute_import, print_function import copy # Import salt libs import salt.loader import salt.utils.data +import salt.utils.json import salt.client.ssh # Import 3rd-party libs @@ -103,8 +103,12 @@ class FunctionWrapper(object): The remote execution function ''' argv = [cmd] - argv.extend([json.dumps(arg) for arg in args]) - argv.extend(['{0}={1}'.format(key, json.dumps(val)) for key, val in six.iteritems(kwargs)]) + argv.extend([salt.utils.json.dumps(arg) for arg in args]) + argv.extend( + ['{0}={1}'.format(salt.utils.stringutils.to_str(key), + salt.utils.json.dumps(val)) + for key, val in six.iteritems(kwargs)] + ) single = salt.client.ssh.Single( self.opts, argv, @@ -121,7 +125,7 @@ class FunctionWrapper(object): 'stderr': stderr, 'retcode': retcode} try: - ret = json.loads(stdout, object_hook=salt.utils.data.decode_dict) + ret = salt.utils.json.loads(stdout, object_hook=salt.utils.data.decode_dict) if len(ret) < 2 and 'local' in ret: ret = ret['local'] ret = ret.get('return', {}) diff --git a/salt/client/ssh/wrapper/config.py b/salt/client/ssh/wrapper/config.py index f46da392093..4b84bc1e6f9 100644 --- a/salt/client/ssh/wrapper/config.py +++ b/salt/client/ssh/wrapper/config.py @@ -4,7 +4,7 @@ Return config information ''' # Import python libs -from __future__ import absolute_import, print_function, unicode_literals +from __future__ import absolute_import, print_function import re import os diff --git a/salt/client/ssh/wrapper/cp.py b/salt/client/ssh/wrapper/cp.py index d5c8b6ce71e..894e62f94c8 100644 --- a/salt/client/ssh/wrapper/cp.py +++ b/salt/client/ssh/wrapper/cp.py @@ -3,7 +3,7 @@ Wrap the cp module allowing for managed ssh file transfers ''' # Import Python libs -from __future__ import absolute_import, print_function, unicode_literals +from __future__ import absolute_import, print_function import logging import os diff --git a/salt/client/ssh/wrapper/grains.py b/salt/client/ssh/wrapper/grains.py index 631ba5ee9ac..372276ef53a 100644 --- a/salt/client/ssh/wrapper/grains.py +++ b/salt/client/ssh/wrapper/grains.py @@ -4,15 +4,15 @@ Return/control aspects of the grains data ''' # Import python libs -from __future__ import absolute_import, print_function, unicode_literals +from __future__ import absolute_import, print_function import collections import copy import math -import json # Import salt libs import salt.utils.data import salt.utils.dictupdate +import salt.utils.json from salt.defaults import DEFAULT_TARGET_DELIM from salt.exceptions import SaltException @@ -74,7 +74,7 @@ def get(key, default='', delimiter=DEFAULT_TARGET_DELIM, ordered=True): if ordered is True: grains = __grains__ else: - grains = json.loads(json.dumps(__grains__)) + grains = salt.utils.json.loads(salt.utils.json.dumps(__grains__)) return salt.utils.data.traverse_dict_and_list( __grains__, key, diff --git a/salt/client/ssh/wrapper/mine.py b/salt/client/ssh/wrapper/mine.py index 7b6308811c0..5f493b9ae52 100644 --- a/salt/client/ssh/wrapper/mine.py +++ b/salt/client/ssh/wrapper/mine.py @@ -7,7 +7,7 @@ Wrapper function for mine operations for salt-ssh ''' # Import python libs -from __future__ import absolute_import, print_function, unicode_literals +from __future__ import absolute_import, print_function import copy # Import salt libs diff --git a/salt/client/ssh/wrapper/pillar.py b/salt/client/ssh/wrapper/pillar.py index 031f593e64d..64d58ac3743 100644 --- a/salt/client/ssh/wrapper/pillar.py +++ b/salt/client/ssh/wrapper/pillar.py @@ -2,7 +2,7 @@ ''' Extract the pillar data for this minion ''' -from __future__ import absolute_import, print_function, unicode_literals +from __future__ import absolute_import, print_function # Import python libs import collections diff --git a/salt/client/ssh/wrapper/publish.py b/salt/client/ssh/wrapper/publish.py index 1c90cc3c7e9..76a024382fb 100644 --- a/salt/client/ssh/wrapper/publish.py +++ b/salt/client/ssh/wrapper/publish.py @@ -10,7 +10,7 @@ salt-ssh calls and return the data from them. No access control is needed because calls cannot originate from the minions. ''' # Import python libs -from __future__ import absolute_import, print_function, unicode_literals +from __future__ import absolute_import, print_function import copy import logging diff --git a/salt/client/ssh/wrapper/state.py b/salt/client/ssh/wrapper/state.py index e459dcb9c2f..a1729843c81 100644 --- a/salt/client/ssh/wrapper/state.py +++ b/salt/client/ssh/wrapper/state.py @@ -2,12 +2,10 @@ ''' Create ssh executor system ''' - -from __future__ import absolute_import, print_function, unicode_literals +from __future__ import absolute_import, print_function # Import python libs import os import time -import json import logging # Import salt libs @@ -19,6 +17,7 @@ import salt.utils.data import salt.utils.files import salt.utils.hashutils import salt.utils.jid +import salt.utils.json import salt.utils.platform import salt.utils.state import salt.utils.thin @@ -174,7 +173,7 @@ def sls(mods, saltenv='base', test=None, exclude=None, **kwargs): # Read in the JSON data and return the data structure try: - return json.loads(stdout, object_hook=salt.utils.data.decode_dict) + return salt.utils.json.loads(stdout, object_hook=salt.utils.data.decode_dict) except Exception as e: log.error("JSON Render failed for: %s\n%s", stdout, stderr) log.error(six.text_type(e)) @@ -318,7 +317,7 @@ def low(data, **kwargs): # Read in the JSON data and return the data structure try: - return json.loads(stdout, object_hook=salt.utils.data.decode_dict) + return salt.utils.json.loads(stdout, object_hook=salt.utils.data.decode_dict) except Exception as e: log.error("JSON Render failed for: %s\n%s", stdout, stderr) log.error(six.text_type(e)) @@ -407,7 +406,7 @@ def high(data, **kwargs): # Read in the JSON data and return the data structure try: - return json.loads(stdout, object_hook=salt.utils.data.decode_dict) + return salt.utils.json.loads(stdout, object_hook=salt.utils.data.decode_dict) except Exception as e: log.error("JSON Render failed for: %s\n%s", stdout, stderr) log.error(six.text_type(e)) @@ -650,7 +649,7 @@ def highstate(test=None, **kwargs): # Read in the JSON data and return the data structure try: - return json.loads(stdout, object_hook=salt.utils.data.decode_dict) + return salt.utils.json.loads(stdout, object_hook=salt.utils.data.decode_dict) except Exception as e: log.error("JSON Render failed for: %s\n%s", stdout, stderr) log.error(six.text_type(e)) @@ -731,7 +730,7 @@ def top(topfn, test=None, **kwargs): # Read in the JSON data and return the data structure try: - return json.loads(stdout, object_hook=salt.utils.data.decode_dict) + return salt.utils.json.loads(stdout, object_hook=salt.utils.data.decode_dict) except Exception as e: log.error("JSON Render failed for: %s\n%s", stdout, stderr) log.error(six.text_type(e)) @@ -1094,7 +1093,7 @@ def single(fun, name, test=None, **kwargs): # Read in the JSON data and return the data structure try: - return json.loads(stdout, object_hook=salt.utils.data.decode_dict) + return salt.utils.json.loads(stdout, object_hook=salt.utils.data.decode_dict) except Exception as e: log.error("JSON Render failed for: %s\n%s", stdout, stderr) log.error(six.text_type(e)) diff --git a/salt/cloud/clouds/aliyun.py b/salt/cloud/clouds/aliyun.py index c6635b68330..c2585931bbc 100644 --- a/salt/cloud/clouds/aliyun.py +++ b/salt/cloud/clouds/aliyun.py @@ -28,7 +28,6 @@ Set up the cloud configuration at ``/etc/salt/cloud.providers`` or # Import python libs from __future__ import absolute_import import time -import json import pprint import logging import hmac @@ -43,6 +42,7 @@ from salt.ext.six.moves.urllib.parse import quote as _quote # pylint: disable=i # Import salt cloud libs import salt.utils.cloud import salt.utils.data +import salt.utils.json import salt.config as config from salt.exceptions import ( SaltCloudNotFound, @@ -824,7 +824,7 @@ def query(params=None): content = request.text - result = json.loads(content, object_hook=salt.utils.data.encode_dict) + result = salt.utils.json.loads(content, object_hook=salt.utils.data.encode_dict) if 'Code' in result: raise SaltCloudSystemExit( pprint.pformat(result.get('Message', {})) diff --git a/salt/cloud/clouds/clc.py b/salt/cloud/clouds/clc.py index c1a2b7531cc..c5f5457ead7 100644 --- a/salt/cloud/clouds/clc.py +++ b/salt/cloud/clouds/clc.py @@ -63,14 +63,14 @@ cloud configuration at # Import python libs from __future__ import absolute_import +import importlib import logging import time -import json + # Import salt libs -import importlib -from salt.exceptions import SaltCloudSystemExit -# Import salt cloud libs import salt.config as config +import salt.utils.json +from salt.exceptions import SaltCloudSystemExit # Get logging started log = logging.getLogger(__name__) @@ -157,8 +157,8 @@ def list_nodes_full(call=None, for_output=True): creds = get_creds() clc.v1.SetCredentials(creds["token"], creds["token_pass"]) servers_raw = clc.v1.Server.GetServers(location=None) - servers_raw = json.dumps(servers_raw) - servers = json.loads(servers_raw) + servers_raw = salt.utils.json.dumps(servers_raw) + servers = salt.utils.json.loads(servers_raw) return servers @@ -181,8 +181,8 @@ def get_monthly_estimate(call=None, for_output=True): ) try: billing_raw = clc.v1.Billing.GetAccountSummary(alias=creds["accountalias"]) - billing_raw = json.dumps(billing_raw) - billing = json.loads(billing_raw) + billing_raw = salt.utils.json.dumps(billing_raw) + billing = salt.utils.json.loads(billing_raw) billing = round(billing["MonthlyEstimate"], 2) return {"Monthly Estimate": billing} except RuntimeError: @@ -201,8 +201,8 @@ def get_month_to_date(call=None, for_output=True): ) try: billing_raw = clc.v1.Billing.GetAccountSummary(alias=creds["accountalias"]) - billing_raw = json.dumps(billing_raw) - billing = json.loads(billing_raw) + billing_raw = salt.utils.json.dumps(billing_raw) + billing = salt.utils.json.loads(billing_raw) billing = round(billing["MonthToDateTotal"], 2) return {"Month To Date": billing} except RuntimeError: @@ -243,8 +243,8 @@ def get_group_estimate(call=None, for_output=True, **kwargs): ) try: billing_raw = clc.v1.Billing.GetGroupEstimate(group=group, alias=creds["accountalias"], location=location) - billing_raw = json.dumps(billing_raw) - billing = json.loads(billing_raw) + billing_raw = salt.utils.json.dumps(billing_raw) + billing = salt.utils.json.loads(billing_raw) estimate = round(billing["MonthlyEstimate"], 2) month_to_date = round(billing["MonthToDate"], 2) return {"Monthly Estimate": estimate, "Month to Date": month_to_date} diff --git a/salt/cloud/clouds/digitalocean.py b/salt/cloud/clouds/digitalocean.py index aee635b3e54..c26b498b059 100644 --- a/salt/cloud/clouds/digitalocean.py +++ b/salt/cloud/clouds/digitalocean.py @@ -27,16 +27,16 @@ under the "SSH Keys" section. # Import Python Libs from __future__ import absolute_import -import os -import time -import json -import pprint -import logging import decimal +import logging +import os +import pprint +import time # Import Salt Libs import salt.utils.cloud import salt.utils.files +import salt.utils.json import salt.config as config from salt.exceptions import ( SaltCloudConfigError, @@ -562,7 +562,7 @@ def query(method='droplets', droplet_id=None, command=None, args=None, http_meth 'personal_access_token', get_configured_provider(), __opts__, search_global=False ) - data = json.dumps(args) + data = salt.utils.json.dumps(args) requester = getattr(requests, http_method) request = requester(path, data=data, headers={'Authorization': 'Bearer ' + personal_access_token, 'Content-Type': 'application/json'}) @@ -584,7 +584,7 @@ def query(method='droplets', droplet_id=None, command=None, args=None, http_meth content = request.text - result = json.loads(content) + result = salt.utils.json.loads(content) if result.get('status', '').lower() == 'error': raise SaltCloudSystemExit( pprint.pformat(result.get('error_message', {})) diff --git a/salt/cloud/clouds/ec2.py b/salt/cloud/clouds/ec2.py index b7a9061c003..acf7b2eefda 100644 --- a/salt/cloud/clouds/ec2.py +++ b/salt/cloud/clouds/ec2.py @@ -84,7 +84,6 @@ import binascii import datetime import base64 import msgpack -import json import re import decimal @@ -92,6 +91,7 @@ import decimal import salt.utils.cloud import salt.utils.files import salt.utils.hashutils +import salt.utils.json from salt._compat import ElementTree as ET import salt.utils.http as http import salt.utils.aws as aws @@ -4845,7 +4845,7 @@ def _parse_pricing(url, name): # Turn the data into something that's easier/faster to process regions = {} - price_json = json.loads(price_js) + price_json = salt.utils.json.loads(price_js) for region in price_json['config']['regions']: sizes = {} for itype in region['instanceTypes']: diff --git a/salt/cloud/clouds/joyent.py b/salt/cloud/clouds/joyent.py index 90e7208a935..d6deb94d361 100644 --- a/salt/cloud/clouds/joyent.py +++ b/salt/cloud/clouds/joyent.py @@ -53,7 +53,6 @@ included: # Import python libs from __future__ import absolute_import import os -import json import logging import base64 import pprint @@ -70,6 +69,7 @@ from salt.ext.six.moves import http_client # pylint: disable=import-error,no-na import salt.utils.cloud import salt.utils.files import salt.utils.http +import salt.utils.json import salt.config as config from salt.cloud.libcloudfuncs import node_state from salt.exceptions import ( @@ -357,7 +357,7 @@ def create_node(**kwargs): if firewall_enabled is not None: create_data['firewall_enabled'] = firewall_enabled - data = json.dumps(create_data) + data = salt.utils.json.dumps(create_data) ret = query(command='/my/machines', data=data, method='POST', location=location) @@ -503,7 +503,7 @@ def take_action(name=None, call=None, command=None, data=None, method='GET', ) if data: - data = json.dumps(data) + data = salt.utils.json.dumps(data) ret = [] try: @@ -956,7 +956,7 @@ def import_key(kwargs=None, call=None): kwargs['key'] = fp_.read() send_data = {'name': kwargs['keyname'], 'key': kwargs['key']} - kwargs['data'] = json.dumps(send_data) + kwargs['data'] = salt.utils.json.dumps(send_data) rcode, data = query( command='my/keys', @@ -1095,7 +1095,7 @@ def query(action=None, # post form data if not data: - data = json.dumps({}) + data = salt.utils.json.dumps({}) return_content = None result = salt.utils.http.query( diff --git a/salt/cloud/clouds/lxc.py b/salt/cloud/clouds/lxc.py index 64d0238fa50..260d75167eb 100644 --- a/salt/cloud/clouds/lxc.py +++ b/salt/cloud/clouds/lxc.py @@ -10,7 +10,6 @@ Please read :ref:`core config documentation `. # Import python libs from __future__ import absolute_import -import json import os import logging import copy @@ -19,6 +18,7 @@ from pprint import pformat # Import salt cloud libs import salt.utils.cloud +import salt.utils.json import salt.config as config from salt.exceptions import SaltCloudSystemExit @@ -137,15 +137,15 @@ def _salt(fun, *args, **kw): cache = True laps = laps // __CACHED_FUNS[fun] try: - sargs = json.dumps(args) + sargs = salt.utils.json.dumps(args) except TypeError: sargs = '' try: - skw = json.dumps(kw) + skw = salt.utils.json.dumps(kw) except TypeError: skw = '' try: - skwargs = json.dumps(kwargs) + skwargs = salt.utils.json.dumps(kwargs) except TypeError: skwargs = '' cache_key = (laps, target, fun, sargs, skw, skwargs) diff --git a/salt/cloud/clouds/openstack.py b/salt/cloud/clouds/openstack.py index e14283125f9..c0d1c7234fe 100644 --- a/salt/cloud/clouds/openstack.py +++ b/salt/cloud/clouds/openstack.py @@ -194,13 +194,13 @@ from __future__ import absolute_import # Import Python Libs import copy -import json import logging import os import pprint import socket # Import Salt Libs +import salt.utils.json import salt.config as config import salt.ext.six as six from salt.exceptions import ( @@ -817,7 +817,7 @@ def call(conn=None, call=None, kwargs=None): func = kwargs.pop('func') for key, value in kwargs.items(): try: - kwargs[key] = json.loads(value) + kwargs[key] = salt.utils.json.loads(value) except ValueError: continue try: diff --git a/salt/cloud/clouds/proxmox.py b/salt/cloud/clouds/proxmox.py index 8690a538e11..2478e442bdc 100644 --- a/salt/cloud/clouds/proxmox.py +++ b/salt/cloud/clouds/proxmox.py @@ -32,11 +32,10 @@ import time import pprint import logging import re -import json # Import salt libs -from salt.ext import six -import salt.utils +import salt.utils.cloud +import salt.utils.json # Import salt cloud libs import salt.config as config @@ -45,9 +44,10 @@ from salt.exceptions import ( SaltCloudExecutionFailure, SaltCloudExecutionTimeout ) -from salt.ext.six.moves import range -# Import Third Party Libs +# Import 3rd-party Libs +from salt.ext import six +from salt.ext.six.moves import range try: import requests HAS_REQUESTS = True @@ -632,7 +632,7 @@ def _import_api(): re_filter = re.compile('(?<=pveapi =)(.*)(?=^;)', re.DOTALL | re.MULTILINE) api_json = re_filter.findall(returned_data.text)[0] - api = json.loads(api_json) + api = salt.utils.json.loads(api_json) def _get_properties(path="", method="GET", forced_params=None): diff --git a/salt/cloud/clouds/qingcloud.py b/salt/cloud/clouds/qingcloud.py index fc6c7112151..18a2d7396fb 100644 --- a/salt/cloud/clouds/qingcloud.py +++ b/salt/cloud/clouds/qingcloud.py @@ -29,7 +29,6 @@ Set up the cloud configuration at ``/etc/salt/cloud.providers`` or # Import python libs from __future__ import absolute_import import time -import json import pprint import logging import hmac @@ -41,6 +40,7 @@ from salt.ext.six.moves.urllib.parse import quote as _quote # pylint: disable=i from salt.ext.six.moves import range import salt.utils.cloud import salt.utils.data +import salt.utils.json import salt.config as config from salt.exceptions import ( SaltCloudNotFound, @@ -156,8 +156,7 @@ def query(params=None): if isinstance(value[i - 1], dict): for sk, sv in value[i - 1].items(): if isinstance(sv, dict) or isinstance(sv, list): - # sv = json_dump(sv) - sv = json.dumps(sv, separators=(',', ':')) + sv = salt.utils.json.dumps(sv, separators=(',', ':')) real_parameters['{0}.{1}.{2}'.format(key, i, sk)] = sv else: real_parameters['{0}.{1}'.format(key, i)] = value[i - 1] @@ -188,7 +187,7 @@ def query(params=None): log.debug(request.url) content = request.text - result = json.loads(content, object_hook=salt.utils.data.encode_dict) + result = salt.utils.json.loads(content, object_hook=salt.utils.data.encode_dict) # print('response:') # pprint.pprint(result) diff --git a/salt/cloud/clouds/scaleway.py b/salt/cloud/clouds/scaleway.py index 171401ce3b8..d00be013c63 100644 --- a/salt/cloud/clouds/scaleway.py +++ b/salt/cloud/clouds/scaleway.py @@ -25,7 +25,6 @@ the cloud configuration at ``/etc/salt/cloud.providers`` or # Import Python Libs from __future__ import absolute_import -import json import logging import pprint import time @@ -33,6 +32,7 @@ import time # Import Salt Libs from salt.ext.six.moves import range import salt.utils.cloud +import salt.utils.json import salt.config as config from salt.exceptions import ( SaltCloudNotFound, @@ -347,7 +347,7 @@ def query(method='servers', server_id=None, command=None, args=None, 'token', get_configured_provider(), __opts__, search_global=False ) - data = json.dumps(args) + data = salt.utils.json.dumps(args) requester = getattr(requests, http_method) request = requester( diff --git a/salt/daemons/flo/jobber.py b/salt/daemons/flo/jobber.py index cf68fbc714f..63c66225648 100644 --- a/salt/daemons/flo/jobber.py +++ b/salt/daemons/flo/jobber.py @@ -14,7 +14,6 @@ import logging import traceback import multiprocessing import subprocess -import json # Import salt libs from salt.ext import six @@ -22,6 +21,7 @@ import salt.daemons.masterapi import salt.utils.args import salt.utils.data import salt.utils.files +import salt.utils.json import salt.utils.kinds as kinds import salt.utils.process import salt.utils.stringutils @@ -62,7 +62,7 @@ def jobber_check(self): rms.append(jid) data = self.shells.value[jid] stdout, stderr = data['proc'].communicate() - ret = json.loads(salt.utils.stringutils.to_str(stdout), object_hook=salt.utils.data.encode_dict)['local'] + ret = salt.utils.json.loads(stdout, object_hook=salt.utils.data.encode_dict)['local'] route = {'src': (self.stack.value.local.name, 'manor', 'jid_ret'), 'dst': (data['msg']['route']['src'][0], None, 'remote_cmd')} ret['cmd'] = '_return' diff --git a/salt/engines/docker_events.py b/salt/engines/docker_events.py index d28d3452678..54551112bb6 100644 --- a/salt/engines/docker_events.py +++ b/salt/engines/docker_events.py @@ -7,10 +7,10 @@ Send events from Docker events # Import Python Libs from __future__ import absolute_import -import json import logging import traceback +import salt.utils.json import salt.utils.event # pylint: disable=import-error @@ -83,7 +83,7 @@ def start(docker_url='unix://var/run/docker.sock', try: events = client.events() for event in events: - data = json.loads(event.decode(__salt_system_encoding__, errors='replace')) + data = salt.utils.json.loads(event.decode(__salt_system_encoding__, errors='replace')) # https://github.com/docker/cli/blob/master/cli/command/system/events.go#L109 # https://github.com/docker/engine-api/blob/master/types/events/events.go # Each output includes the event type, actor id, name and action. diff --git a/salt/engines/hipchat.py b/salt/engines/hipchat.py index f28e4900fdd..1ed79caa884 100644 --- a/salt/engines/hipchat.py +++ b/salt/engines/hipchat.py @@ -38,7 +38,6 @@ keys make the engine interactive. from __future__ import absolute_import import logging import time -import json import os @@ -52,6 +51,7 @@ import salt.utils.args import salt.utils.event import salt.utils.files import salt.utils.http +import salt.utils.json import salt.runner import salt.client import salt.loader @@ -93,10 +93,11 @@ def _publish_file(token, room, filepath, message='', outputter=None, api_url=Non url = "{0}/v2/room/{1}/share/file".format(api_url, room) headers = {'Content-type': 'multipart/related; boundary=boundary123456'} headers['Authorization'] = "Bearer " + token - msg = json.dumps({'message': message}) + msg = salt.utils.json.dumps({'message': message}) + # future lint: disable=blacklisted-function with salt.utils.files.fopen(filepath, 'rb') as rfh: - payload = """\ + payload = str("""\ --boundary123456 Content-Type: application/json; charset=UTF-8 Content-Disposition: attachment; name="metadata" @@ -109,7 +110,10 @@ Content-Disposition: attachment; name="file"; filename="{1}" {2} --boundary123456--\ -""".format(msg, os.path.basename(filepath), rfh.read()) +""").format(msg, + os.path.basename(salt.utils.stringutils.to_str(filepath)), + salt.utils.stringutils.to_str(rfh.read())) + # future lint: enable=blacklisted-function salt.utils.http.query(url, method='POST', header_dict=headers, data=payload) @@ -414,7 +418,7 @@ def start(token, else: tmp_path_fn = salt.utils.files.mkstemp() with salt.utils.files.fopen(tmp_path_fn, 'w+') as fp_: - fp_.write(json.dumps(ret, sort_keys=True, indent=4)) + salt.utils.json.dump(ret, fp_, sort_keys=True, indent=4) _publish_file(token, room, tmp_path_fn, message=message_string, api_url=api_url) salt.utils.files.safe_rm(tmp_path_fn) time.sleep(wait_time or _DEFAULT_SLEEP) diff --git a/salt/engines/http_logstash.py b/salt/engines/http_logstash.py index b0a4c1b41ed..a6c894797a7 100644 --- a/salt/engines/http_logstash.py +++ b/salt/engines/http_logstash.py @@ -32,12 +32,12 @@ them onto a logstash endpoint via HTTP requests. from __future__ import absolute_import # Import python lib -import json import fnmatch # Import salt libs import salt.utils.http import salt.utils.event +import salt.utils.json # ---------------------------------------------------------------------------------------------------------------------- # module properties @@ -62,7 +62,7 @@ def _logstash(url, data): url, 'POST', header_dict=_HEADERS, - data=json.dumps(data), + data=salt.utils.json.dumps(data), decode=True, status=True, opts=__opts__ diff --git a/salt/engines/logentries.py b/salt/engines/logentries.py index 9dd0fc37358..7b59ba48348 100644 --- a/salt/engines/logentries.py +++ b/salt/engines/logentries.py @@ -42,11 +42,10 @@ To test this engine salt '*' test.ping cmd.run uptime ''' - -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals # Import Salt libs import salt.utils.event -from salt.ext import six +import salt.utils.json # Import third party libs try: @@ -67,33 +66,14 @@ except ImportError: # for systems without TLS support. import socket import random import time -import codecs import uuid import logging -import json log = logging.getLogger(__name__) def __virtual__(): - if not HAS_CERTIFI: - return False - if not HAS_SSL: - return False - - return True - - -def _to_unicode(ch): - return codecs.unicode_escape_decode(ch)[0] - - -def _is_unicode(ch): - return isinstance(ch, six.text_type) - - -def _create_unicode(ch): - return six.text_type(ch, 'utf-8') + return True if HAS_CERTIFI and HAS_SSL else False class PlainTextSocketAppender(object): @@ -111,8 +91,8 @@ class PlainTextSocketAppender(object): # Error message displayed when an incorrect Token has been detected self.INVALID_TOKEN = ("\n\nIt appears the LOGENTRIES_TOKEN " "parameter you entered is incorrect!\n\n") - # Unicode Line separator character \u2028 - self.LINE_SEP = _to_unicode(r'\u2028') + # Encoded unicode line separator + self.LINE_SEP = salt.utils.stringutils.to_str('\u2028') self.verbose = verbose self._conn = None @@ -149,17 +129,12 @@ class PlainTextSocketAppender(object): self._conn.close() def put(self, data): - # Replace newlines with Unicode line separator - # for multi-line events - if not _is_unicode(data): - multiline = _create_unicode(data).replace('\n', self.LINE_SEP) - else: - multiline = data.replace('\n', self.LINE_SEP) - multiline += "\n" + # Replace newlines with Unicode line separator for multi-line events + multiline = data.replace('\n', self.LINE_SEP) + str('\n') # future lint: disable=blacklisted-function # Send data, reconnect if needed while True: try: - self._conn.send(multiline.encode('utf-8')) + self._conn.send(multiline) except socket.error: self.reopen_connection() continue @@ -196,14 +171,6 @@ else: SocketAppender = TLSSocketAppender -def _get_appender(endpoint='data.logentries.com', port=10000): - return SocketAppender(verbose=False, LE_API=endpoint, LE_PORT=port) - - -def _emit(token, msg): - return '{0} {1}'.format(token, msg) - - def start(endpoint='data.logentries.com', port=10000, token=None, @@ -230,13 +197,19 @@ def start(endpoint='data.logentries.com', except ValueError: log.warning('Not a valid logentries token') - appender = _get_appender(endpoint, port) + appender = SocketAppender(verbose=False, LE_API=endpoint, LE_PORT=port) appender.reopen_connection() while True: event = event_bus.get_event() if event: - msg = '{0} {1}'.format(tag, json.dumps(event)) - appender.put(_emit(token, msg)) + # future lint: disable=blacklisted-function + msg = str(' ').join(( + salt.utils.stringutils.to_str(token), + salt.utils.stringutils.to_str(tag), + salt.utils.json.dumps(event) + )) + # future lint: enable=blacklisted-function + appender.put(msg) appender.close_connection() diff --git a/salt/engines/slack.py b/salt/engines/slack.py index ca68496d82f..9a36da09308 100644 --- a/salt/engines/slack.py +++ b/salt/engines/slack.py @@ -94,7 +94,6 @@ In addition, other groups are being loaded from pillars. from __future__ import absolute_import import ast import datetime -import json import itertools import logging import time @@ -118,6 +117,7 @@ import salt.runner import salt.utils.args import salt.utils.event import salt.utils.http +import salt.utils.json import salt.utils.slack from salt.utils.yamldumper import OrderedDumper @@ -372,7 +372,7 @@ class SlackClient(object): log.warn('Got a message that I could not log. The reason is: {}'.format(uee)) # Convert UTF to string - _text = json.dumps(_text) + _text = salt.utils.json.dumps(_text) _text = yaml.safe_load(_text) if not _text: @@ -574,7 +574,7 @@ class SlackClient(object): indent=0) try: #return yaml.dump(data, **params).replace("\n\n", "\n") - return json.dumps(data, sort_keys=True, indent=1) + return salt.utils.json.dumps(data, sort_keys=True, indent=1) # pylint: disable=broad-except except Exception as exc: import pprint @@ -626,7 +626,7 @@ class SlackClient(object): # emulate lookup_jid's return, which is just minion:return # pylint is tripping # pylint: disable=missing-whitespace-after-comma - job_data = json.dumps({key:val['return'] for key, val in jid_result.items()}) + job_data = salt.utils.json.dumps({key:val['return'] for key, val in jid_result.items()}) results[jid] = yaml.load(job_data) return results @@ -692,7 +692,7 @@ class SlackClient(object): content=return_text) # Handle unicode return log.debug('Got back {} via the slack client'.format(r)) - resp = yaml.safe_load(json.dumps(r)) + resp = yaml.safe_load(salt.utils.json.dumps(r)) if 'ok' in resp and resp['ok'] is False: this_job['channel'].send_message('Error: {0}'.format(resp['error'])) del outstanding[jid] diff --git a/salt/engines/sqs_events.py b/salt/engines/sqs_events.py index 2120a4230c9..2589ae6c9c0 100644 --- a/salt/engines/sqs_events.py +++ b/salt/engines/sqs_events.py @@ -77,9 +77,9 @@ Additionally you can define cross account sqs: from __future__ import absolute_import import logging import time -import json # Import salt libs +import salt.utils.json import salt.utils.event # Import third party libs @@ -140,7 +140,7 @@ def _process_queue(q, q_name, fire_master, tag='salt/engine/sqs', owner_acct_id= msgs = q.get_messages(wait_time_seconds=20) for msg in msgs: if message_format == "json": - fire_master(tag=tag, data={'message': json.loads(msg.get_body())}) + fire_master(tag=tag, data={'message': salt.utils.json.loads(msg.get_body())}) else: fire_master(tag=tag, data={'message': msg.get_body()}) msg.delete() diff --git a/salt/engines/test.py b/salt/engines/test.py index 161a3193573..8cc29a8c45e 100644 --- a/salt/engines/test.py +++ b/salt/engines/test.py @@ -5,11 +5,11 @@ A simple test engine, not intended for real use but as an example # Import python libs from __future__ import absolute_import -import json import logging # Import salt libs import salt.utils.event +import salt.utils.json log = logging.getLogger(__name__) @@ -34,6 +34,6 @@ def start(): while True: event = event_bus.get_event() - jevent = json.dumps(event) + jevent = salt.utils.json.dumps(event) if event: log.debug(jevent) diff --git a/salt/executors/sudo.py b/salt/executors/sudo.py index 61ada996656..d61c9149ab5 100644 --- a/salt/executors/sudo.py +++ b/salt/executors/sudo.py @@ -4,13 +4,13 @@ Sudo executor module ''' # Import python libs from __future__ import absolute_import -import json try: from shlex import quote as _cmd_quote # pylint: disable=E0611 except ImportError: from pipes import quote as _cmd_quote # Import salt libs +import salt.utils.json import salt.utils.path import salt.syspaths @@ -70,7 +70,7 @@ def execute(opts, data, func, args, kwargs): cmd_ret = __salt__['cmd.run_all'](cmd, use_vt=True, python_shell=False) if cmd_ret['retcode'] == 0: - cmd_meta = json.loads(cmd_ret['stdout'])['local'] + cmd_meta = salt.utils.json.loads(cmd_ret['stdout'])['local'] ret = cmd_meta['return'] __context__['retcode'] = cmd_meta.get('retcode', 0) else: diff --git a/salt/fileserver/azurefs.py b/salt/fileserver/azurefs.py index 7fe855da578..11d4f1d72e5 100644 --- a/salt/fileserver/azurefs.py +++ b/salt/fileserver/azurefs.py @@ -47,10 +47,8 @@ permissions. # Import python libs from __future__ import absolute_import import base64 -import json import logging import os -import os.path import shutil # Import salt libs @@ -58,6 +56,7 @@ import salt.fileserver import salt.utils.files import salt.utils.gzip_util import salt.utils.hashutils +import salt.utils.json import salt.utils.path from salt.utils.versions import LooseVersion @@ -260,7 +259,7 @@ def update(): with salt.utils.files.fopen(lk_fn, 'w+') as fp_: fp_.write('') with salt.utils.files.fopen(container_list, 'w') as fp_: - fp_.write(json.dumps(blob_names)) + salt.utils.json.dump(blob_names, fp_) try: os.unlink(lk_fn) except Exception: @@ -314,7 +313,7 @@ def file_list(load): if not os.path.exists(container_list): continue with salt.utils.files.fopen(container_list, 'r') as fp_: - ret.update(set(json.load(fp_))) + ret.update(set(salt.utils.json.load(fp_))) except Exception as exc: log.error('azurefs: an error ocurred retrieving file lists. ' 'It should be resolved next time the fileserver ' diff --git a/salt/grains/mdata.py b/salt/grains/mdata.py index 9e528e8cfe8..1782d9d92e0 100644 --- a/salt/grains/mdata.py +++ b/salt/grains/mdata.py @@ -14,11 +14,11 @@ from __future__ import absolute_import # Import python libs import os -import json import logging # Import salt libs import salt.utils.dictupdate +import salt.utils.json import salt.utils.path import salt.utils.platform @@ -119,7 +119,7 @@ def _sdc_mdata(mdata_list=None, mdata_get=None): mdata_grain = mdata_grain.replace('-', '_') mdata_grain = mdata_grain.replace(':', '_') if mdata_grain in sdc_json_keys: - grains['mdata']['sdc'][mdata_grain] = json.loads(mdata_value) + grains['mdata']['sdc'][mdata_grain] = salt.utils.json.loads(mdata_value) else: grains['mdata']['sdc'][mdata_grain] = mdata_value diff --git a/salt/grains/metadata.py b/salt/grains/metadata.py index 15ed571a76e..6c220970f84 100644 --- a/salt/grains/metadata.py +++ b/salt/grains/metadata.py @@ -17,12 +17,12 @@ metadata server set `metadata_server_grains: True`. from __future__ import absolute_import # Import python libs -import json import os import socket # Import salt libs import salt.utils.http as http +import salt.utils.json # metadata server information @@ -69,7 +69,7 @@ def _search(prefix="latest/"): # (gtmanfred) This try except block is slightly faster than # checking if the string starts with a curly brace try: - ret[line] = json.loads(retdata) + ret[line] = salt.utils.json.loads(retdata) except ValueError: ret[line] = retdata return ret diff --git a/salt/grains/smartos.py b/salt/grains/smartos.py index f17f7bd89e5..fb39f8a10c8 100644 --- a/salt/grains/smartos.py +++ b/salt/grains/smartos.py @@ -15,11 +15,11 @@ from __future__ import absolute_import # Import python libs import os import re -import json import logging # Import salt libs import salt.utils.dictupdate +import salt.utils.json import salt.utils.path import salt.utils.platform from salt.ext.six.moves import zip @@ -84,7 +84,7 @@ def _smartos_computenode_data(): grains['computenode_vms_type'][vms[vm]['type']] += 1 # sysinfo derived grains - sysinfo = json.loads(__salt__['cmd.run']('sysinfo')) + sysinfo = salt.utils.json.loads(__salt__['cmd.run']('sysinfo')) grains['computenode_sdc_version'] = sysinfo['SDC Version'] grains['computenode_vm_capable'] = sysinfo['VM Capable'] if sysinfo['VM Capable']: diff --git a/salt/key.py b/salt/key.py index 23387d5b180..4163a1274e4 100644 --- a/salt/key.py +++ b/salt/key.py @@ -8,7 +8,6 @@ used to manage salt keys directly without interfacing with the CLI. from __future__ import absolute_import, print_function, unicode_literals import os import copy -import json import stat import shutil import fnmatch @@ -27,6 +26,7 @@ import salt.utils.crypt import salt.utils.data import salt.utils.event import salt.utils.files +import salt.utils.json import salt.utils.kinds import salt.utils.master import salt.utils.sdb @@ -1032,10 +1032,8 @@ class RaetKey(Key): continue path = os.path.join(road_cache, road) with salt.utils.files.fopen(path, 'rb') as fp_: - # Do not use to_unicode to decode this. It needs to stay as - # bytes to be deserialized. if ext == '.json': - data = json.load(fp_) + data = salt.utils.json.load(fp_) elif ext == '.msgpack': data = msgpack.load(fp_) role = salt.utils.stringutils.to_unicode(data['role']) diff --git a/salt/log/handlers/logstash_mod.py b/salt/log/handlers/logstash_mod.py index a00435b5be7..a563868b124 100644 --- a/salt/log/handlers/logstash_mod.py +++ b/salt/log/handlers/logstash_mod.py @@ -158,7 +158,6 @@ # Import python libs from __future__ import absolute_import import os -import json import logging import logging.handlers import datetime @@ -166,6 +165,7 @@ import datetime # Import salt libs from salt.log.setup import LOG_LEVELS from salt.log.mixins import NewStyleClassMixIn +import salt.utils.json import salt.utils.network # Import Third party libs @@ -325,7 +325,7 @@ class LogstashFormatter(logging.Formatter, NewStyleClassMixIn): continue message_dict['@fields'][key] = repr(value) - return json.dumps(message_dict) + return salt.utils.json.dumps(message_dict) def format_v1(self, record): message_dict = { @@ -369,7 +369,7 @@ class LogstashFormatter(logging.Formatter, NewStyleClassMixIn): continue message_dict[key] = repr(value) - return json.dumps(message_dict) + return salt.utils.json.dumps(message_dict) class DatagramLogstashHandler(logging.handlers.DatagramHandler): diff --git a/salt/log/setup.py b/salt/log/setup.py index 9dc1966b30c..3aef690dad3 100644 --- a/salt/log/setup.py +++ b/salt/log/setup.py @@ -338,14 +338,15 @@ class SaltLoggingClass(six.with_metaclass(LoggingMixInMeta, LOGGING_LOGGER_CLASS extra = None # Let's try to make every logging message unicode + salt_system_encoding = __salt_system_encoding__ + if salt_system_encoding == 'ascii': + # Encoding detection most likely failed, let's use the utf-8 + # value which we defaulted before __salt_system_encoding__ was + # implemented + salt_system_encoding = 'utf-8' + if isinstance(msg, six.string_types) \ and not isinstance(msg, six.text_type): - salt_system_encoding = __salt_system_encoding__ - if salt_system_encoding == 'ascii': - # Encoding detection most likely failed, let's use the utf-8 - # value which we defaulted before __salt_system_encoding__ was - # implemented - salt_system_encoding = 'utf-8' try: _msg = msg.decode(salt_system_encoding, 'replace') except UnicodeDecodeError: @@ -353,11 +354,23 @@ class SaltLoggingClass(six.with_metaclass(LoggingMixInMeta, LOGGING_LOGGER_CLASS else: _msg = msg + _args = [] + for item in args: + if isinstance(item, six.string_types) \ + and not isinstance(item, six.text_type): + try: + _args.append(item.decode(salt_system_encoding, 'replace')) + except UnicodeDecodeError: + _args.append(item.decode(salt_system_encoding, 'ignore')) + else: + _args.append(item) + _args = tuple(_args) + if six.PY3: - logrecord = _LOG_RECORD_FACTORY(name, level, fn, lno, _msg, args, + logrecord = _LOG_RECORD_FACTORY(name, level, fn, lno, _msg, _args, exc_info, func, sinfo) else: - logrecord = _LOG_RECORD_FACTORY(name, level, fn, lno, _msg, args, + logrecord = _LOG_RECORD_FACTORY(name, level, fn, lno, _msg, _args, exc_info, func) if extra is not None: diff --git a/salt/modules/ansiblegate.py b/salt/modules/ansiblegate.py index 8e59d61402a..16687c31142 100644 --- a/salt/modules/ansiblegate.py +++ b/salt/modules/ansiblegate.py @@ -36,8 +36,8 @@ import importlib import yaml import fnmatch import subprocess -import json +import salt.utils.json from salt.exceptions import LoaderError, CommandExecutionError from salt.utils import timed_subprocess @@ -150,7 +150,8 @@ class AnsibleModuleCaller(object): ''))) if args: kwargs['_raw_params'] = ' '.join(args) - js_args = '{{"ANSIBLE_MODULE_ARGS": {args}}}'.format(args=json.dumps(kwargs)) + js_args = str('{{"ANSIBLE_MODULE_ARGS": {args}}}') # future lint: disable=blacklisted-function + js_args = js_args.format(args=salt.utils.json.dumps(kwargs)) proc_out = timed_subprocess.TimedProc(["echo", "{0}".format(js_args)], stdout=subprocess.PIPE, timeout=self.timeout) @@ -160,7 +161,7 @@ class AnsibleModuleCaller(object): proc_exc.run() try: - out = json.loads(proc_exc.stdout) + out = salt.utils.json.loads(proc_exc.stdout) except ValueError as ex: out = {'Error': (proc_exc.stderr and (proc_exc.stderr + '.') or str(ex))} if proc_exc.stdout: diff --git a/salt/modules/aptly.py b/salt/modules/aptly.py index ab6acdd029e..82227f09a10 100644 --- a/salt/modules/aptly.py +++ b/salt/modules/aptly.py @@ -6,13 +6,13 @@ Aptly Debian repository manager. ''' # Import python libs from __future__ import absolute_import -import json import logging import os import re # Import salt libs from salt.exceptions import SaltInvocationError +import salt.utils.json import salt.utils.path import salt.utils.stringutils as stringutils @@ -123,7 +123,7 @@ def get_config(config_path=_DEFAULT_CONFIG_PATH): cmd_ret = _cmd_run(cmd) - return json.loads(cmd_ret) + return salt.utils.json.loads(cmd_ret) def list_repos(config_path=_DEFAULT_CONFIG_PATH, with_packages=False): diff --git a/salt/modules/aptpkg.py b/salt/modules/aptpkg.py index 5fd5f4e9f75..4224d5b55e7 100644 --- a/salt/modules/aptpkg.py +++ b/salt/modules/aptpkg.py @@ -23,7 +23,6 @@ import os import re import logging import time -import json # Import third party libs import yaml @@ -42,6 +41,7 @@ import salt.utils.data import salt.utils.files import salt.utils.functools import salt.utils.itertools +import salt.utils.json import salt.utils.path import salt.utils.pkg import salt.utils.pkg.deb @@ -137,7 +137,7 @@ def _get_ppa_info_from_launchpad(owner_name, ppa_name): owner_name, ppa_name) request = _Request(lp_url, headers={'Accept': 'application/json'}) lp_page = _urlopen(request) - return json.load(lp_page) + return salt.utils.json.load(lp_page) def _reconstruct_ppa_name(owner_name, ppa_name): diff --git a/salt/modules/aws_sqs.py b/salt/modules/aws_sqs.py index 467912d5e59..10dc7ab0fd6 100644 --- a/salt/modules/aws_sqs.py +++ b/salt/modules/aws_sqs.py @@ -6,9 +6,9 @@ Support for the Amazon Simple Queue Service. # Import Python libs from __future__ import absolute_import import logging -import json # Import salt libs +import salt.utils.json import salt.utils.path from salt.ext import six @@ -65,7 +65,7 @@ def _run_aws(cmd, region, opts, user, **kwargs): rtn = __salt__['cmd.run'](cmd, runas=user, python_shell=False) - return json.loads(rtn) if rtn else '' + return salt.utils.json.loads(rtn) if rtn else '' def receive_message(queue, region, num=1, opts=None, user=None): diff --git a/salt/modules/bigip.py b/salt/modules/bigip.py index bc3427ac061..e26c3f3825d 100644 --- a/salt/modules/bigip.py +++ b/salt/modules/bigip.py @@ -7,7 +7,7 @@ An execution module which can manipulate an f5 bigip via iControl REST # Import python libs from __future__ import absolute_import -import json +import salt.utils.json import logging as logger # Import third party libs @@ -72,7 +72,7 @@ def _load_response(response): ''' try: - data = json.loads(response.text) + data = salt.utils.json.loads(response.text) except ValueError: data = response.text @@ -181,7 +181,7 @@ def _set_value(value): value = value.replace('}j', '}') try: - return json.loads(value) + return salt.utils.json.loads(value) except Exception: raise salt.exceptions.CommandExecutionError @@ -252,7 +252,10 @@ def start_transaction(hostname, username, password, label): #post to REST to get trans id try: - response = bigip_session.post(BIG_IP_URL_BASE.format(host=hostname)+'/transaction', data=json.dumps(payload)) + response = bigip_session.post( + BIG_IP_URL_BASE.format(host=hostname) + '/transaction', + data=salt.utils.json.dumps(payload) + ) except requests.exceptions.ConnectionError as e: return _load_connection_error(hostname, e) @@ -342,7 +345,10 @@ def commit_transaction(hostname, username, password, label): #patch to REST to get trans id try: - response = bigip_session.patch(BIG_IP_URL_BASE.format(host=hostname)+'/transaction/{trans_id}'.format(trans_id=trans_id), data=json.dumps(payload)) + response = bigip_session.patch( + BIG_IP_URL_BASE.format(host=hostname) + '/transaction/{trans_id}'.format(trans_id=trans_id), + data=salt.utils.json.dumps(payload) + ) return _load_response(response) except requests.exceptions.ConnectionError as e: return _load_connection_error(hostname, e) @@ -460,7 +466,9 @@ def create_node(hostname, username, password, name, address, trans_label=None): #post to REST try: - response = bigip_session.post(BIG_IP_URL_BASE.format(host=hostname)+'/ltm/node', data=json.dumps(payload)) + response = bigip_session.post( + BIG_IP_URL_BASE.format(host=hostname) + '/ltm/node', + data=salt.utils.json.dumps(payload)) except requests.exceptions.ConnectionError as e: return _load_connection_error(hostname, e) @@ -537,7 +545,10 @@ def modify_node(hostname, username, password, name, #put to REST try: - response = bigip_session.put(BIG_IP_URL_BASE.format(host=hostname)+'/ltm/node/{name}'.format(name=name), data=json.dumps(payload)) + response = bigip_session.put( + BIG_IP_URL_BASE.format(host=hostname) + '/ltm/node/{name}'.format(name=name), + data=salt.utils.json.dumps(payload) + ) except requests.exceptions.ConnectionError as e: return _load_connection_error(hostname, e) @@ -758,7 +769,10 @@ def create_pool(hostname, username, password, name, members=None, #post to REST try: - response = bigip_session.post(BIG_IP_URL_BASE.format(host=hostname)+'/ltm/pool', data=json.dumps(payload)) + response = bigip_session.post( + BIG_IP_URL_BASE.format(host=hostname) + '/ltm/pool', + data=salt.utils.json.dumps(payload) + ) except requests.exceptions.ConnectionError as e: return _load_connection_error(hostname, e) @@ -902,7 +916,10 @@ def modify_pool(hostname, username, password, name, #post to REST try: - response = bigip_session.put(BIG_IP_URL_BASE.format(host=hostname)+'/ltm/pool/{name}'.format(name=name), data=json.dumps(payload)) + response = bigip_session.put( + BIG_IP_URL_BASE.format(host=hostname) + '/ltm/pool/{name}'.format(name=name), + data=salt.utils.json.dumps(payload) + ) except requests.exceptions.ConnectionError as e: return _load_connection_error(hostname, e) @@ -999,7 +1016,10 @@ def replace_pool_members(hostname, username, password, name, members): #put to REST try: - response = bigip_session.put(BIG_IP_URL_BASE.format(host=hostname)+'/ltm/pool/{name}'.format(name=name), data=json.dumps(payload)) + response = bigip_session.put( + BIG_IP_URL_BASE.format(host=hostname) + '/ltm/pool/{name}'.format(name=name), + data=salt.utils.json.dumps(payload) + ) except requests.exceptions.ConnectionError as e: return _load_connection_error(hostname, e) @@ -1052,7 +1072,10 @@ def add_pool_member(hostname, username, password, name, member): #post to REST try: - response = bigip_session.post(BIG_IP_URL_BASE.format(host=hostname)+'/ltm/pool/{name}/members'.format(name=name), data=json.dumps(payload)) + response = bigip_session.post( + BIG_IP_URL_BASE.format(host=hostname) + '/ltm/pool/{name}/members'.format(name=name), + data=salt.utils.json.dumps(payload) + ) except requests.exceptions.ConnectionError as e: return _load_connection_error(hostname, e) @@ -1138,7 +1161,10 @@ def modify_pool_member(hostname, username, password, name, member, #put to REST try: - response = bigip_session.put(BIG_IP_URL_BASE.format(host=hostname)+'/ltm/pool/{name}/members/{member}'.format(name=name, member=member), data=json.dumps(payload)) + response = bigip_session.put( + BIG_IP_URL_BASE.format(host=hostname) + '/ltm/pool/{name}/members/{member}'.format(name=name, member=member), + data=salt.utils.json.dumps(payload) + ) except requests.exceptions.ConnectionError as e: return _load_connection_error(hostname, e) @@ -1493,7 +1519,10 @@ def create_virtual(hostname, username, password, name, destination, #post to REST try: - response = bigip_session.post(BIG_IP_URL_BASE.format(host=hostname)+'/ltm/virtual', data=json.dumps(payload)) + response = bigip_session.post( + BIG_IP_URL_BASE.format(host=hostname) + '/ltm/virtual', + data=salt.utils.json.dumps(payload) + ) except requests.exceptions.ConnectionError as e: return _load_connection_error(hostname, e) @@ -1762,7 +1791,10 @@ def modify_virtual(hostname, username, password, name, #put to REST try: - response = bigip_session.put(BIG_IP_URL_BASE.format(host=hostname)+'/ltm/virtual/{name}'.format(name=name), data=json.dumps(payload)) + response = bigip_session.put( + BIG_IP_URL_BASE.format(host=hostname) + '/ltm/virtual/{name}'.format(name=name), + data=salt.utils.json.dumps(payload) + ) except requests.exceptions.ConnectionError as e: return _load_connection_error(hostname, e) @@ -1879,7 +1911,10 @@ def create_monitor(hostname, username, password, monitor_type, name, **kwargs): #post to REST try: - response = bigip_session.post(BIG_IP_URL_BASE.format(host=hostname)+'/ltm/monitor/{type}'.format(type=monitor_type), data=json.dumps(payload)) + response = bigip_session.post( + BIG_IP_URL_BASE.format(host=hostname) + '/ltm/monitor/{type}'.format(type=monitor_type), + data=salt.utils.json.dumps(payload) + ) except requests.exceptions.ConnectionError as e: return _load_connection_error(hostname, e) @@ -1926,7 +1961,10 @@ def modify_monitor(hostname, username, password, monitor_type, name, **kwargs): #put to REST try: - response = bigip_session.put(BIG_IP_URL_BASE.format(host=hostname)+'/ltm/monitor/{type}/{name}'.format(type=monitor_type, name=name), data=json.dumps(payload)) + response = bigip_session.put( + BIG_IP_URL_BASE.format(host=hostname) + '/ltm/monitor/{type}/{name}'.format(type=monitor_type, name=name), + data=salt.utils.json.dumps(payload) + ) except requests.exceptions.ConnectionError as e: return _load_connection_error(hostname, e) @@ -2077,7 +2115,10 @@ def create_profile(hostname, username, password, profile_type, name, **kwargs): #post to REST try: - response = bigip_session.post(BIG_IP_URL_BASE.format(host=hostname)+'/ltm/profile/{type}'.format(type=profile_type), data=json.dumps(payload)) + response = bigip_session.post( + BIG_IP_URL_BASE.format(host=hostname) + '/ltm/profile/{type}'.format(type=profile_type), + data=salt.utils.json.dumps(payload) + ) except requests.exceptions.ConnectionError as e: return _load_connection_error(hostname, e) @@ -2162,7 +2203,10 @@ def modify_profile(hostname, username, password, profile_type, name, **kwargs): #put to REST try: - response = bigip_session.put(BIG_IP_URL_BASE.format(host=hostname)+'/ltm/profile/{type}/{name}'.format(type=profile_type, name=name), data=json.dumps(payload)) + response = bigip_session.put( + BIG_IP_URL_BASE.format(host=hostname) + '/ltm/profile/{type}/{name}'.format(type=profile_type, name=name), + data=salt.utils.json.dumps(payload) + ) except requests.exceptions.ConnectionError as e: return _load_connection_error(hostname, e) diff --git a/salt/modules/boto_apigateway.py b/salt/modules/boto_apigateway.py index dc14e9e8448..a5c9c56fd1b 100644 --- a/salt/modules/boto_apigateway.py +++ b/salt/modules/boto_apigateway.py @@ -78,13 +78,13 @@ Connection module for Amazon APIGateway # Import Python libs from __future__ import absolute_import import logging -import json import datetime # Import Salt libs from salt.ext import six import salt.utils.boto3 import salt.utils.compat +import salt.utils.json from salt.utils.versions import LooseVersion as _LooseVersion log = logging.getLogger(__name__) @@ -1161,7 +1161,7 @@ def update_api_model_schema(restApiId, modelName, schema, region=None, key=None, ''' try: - schema_json = json.dumps(schema) if isinstance(schema, dict) else schema + schema_json = salt.utils.json.dumps(schema) if isinstance(schema, dict) else schema conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) response = _api_model_patch_replace(conn, restApiId, modelName, '/schema', schema_json) return {'updated': True, 'model': _convert_datetime_str(response)} @@ -1202,7 +1202,7 @@ def create_api_model(restApiId, modelName, modelDescription, schema, contentType ''' try: - schema_json = json.dumps(schema) if isinstance(schema, dict) else schema + schema_json = salt.utils.json.dumps(schema) if isinstance(schema, dict) else schema conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) model = conn.create_model(restApiId=restApiId, name=modelName, description=modelDescription, schema=schema_json, contentType=contentType) diff --git a/salt/modules/boto_asg.py b/salt/modules/boto_asg.py index 4a5f365e1d8..3795a5a6703 100644 --- a/salt/modules/boto_asg.py +++ b/salt/modules/boto_asg.py @@ -50,7 +50,6 @@ from __future__ import absolute_import import datetime import time import logging -import json import sys import time import email.mime.multipart @@ -79,6 +78,7 @@ except ImportError: # Import Salt libs import salt.utils.boto3 import salt.utils.compat +import salt.utils.json import salt.utils.odict as odict @@ -235,13 +235,13 @@ def create(name, launch_config_name, availability_zones, min_size, max_size, ''' conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) if isinstance(availability_zones, six.string_types): - availability_zones = json.loads(availability_zones) + availability_zones = salt.utils.json.loads(availability_zones) if isinstance(load_balancers, six.string_types): - load_balancers = json.loads(load_balancers) + load_balancers = salt.utils.json.loads(load_balancers) if isinstance(vpc_zone_identifier, six.string_types): - vpc_zone_identifier = json.loads(vpc_zone_identifier) + vpc_zone_identifier = salt.utils.json.loads(vpc_zone_identifier) if isinstance(tags, six.string_types): - tags = json.loads(tags) + tags = salt.utils.json.loads(tags) # Make a list of tag objects from the dict. _tags = [] if tags: @@ -261,11 +261,11 @@ def create(name, launch_config_name, availability_zones, min_size, max_size, propagate_at_launch=propagate_at_launch) _tags.append(_tag) if isinstance(termination_policies, six.string_types): - termination_policies = json.loads(termination_policies) + termination_policies = salt.utils.json.loads(termination_policies) if isinstance(suspended_processes, six.string_types): - suspended_processes = json.loads(suspended_processes) + suspended_processes = salt.utils.json.loads(suspended_processes) if isinstance(scheduled_actions, six.string_types): - scheduled_actions = json.loads(scheduled_actions) + scheduled_actions = salt.utils.json.loads(scheduled_actions) retries = 30 while True: try: @@ -325,19 +325,19 @@ def update(name, launch_config_name, availability_zones, min_size, max_size, if not conn: return False, "failed to connect to AWS" if isinstance(availability_zones, six.string_types): - availability_zones = json.loads(availability_zones) + availability_zones = salt.utils.json.loads(availability_zones) if isinstance(load_balancers, six.string_types): - load_balancers = json.loads(load_balancers) + load_balancers = salt.utils.json.loads(load_balancers) if isinstance(vpc_zone_identifier, six.string_types): - vpc_zone_identifier = json.loads(vpc_zone_identifier) + vpc_zone_identifier = salt.utils.json.loads(vpc_zone_identifier) if isinstance(tags, six.string_types): - tags = json.loads(tags) + tags = salt.utils.json.loads(tags) if isinstance(termination_policies, six.string_types): - termination_policies = json.loads(termination_policies) + termination_policies = salt.utils.json.loads(termination_policies) if isinstance(suspended_processes, six.string_types): - suspended_processes = json.loads(suspended_processes) + suspended_processes = salt.utils.json.loads(suspended_processes) if isinstance(scheduled_actions, six.string_types): - scheduled_actions = json.loads(scheduled_actions) + scheduled_actions = salt.utils.json.loads(scheduled_actions) # Massage our tagset into add / remove lists # Use a boto3 call here b/c the boto2 call doeesn't implement filters @@ -510,7 +510,7 @@ def get_cloud_init_mime(cloud_init): salt myminion boto.get_cloud_init_mime ''' if isinstance(cloud_init, six.string_types): - cloud_init = json.loads(cloud_init) + cloud_init = salt.utils.json.loads(cloud_init) _cloud_init = email.mime.multipart.MIMEMultipart() if 'boothooks' in cloud_init: for script_name, script in six.iteritems(cloud_init['boothooks']): @@ -655,9 +655,9 @@ def create_launch_configuration(name, image_id, key_name=None, ''' conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) if isinstance(security_groups, six.string_types): - security_groups = json.loads(security_groups) + security_groups = salt.utils.json.loads(security_groups) if isinstance(block_device_mappings, six.string_types): - block_device_mappings = json.loads(block_device_mappings) + block_device_mappings = salt.utils.json.loads(block_device_mappings) _bdms = [] if block_device_mappings: # Boto requires objects for the mappings and the devices. diff --git a/salt/modules/boto_cloudwatch.py b/salt/modules/boto_cloudwatch.py index 88d1dcd54d8..f8e6f4b1bcb 100644 --- a/salt/modules/boto_cloudwatch.py +++ b/salt/modules/boto_cloudwatch.py @@ -48,9 +48,9 @@ from __future__ import absolute_import # Import Python libs import logging -import json import yaml +import salt.utils.json import salt.utils.odict as odict log = logging.getLogger(__name__) @@ -66,7 +66,7 @@ try: except ImportError: HAS_BOTO = False -from salt.ext.six import string_types +from salt.ext import six def __virtual__(): @@ -220,16 +220,16 @@ def create_or_update_alarm( period = int(period) if evaluation_periods: evaluation_periods = int(evaluation_periods) - if isinstance(dimensions, string_types): - dimensions = json.loads(dimensions) + if isinstance(dimensions, six.string_types): + dimensions = salt.utils.json.loads(dimensions) if not isinstance(dimensions, dict): log.error("could not parse dimensions argument: must be json encoding of a dict: '{0}'".format(dimensions)) return False - if isinstance(alarm_actions, string_types): + if isinstance(alarm_actions, six.string_types): alarm_actions = alarm_actions.split(",") - if isinstance(insufficient_data_actions, string_types): + if isinstance(insufficient_data_actions, six.string_types): insufficient_data_actions = insufficient_data_actions.split(",") - if isinstance(ok_actions, string_types): + if isinstance(ok_actions, six.string_types): ok_actions = ok_actions.split(",") # convert provided action names into ARN's diff --git a/salt/modules/boto_cloudwatch_event.py b/salt/modules/boto_cloudwatch_event.py index 9eb49e8e92b..66d87796f51 100644 --- a/salt/modules/boto_cloudwatch_event.py +++ b/salt/modules/boto_cloudwatch_event.py @@ -48,7 +48,7 @@ from __future__ import absolute_import # Import Python libs import logging -import json +import salt.utils.json import salt.utils.compat log = logging.getLogger(__name__) @@ -67,7 +67,7 @@ except ImportError as e: HAS_BOTO = False # pylint: enable=import-error -from salt.ext.six import string_types +from salt.ext import six def __virtual__(): @@ -281,8 +281,8 @@ def put_targets(Rule, Targets, ''' try: conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) - if isinstance(Targets, string_types): - Targets = json.loads(Targets) + if isinstance(Targets, six.string_types): + Targets = salt.utils.json.loads(Targets) failures = conn.put_targets(Rule=Rule, Targets=Targets) if failures and failures.get('FailedEntryCount', 0) > 0: return {'failures': failures.get('FailedEntries')} @@ -311,8 +311,8 @@ def remove_targets(Rule, Ids, ''' try: conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) - if isinstance(Ids, string_types): - Ids = json.loads(Ids) + if isinstance(Ids, six.string_types): + Ids = salt.utils.json.loads(Ids) failures = conn.remove_targets(Rule=Rule, Ids=Ids) if failures and failures.get('FailedEntryCount', 0) > 0: return {'failures': failures.get('FailedEntries', 1)} diff --git a/salt/modules/boto_ec2.py b/salt/modules/boto_ec2.py index bbaa6c56843..3b4f74ea3f7 100644 --- a/salt/modules/boto_ec2.py +++ b/salt/modules/boto_ec2.py @@ -49,11 +49,11 @@ Connection module for Amazon EC2 from __future__ import absolute_import import logging import time -import json # Import Salt libs import salt.utils.compat import salt.utils.data +import salt.utils.json from salt.ext import six from salt.exceptions import SaltInvocationError, CommandExecutionError from salt.utils.versions import LooseVersion as _LooseVersion @@ -822,7 +822,7 @@ def _to_blockdev_map(thing): if isinstance(thing, BlockDeviceMapping): return thing if isinstance(thing, six.string_types): - thing = json.loads(thing) + thing = salt.utils.json.loads(thing) if not isinstance(thing, dict): log.error("Can't convert '{0}' of type {1} to a " "boto.ec2.blockdevicemapping.BlockDeviceMapping".format(thing, type(thing))) diff --git a/salt/modules/boto_elasticsearch_domain.py b/salt/modules/boto_elasticsearch_domain.py index 68e6c628fb7..708bea82ec9 100644 --- a/salt/modules/boto_elasticsearch_domain.py +++ b/salt/modules/boto_elasticsearch_domain.py @@ -77,12 +77,12 @@ Connection module for Amazon Elasticsearch Service # Import Python libs from __future__ import absolute_import import logging -import json # Import Salt libs from salt.ext import six import salt.utils.boto3 import salt.utils.compat +import salt.utils.json from salt.exceptions import SaltInvocationError from salt.utils.versions import LooseVersion as _LooseVersion @@ -256,12 +256,12 @@ def create(DomainName, ElasticsearchClusterConfig=None, EBSOptions=None, val = locals()[k] if isinstance(val, six.string_types): try: - val = json.loads(val) + val = salt.utils.json.loads(val) except ValueError as e: return {'updated': False, 'error': 'Error parsing {0}: {1}'.format(k, e.message)} kwargs[k] = val if 'AccessPolicies' in kwargs: - kwargs['AccessPolicies'] = json.dumps(kwargs['AccessPolicies']) + kwargs['AccessPolicies'] = salt.utils.json.dumps(kwargs['AccessPolicies']) if 'ElasticsearchVersion' in kwargs: kwargs['ElasticsearchVersion'] = str(kwargs['ElasticsearchVersion']) domain = conn.create_elasticsearch_domain(DomainName=DomainName, **kwargs) @@ -330,12 +330,12 @@ def update(DomainName, ElasticsearchClusterConfig=None, EBSOptions=None, val = locals()[k] if isinstance(val, six.string_types): try: - val = json.loads(val) + val = salt.utils.json.loads(val) except ValueError as e: return {'updated': False, 'error': 'Error parsing {0}: {1}'.format(k, e.message)} call_args[k] = val if 'AccessPolicies' in call_args: - call_args['AccessPolicies'] = json.dumps(call_args['AccessPolicies']) + call_args['AccessPolicies'] = salt.utils.json.dumps(call_args['AccessPolicies']) try: conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) domain = conn.update_elasticsearch_domain_config(DomainName=DomainName, **call_args) diff --git a/salt/modules/boto_elb.py b/salt/modules/boto_elb.py index 4df635ab2dd..379acbfd655 100644 --- a/salt/modules/boto_elb.py +++ b/salt/modules/boto_elb.py @@ -48,18 +48,17 @@ from __future__ import absolute_import # Import Python libs import logging -import json import time log = logging.getLogger(__name__) # Import Salt libs +import salt.utils.json import salt.utils.odict as odict from salt.utils.versions import LooseVersion as _LooseVersion # Import third party libs from salt.ext import six -from salt.ext.six import string_types try: import boto import boto.ec2 # pylint: enable=unused-import @@ -254,11 +253,11 @@ def create(name, availability_zones, listeners, subnets=None, conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) if exists(name, region, key, keyid, profile): return True - if isinstance(availability_zones, string_types): - availability_zones = json.loads(availability_zones) + if isinstance(availability_zones, six.string_types): + availability_zones = salt.utils.json.loads(availability_zones) - if isinstance(listeners, string_types): - listeners = json.loads(listeners) + if isinstance(listeners, six.string_types): + listeners = salt.utils.json.loads(listeners) _complex_listeners = [] for listener in listeners: @@ -321,8 +320,8 @@ def create_listeners(name, listeners, region=None, key=None, keyid=None, ''' conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) - if isinstance(listeners, string_types): - listeners = json.loads(listeners) + if isinstance(listeners, six.string_types): + listeners = salt.utils.json.loads(listeners) _complex_listeners = [] for listener in listeners: @@ -352,8 +351,8 @@ def delete_listeners(name, ports, region=None, key=None, keyid=None, ''' conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) - if isinstance(ports, string_types): - ports = json.loads(ports) + if isinstance(ports, six.string_types): + ports = salt.utils.json.loads(ports) try: conn.delete_load_balancer_listeners(name, ports) msg = 'Deleted ELB listeners on {0}'.format(name) @@ -379,8 +378,8 @@ def apply_security_groups(name, security_groups, region=None, key=None, ''' conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) - if isinstance(security_groups, string_types): - security_groups = json.loads(security_groups) + if isinstance(security_groups, six.string_types): + security_groups = salt.utils.json.loads(security_groups) try: conn.apply_security_groups_to_lb(name, security_groups) msg = 'Applied security_groups on ELB {0}'.format(name) @@ -407,8 +406,8 @@ def enable_availability_zones(name, availability_zones, region=None, key=None, ''' conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) - if isinstance(availability_zones, string_types): - availability_zones = json.loads(availability_zones) + if isinstance(availability_zones, six.string_types): + availability_zones = salt.utils.json.loads(availability_zones) try: conn.enable_availability_zones(name, availability_zones) msg = 'Enabled availability_zones on ELB {0}'.format(name) @@ -434,8 +433,8 @@ def disable_availability_zones(name, availability_zones, region=None, key=None, ''' conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) - if isinstance(availability_zones, string_types): - availability_zones = json.loads(availability_zones) + if isinstance(availability_zones, six.string_types): + availability_zones = salt.utils.json.loads(availability_zones) try: conn.disable_availability_zones(name, availability_zones) msg = 'Disabled availability_zones on ELB {0}'.format(name) @@ -461,8 +460,8 @@ def attach_subnets(name, subnets, region=None, key=None, keyid=None, ''' conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) - if isinstance(subnets, string_types): - subnets = json.loads(subnets) + if isinstance(subnets, six.string_types): + subnets = salt.utils.json.loads(subnets) try: conn.attach_lb_to_subnets(name, subnets) msg = 'Attached ELB {0} on subnets.'.format(name) @@ -488,8 +487,8 @@ def detach_subnets(name, subnets, region=None, key=None, keyid=None, ''' conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) - if isinstance(subnets, string_types): - subnets = json.loads(subnets) + if isinstance(subnets, six.string_types): + subnets = salt.utils.json.loads(subnets) try: conn.detach_lb_from_subnets(name, subnets) msg = 'Detached ELB {0} from subnets.'.format(name) diff --git a/salt/modules/boto_iam.py b/salt/modules/boto_iam.py index 5e1ba2d03db..ff2f93e0a31 100644 --- a/salt/modules/boto_iam.py +++ b/salt/modules/boto_iam.py @@ -38,15 +38,15 @@ Connection module for Amazon IAM #pylint: disable=E0602 # Import Python libs -from __future__ import absolute_import +from __future__ import absolute_import, print_function, unicode_literals import logging -import json import yaml import time # Import salt libs from salt.ext import six import salt.utils.compat +import salt.utils.json import salt.utils.odict as odict # Import third party libs @@ -123,11 +123,10 @@ def create_instance_profile(name, region=None, key=None, keyid=None, # This call returns an instance profile if successful and an exception # if not. It's annoying. conn.create_instance_profile(name) - log.info('Created {0} instance profile.'.format(name)) + log.info('Created %s instance profile.', name) except boto.exception.BotoServerError as e: log.debug(e) - msg = 'Failed to create {0} instance profile.' - log.error(msg.format(name)) + log.error('Failed to create %s instance profile.', name) return False return True @@ -149,11 +148,10 @@ def delete_instance_profile(name, region=None, key=None, keyid=None, return True try: conn.delete_instance_profile(name) - log.info('Deleted {0} instance profile.'.format(name)) + log.info('Deleted %s instance profile.', name) except boto.exception.BotoServerError as e: log.debug(e) - msg = 'Failed to delete {0} instance profile.' - log.error(msg.format(name)) + log.error('Failed to delete %s instance profile.', name) return False return True @@ -192,7 +190,7 @@ def describe_role(name, region=None, key=None, keyid=None, profile=None): if not info: return False role = info.get_role_response.get_role_result.role - role['assume_role_policy_document'] = json.loads(_unquote( + role['assume_role_policy_document'] = salt.utils.json.loads(_unquote( role.assume_role_policy_document )) # If Sid wasn't defined by the user, boto will still return a Sid in @@ -206,8 +204,7 @@ def describe_role(name, region=None, key=None, keyid=None, profile=None): return role except boto.exception.BotoServerError as e: log.debug(e) - msg = 'Failed to get {0} information.' - log.error(msg.format(name)) + log.error('Failed to get %s information.', name) return False @@ -231,12 +228,11 @@ def create_user(user_name, path=None, region=None, key=None, keyid=None, conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) try: conn.create_user(user_name, path) - log.info('Created user : {0}.'.format(user_name)) + log.info('Created IAM user : %s.', user_name) return True except boto.exception.BotoServerError as e: log.debug(e) - msg = 'Failed to create user {0}.' - log.error(msg.format(user_name)) + log.error('Failed to create IAM user %s.', user_name) return False @@ -258,8 +254,8 @@ def get_all_access_keys(user_name, marker=None, max_items=None, return conn.get_all_access_keys(user_name, marker, max_items) except boto.exception.BotoServerError as e: log.debug(e) - log.error('Failed to get user\'s {0} access keys.'.format(user_name)) - return str(e) + log.error('Failed to get access keys for IAM user %s.', user_name) + return six.text_type(e) def create_access_key(user_name, region=None, key=None, keyid=None, profile=None): @@ -280,7 +276,7 @@ def create_access_key(user_name, region=None, key=None, keyid=None, profile=None except boto.exception.BotoServerError as e: log.debug(e) log.error('Failed to create access key.') - return str(e) + return six.text_type(e) def delete_access_key(access_key_id, user_name=None, region=None, key=None, @@ -301,8 +297,8 @@ def delete_access_key(access_key_id, user_name=None, region=None, key=None, return conn.delete_access_key(access_key_id, user_name) except boto.exception.BotoServerError as e: log.debug(e) - log.error('Failed to delete access key id {0}.'.format(access_key_id)) - return str(e) + log.error('Failed to delete access key id %s.', access_key_id) + return six.text_type(e) def delete_user(user_name, region=None, key=None, keyid=None, @@ -323,12 +319,12 @@ def delete_user(user_name, region=None, key=None, keyid=None, conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) try: conn.delete_user(user_name) - log.info('Deleted user : {0} .'.format(user_name)) + log.info('Deleted IAM user : %s .', user_name) return True except boto.exception.BotoServerError as e: log.debug(e) - log.error('Failed to delete user {0}'.format(user_name)) - return str(e) + log.error('Failed to delete IAM user %s', user_name) + return six.text_type(e) def get_user(user_name=None, region=None, key=None, keyid=None, profile=None): @@ -351,8 +347,7 @@ def get_user(user_name=None, region=None, key=None, keyid=None, profile=None): return info except boto.exception.BotoServerError as e: log.debug(e) - msg = 'Failed to get user {0} info.' - log.error(msg.format(user_name)) + log.error('Failed to get IAM user %s info.', user_name) return False @@ -377,12 +372,11 @@ def create_group(group_name, path=None, region=None, key=None, keyid=None, conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) try: conn.create_group(group_name, path) - log.info('Created group : {0}.'.format(group_name)) + log.info('Created IAM group : %s.', group_name) return True except boto.exception.BotoServerError as e: log.debug(e) - msg = 'Failed to create group {0}.' - log.error(msg.format(group_name)) + log.error('Failed to create IAM group %s.', group_name) return False @@ -406,8 +400,7 @@ def get_group(group_name, region=None, key=None, keyid=None, profile=None): return info['get_group_response']['get_group_result']['group'] except boto.exception.BotoServerError as e: log.debug(e) - msg = 'Failed to get group {0} info.' - log.error(msg.format(group_name)) + log.error('Failed to get IAM group %s info.', group_name) return False @@ -442,8 +435,7 @@ def get_group_members(group_name, region=None, key=None, keyid=None, profile=Non return users except boto.exception.BotoServerError as e: log.debug(e) - msg = 'Failed to get group {0} members.' - log.error(msg.format(group_name)) + log.error('Failed to get members for IAM group %s.', group_name) return False @@ -462,8 +454,7 @@ def add_user_to_group(user_name, group_name, region=None, key=None, keyid=None, ''' user = get_user(user_name, region, key, keyid, profile) if not user: - msg = 'Username : {0} does not exist.' - log.error(msg.format(user_name, group_name)) + log.error('Username : %s does not exist.', user_name) return False if user_exists_in_group(user_name, group_name, region=region, key=key, keyid=keyid, profile=profile): @@ -476,8 +467,7 @@ def add_user_to_group(user_name, group_name, region=None, key=None, keyid=None, return info except boto.exception.BotoServerError as e: log.debug(e) - msg = 'Failed to add user {0} to group {1}.' - log.error(msg.format(user_name, group_name)) + log.error('Failed to add IAM user %s to group %s.', user_name, group_name) return False @@ -502,8 +492,7 @@ def user_exists_in_group(user_name, group_name, region=None, key=None, keyid=Non if users: for _user in users: if user_name == _user['user_name']: - msg = 'Username : {0} is already in group {1}.' - log.info(msg.format(user_name, group_name)) + log.debug('IAM user %s is already in IAM group %s.', user_name, group_name) return True return False @@ -523,8 +512,7 @@ def remove_user_from_group(group_name, user_name, region=None, key=None, keyid=N ''' user = get_user(user_name, region, key, keyid, profile) if not user: - msg = 'Username : {0} does not exist.' - log.error(msg.format(user_name, group_name)) + log.error('IAM user %s does not exist.', user_name) return False if not user_exists_in_group(user_name, group_name, region=region, key=key, keyid=keyid, profile=profile): @@ -537,8 +525,7 @@ def remove_user_from_group(group_name, user_name, region=None, key=None, keyid=N return info except boto.exception.BotoServerError as e: log.debug(e) - msg = 'Failed to remove user {0} from group {1}.' - log.error(msg.format(user_name, group_name)) + log.error('Failed to remove IAM user %s from group %s', user_name, group_name) return False @@ -558,23 +545,21 @@ def put_group_policy(group_name, policy_name, policy_json, region=None, key=None group = get_group(group_name, region=region, key=key, keyid=keyid, profile=profile) if not group: - log.error('Group {0} does not exist'.format(group_name)) + log.error('Group %s does not exist', group_name) return False conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) try: if not isinstance(policy_json, six.string_types): - policy_json = json.dumps(policy_json) + policy_json = salt.utils.json.dumps(policy_json) created = conn.put_group_policy(group_name, policy_name, policy_json) if created: - log.info('Created policy for group {0}.'.format(group_name)) + log.info('Created policy for IAM group %s.', group_name) return True - msg = 'Could not create policy for group {0}' - log.error(msg.format(group_name)) + log.error('Could not create policy for IAM group %s', group_name) except boto.exception.BotoServerError as e: log.debug(e) - msg = 'Failed to create policy for group {0}' - log.error(msg.format(group_name)) + log.error('Failed to create policy for IAM group %s', group_name) return False @@ -599,13 +584,11 @@ def delete_group_policy(group_name, policy_name, region=None, key=None, return True try: conn.delete_group_policy(group_name, policy_name) - msg = 'Successfully deleted {0} policy for group {1}.' - log.info(msg.format(policy_name, group_name)) + log.info('Successfully deleted policy %s for IAM group %s.', policy_name, group_name) return True except boto.exception.BotoServerError as e: log.debug(e) - msg = 'Failed to delete {0} policy for group {1}.' - log.error(msg.format(policy_name, group_name)) + log.error('Failed to delete policy %s for IAM group %s.', policy_name, group_name) return False @@ -625,17 +608,16 @@ def get_group_policy(group_name, policy_name, region=None, key=None, conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) try: info = conn.get_group_policy(group_name, policy_name) - log.debug('info for group policy is : {0}'.format(info)) + log.debug('info for group policy is : %s', info) if not info: return False info = info.get_group_policy_response.get_group_policy_result.policy_document info = _unquote(info) - info = json.loads(info, object_pairs_hook=odict.OrderedDict) + info = salt.utils.json.loads(info, object_pairs_hook=odict.OrderedDict) return info except boto.exception.BotoServerError as e: log.debug(e) - msg = 'Failed to get group {0} info.' - log.error(msg.format(group_name)) + log.error('Failed to get IAM group %s info.', group_name) return False @@ -750,13 +732,11 @@ def delete_group(group_name, region=None, key=None, return True try: conn.delete_group(group_name) - msg = 'Successfully deleted group {0}.' - log.info(msg.format(group_name)) + log.info('Successfully deleted IAM group %s.', group_name) return True except boto.exception.BotoServerError as e: log.debug(e) - msg = 'Failed to delete group {0}.' - log.error(msg.format(group_name)) + log.error('Failed to delete IAM group %s.', group_name) return False @@ -776,21 +756,19 @@ def create_login_profile(user_name, password, region=None, key=None, ''' user = get_user(user_name, region, key, keyid, profile) if not user: - msg = 'Username {0} does not exist' - log.error(msg.format(user_name)) + log.error('IAM user %s does not exist', user_name) return False conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) try: info = conn.create_login_profile(user_name, password) - log.info('Created profile for user {0}.'.format(user_name)) + log.info('Created profile for IAM user %s.', user_name) return info except boto.exception.BotoServerError as e: log.debug(e) if 'Conflict' in e: - log.info('Profile already exists for user {0}.'.format(user_name)) + log.info('Profile already exists for IAM user %s.', user_name) return 'Conflict' - msg = 'Failed to update profile for user {0}.' - log.error(msg.format(user_name)) + log.error('Failed to update profile for IAM user %s.', user_name) return False @@ -809,21 +787,19 @@ def delete_login_profile(user_name, region=None, key=None, keyid=None, ''' user = get_user(user_name, region, key, keyid, profile) if not user: - msg = 'Username {0} does not exist' - log.error(msg.format(user_name)) + log.error('IAM user %s does not exist', user_name) return False conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) try: info = conn.delete_login_profile(user_name) - log.info('Deleted login profile for user {0}.'.format(user_name)) + log.info('Deleted login profile for IAM user %s.', user_name) return True except boto.exception.BotoServerError as e: log.debug(e) if 'Not Found' in e: - log.info('Login profile already deleted for user {0}.'.format(user_name)) + log.info('Login profile already deleted for IAM user %s.', user_name) return True - msg = 'Failed to delete login profile for user {0}.' - log.error(msg.format(user_name)) + log.error('Failed to delete login profile for IAM user %s.', user_name) return False @@ -842,8 +818,7 @@ def get_all_mfa_devices(user_name, region=None, key=None, keyid=None, ''' user = get_user(user_name, region, key, keyid, profile) if not user: - msg = 'Username {0} does not exist' - log.error(msg.format(user_name)) + log.error('IAM user %s does not exist', user_name) return False conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) try: @@ -853,10 +828,9 @@ def get_all_mfa_devices(user_name, region=None, key=None, keyid=None, except boto.exception.BotoServerError as e: log.debug(e) if 'Not Found' in e: - log.info('Could not find user {0}.'.format(user_name)) + log.info('Could not find IAM user %s.', user_name) return [] - msg = 'Failed to get all MFA devices for user {0}.' - log.error(msg.format(user_name)) + log.error('Failed to get all MFA devices for IAM user %s.', user_name) return False @@ -876,21 +850,19 @@ def deactivate_mfa_device(user_name, serial, region=None, key=None, keyid=None, ''' user = get_user(user_name, region, key, keyid, profile) if not user: - msg = 'Username {0} does not exist' - log.error(msg.format(user_name)) + log.error('IAM user %s does not exist', user_name) return False conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) try: conn.deactivate_mfa_device(user_name, serial) - log.info('Deactivated MFA device {1} for user {0}.'.format(user_name, serial)) + log.info('Deactivated MFA device %s for IAM user %s.', serial, user_name) return True except boto.exception.BotoServerError as e: log.debug(e) if 'Not Found' in e: - log.info('MFA device {1} not associated with user {0}.'.format(user_name, serial)) + log.info('MFA device %s not associated with IAM user %s.', serial, user_name) return True - msg = 'Failed to deactivate MFA device {1} for user {0}.' - log.error(msg.format(user_name, serial)) + log.error('Failed to deactivate MFA device %s for IAM user %s.', serial, user_name) return False @@ -907,15 +879,14 @@ def delete_virtual_mfa_device(serial, region=None, key=None, keyid=None, profile conn = __utils__['boto3.get_connection_func']('iam')() try: conn.delete_virtual_mfa_device(SerialNumber=serial) - log.info('Deleted virtual MFA device {0}.'.format(serial)) + log.info('Deleted virtual MFA device %s.', serial) return True except botocore.exceptions.ClientError as e: log.debug(e) - if 'NoSuchEntity' in str(e): - log.info('Virtual MFA device {0} not found.'.format(serial)) + if 'NoSuchEntity' in six.text_type(e): + log.info('Virtual MFA device %s not found.', serial) return True - msg = 'Failed to delete virtual MFA device {0}.' - log.error(msg.format(serial)) + log.error('Failed to delete virtual MFA device %s.', serial) return False @@ -1000,12 +971,11 @@ def create_role(name, policy_document=None, path=None, region=None, key=None, try: conn.create_role(name, assume_role_policy_document=policy_document, path=path) - log.info('Created {0} iam role.'.format(name)) + log.info('Created IAM role %s.', name) return True except boto.exception.BotoServerError as e: log.error(e) - msg = 'Failed to create {0} iam role.' - log.error(msg.format(name)) + log.error('Failed to create IAM role %s.', name) return False @@ -1025,12 +995,11 @@ def delete_role(name, region=None, key=None, keyid=None, profile=None): return True try: conn.delete_role(name) - log.info('Deleted {0} iam role.'.format(name)) + log.info('Deleted %s IAM role.', name) return True except boto.exception.BotoServerError as e: log.debug(e) - msg = 'Failed to delete {0} iam role.' - log.error(msg.format(name)) + log.error('Failed to delete %s IAM role.', name) return False @@ -1076,10 +1045,10 @@ def associate_profile_to_role(profile_name, role_name, region=None, key=None, conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) if not role_exists(role_name, region, key, keyid, profile): - log.error('IAM role {0} does not exist.'.format(role_name)) + log.error('IAM role %s does not exist.', role_name) return False if not instance_profile_exists(profile_name, region, key, keyid, profile): - log.error('Instance profile {0} does not exist.'.format(profile_name)) + log.error('Instance profile %s does not exist.', profile_name) return False associated = profile_associated(role_name, profile_name, region, key, keyid, profile) if associated: @@ -1087,13 +1056,11 @@ def associate_profile_to_role(profile_name, role_name, region=None, key=None, else: try: conn.add_role_to_instance_profile(profile_name, role_name) - msg = 'Added {0} instance profile to {1} role.' - log.info(msg.format(profile_name, role_name)) + log.info('Added %s instance profile to IAM role %s.', profile_name, role_name) return True except boto.exception.BotoServerError as e: log.debug(e) - msg = 'Failed to add {0} instance profile to {1} role.' - log.error(msg.format(profile_name, role_name)) + log.error('Failed to add %s instance profile to IAM role %s', profile_name, role_name) return False @@ -1111,10 +1078,10 @@ def disassociate_profile_from_role(profile_name, role_name, region=None, conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) if not role_exists(role_name, region, key, keyid, profile): - log.error('IAM role {0} does not exist.'.format(role_name)) + log.error('IAM role %s does not exist.', role_name) return False if not instance_profile_exists(profile_name, region, key, keyid, profile): - log.error('Instance profile {0} does not exist.'.format(profile_name)) + log.error('Instance profile %s does not exist.', profile_name) return False associated = profile_associated(role_name, profile_name, region, key, keyid, profile) if not associated: @@ -1122,13 +1089,11 @@ def disassociate_profile_from_role(profile_name, role_name, region=None, else: try: conn.remove_role_from_instance_profile(profile_name, role_name) - msg = 'Removed {0} instance profile from {1} role.' - log.info(msg.format(profile_name, role_name)) + log.info('Removed %s instance profile from IAM role %s.', profile_name, role_name) return True except boto.exception.BotoServerError as e: log.debug(e) - msg = 'Failed to remove {0} instance profile from {1} role.' - log.error(msg.format(profile_name, role_name)) + log.error('Failed to remove %s instance profile from IAM role %s.', profile_name, role_name) return False @@ -1173,7 +1138,7 @@ def get_role_policy(role_name, policy_name, region=None, key=None, _policy = _policy.get_role_policy_response.policy_document # Policy is url encoded _policy = _unquote(_policy) - _policy = json.loads(_policy, object_pairs_hook=odict.OrderedDict) + _policy = salt.utils.json.loads(_policy, object_pairs_hook=odict.OrderedDict) return _policy except boto.exception.BotoServerError: return {} @@ -1199,20 +1164,20 @@ def create_role_policy(role_name, policy_name, policy, region=None, key=None, return True mode = 'modify' if isinstance(policy, six.string_types): - policy = json.loads(policy, object_pairs_hook=odict.OrderedDict) + policy = salt.utils.json.loads(policy, object_pairs_hook=odict.OrderedDict) try: - _policy = json.dumps(policy) + _policy = salt.utils.json.dumps(policy) conn.put_role_policy(role_name, policy_name, _policy) if mode == 'create': - msg = 'Successfully added {0} policy to {1} role.' + msg = 'Successfully added policy %s to IAM role %s.' else: - msg = 'Successfully modified {0} policy for role {1}.' - log.info(msg.format(policy_name, role_name)) + msg = 'Successfully modified policy %s for IAM role %s.' + log.info(msg, policy_name, role_name) return True except boto.exception.BotoServerError as e: log.error(e) - msg = 'Failed to {0} {1} policy for role {2}.' - log.error(msg.format(mode, policy_name, role_name)) + log.error('Failed to %s policy %s for IAM role %s.', + mode, policy_name, role_name) return False @@ -1234,13 +1199,13 @@ def delete_role_policy(role_name, policy_name, region=None, key=None, return True try: conn.delete_role_policy(role_name, policy_name) - msg = 'Successfully deleted {0} policy for role {1}.' - log.info(msg.format(policy_name, role_name)) + log.info('Successfully deleted policy %s for IAM role %s.', + policy_name, role_name) return True except boto.exception.BotoServerError as e: log.debug(e) - msg = 'Failed to delete {0} policy for role {1}.' - log.error(msg.format(policy_name, role_name)) + log.error('Failed to delete policy %s for IAM role %s.', + policy_name, role_name) return False @@ -1260,18 +1225,16 @@ def update_assume_role_policy(role_name, policy_document, region=None, conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) if isinstance(policy_document, six.string_types): - policy_document = json.loads(policy_document, + policy_document = salt.utils.json.loads(policy_document, object_pairs_hook=odict.OrderedDict) try: - _policy_document = json.dumps(policy_document) + _policy_document = salt.utils.json.dumps(policy_document) conn.update_assume_role_policy(role_name, _policy_document) - msg = 'Successfully updated assume role policy for role {0}.' - log.info(msg.format(role_name)) + log.info('Successfully updated assume role policy for IAM role %s.', role_name) return True except boto.exception.BotoServerError as e: log.error(e) - msg = 'Failed to update assume role policy for role {0}.' - log.error(msg.format(role_name)) + log.error('Failed to update assume role policy for IAM role %s.', role_name) return False @@ -1289,9 +1252,9 @@ def build_policy(region=None, key=None, keyid=None, profile=None): ''' conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) if hasattr(conn, 'build_policy'): - policy = json.loads(conn.build_policy()) + policy = salt.utils.json.loads(conn.build_policy()) elif hasattr(conn, '_build_policy'): - policy = json.loads(conn._build_policy()) + policy = salt.utils.json.loads(conn._build_policy()) else: return {} # The format we get from build_policy isn't going to be what we get back @@ -1416,7 +1379,7 @@ def get_all_user_policies(user_name, marker=None, max_items=None, region=None, k .. code-block:: bash - salt myminion boto_iam.get_group mygroup + salt myminion boto_iam.get_all_user_policies myuser ''' conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) try: @@ -1427,8 +1390,7 @@ def get_all_user_policies(user_name, marker=None, max_items=None, region=None, k return _list.policy_names except boto.exception.BotoServerError as e: log.debug(e) - msg = 'Failed to get user {0} policy.' - log.error(msg.format(user_name)) + log.error('Failed to get policies for user %s.', user_name) return False @@ -1447,17 +1409,16 @@ def get_user_policy(user_name, policy_name, region=None, key=None, keyid=None, p conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) try: info = conn.get_user_policy(user_name, policy_name) - log.debug('Info for user policy is : {0}.'.format(info)) + log.debug('Info for IAM user %s policy %s: %s.', user_name, policy_name, info) if not info: return False info = info.get_user_policy_response.get_user_policy_result.policy_document info = _unquote(info) - info = json.loads(info, object_pairs_hook=odict.OrderedDict) + info = salt.utils.json.loads(info, object_pairs_hook=odict.OrderedDict) return info except boto.exception.BotoServerError as e: log.debug(e) - msg = 'Failed to get user {0} policy.' - log.error(msg.format(user_name)) + log.error('Failed to get policy %s for IAM user %s.', policy_name, user_name) return False @@ -1475,23 +1436,21 @@ def put_user_policy(user_name, policy_name, policy_json, region=None, key=None, ''' user = get_user(user_name, region, key, keyid, profile) if not user: - log.error('User {0} does not exist'.format(user_name)) + log.error('IAM user %s does not exist', user_name) return False conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) try: if not isinstance(policy_json, six.string_types): - policy_json = json.dumps(policy_json) + policy_json = salt.utils.json.dumps(policy_json) created = conn.put_user_policy(user_name, policy_name, policy_json) if created: - log.info('Created policy for user {0}.'.format(user_name)) + log.info('Created policy %s for IAM user %s.', policy_name, user_name) return True - msg = 'Could not create policy for user {0}.' - log.error(msg.format(user_name)) + log.error('Could not create policy %s for IAM user %s.', policy_name, user_name) except boto.exception.BotoServerError as e: log.debug(e) - msg = 'Failed to create policy for user {0}.' - log.error(msg.format(user_name)) + log.error('Failed to create policy %s for IAM user %s.', policy_name, user_name) return False @@ -1515,13 +1474,11 @@ def delete_user_policy(user_name, policy_name, region=None, key=None, keyid=None return True try: conn.delete_user_policy(user_name, policy_name) - msg = 'Successfully deleted {0} policy for user {1}.' - log.info(msg.format(policy_name, user_name)) + log.info('Successfully deleted policy %s for IAM user %s.', policy_name, user_name) return True except boto.exception.BotoServerError as e: log.debug(e) - msg = 'Failed to delete {0} policy for user {1}.' - log.error(msg.format(policy_name, user_name)) + log.error('Failed to delete policy %s for IAM user %s.', policy_name, user_name) return False @@ -1556,12 +1513,11 @@ def upload_server_cert(cert_name, cert_body, private_key, cert_chain=None, path= conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) try: info = conn.upload_server_cert(cert_name, cert_body, private_key, cert_chain) - log.info('Created certificate {0}.'.format(cert_name)) + log.info('Created certificate %s.', cert_name) return info except boto.exception.BotoServerError as e: log.debug(e) - msg = 'Failed to failed to create certificate {0}.' - log.error(msg.format(cert_name)) + log.error('Failed to failed to create certificate %s.', cert_name) return False @@ -1585,8 +1541,7 @@ def get_server_certificate(cert_name, region=None, key=None, keyid=None, profile return info except boto.exception.BotoServerError as e: log.debug(e) - msg = 'Failed to get certificate {0} information.' - log.error(msg.format(cert_name)) + log.error('Failed to get certificate %s information.', cert_name) return False @@ -1607,8 +1562,7 @@ def delete_server_cert(cert_name, region=None, key=None, keyid=None, profile=Non return conn.delete_server_cert(cert_name) except boto.exception.BotoServerError as e: log.debug(e) - msg = 'Failed to delete certificate {0}.' - log.error(msg.format(cert_name)) + log.error('Failed to delete certificate %s.', cert_name) return False @@ -1651,7 +1605,7 @@ def export_users(path_prefix='/', region=None, key=None, keyid=None, policies = {} for policy_name in _policies: _policy = conn.get_user_policy(name, policy_name) - _policy = json.loads(_unquote( + _policy = salt.utils.json.loads(_unquote( _policy.get_user_policy_response.get_user_policy_result.policy_document )) policies[policy_name] = _policy @@ -1684,14 +1638,14 @@ def export_roles(path_prefix='/', region=None, key=None, keyid=None, profile=Non policies = {} for policy_name in _policies: _policy = conn.get_role_policy(name, policy_name) - _policy = json.loads(_unquote( + _policy = salt.utils.json.loads(_unquote( _policy.get_role_policy_response.get_role_policy_result.policy_document )) policies[policy_name] = _policy role_sls = [] role_sls.append({"name": name}) role_sls.append({"policies": policies}) - role_sls.append({'policy_document': json.loads(_unquote(role.assume_role_policy_document))}) + role_sls.append({'policy_document': salt.utils.json.loads(_unquote(role.assume_role_policy_document))}) role_sls.append({"path": role.path}) results["manage role " + name] = {"boto_iam_role.present": role_sls} return _safe_dump(results) @@ -1763,7 +1717,7 @@ def create_policy(policy_name, policy_document, path=None, description=None, conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) if not isinstance(policy_document, six.string_types): - policy_document = json.dumps(policy_document) + policy_document = salt.utils.json.dumps(policy_document) params = {} for arg in 'path', 'description': if locals()[arg] is not None: @@ -1772,11 +1726,10 @@ def create_policy(policy_name, policy_document, path=None, description=None, return True try: conn.create_policy(policy_name, policy_document, **params) - log.info('Created {0} policy.'.format(policy_name)) + log.info('Created IAM policy %s.', policy_name) except boto.exception.BotoServerError as e: log.debug(e) - msg = 'Failed to create {0} policy.' - log.error(msg.format(policy_name)) + log.error('Failed to create IAM policy %s.', policy_name) return False return True @@ -1799,12 +1752,11 @@ def delete_policy(policy_name, return True try: conn.delete_policy(policy_arn) - log.info('Deleted {0} policy.'.format(policy_name)) + log.info('Deleted %s policy.', policy_name) except boto.exception.BotoServerError as e: aws = __utils__['boto.get_error'](e) log.debug(aws) - msg = 'Failed to delete {0} policy: {1}.' - log.error(msg.format(policy_name, aws.get('message'))) + log.error('Failed to delete %s policy: %s.', policy_name, aws.get('message')) return False return True @@ -1891,7 +1843,7 @@ def create_policy_version(policy_name, policy_document, set_as_default=None, conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) if not isinstance(policy_document, six.string_types): - policy_document = json.dumps(policy_document) + policy_document = salt.utils.json.dumps(policy_document) params = {} for arg in ('set_as_default',): if locals()[arg] is not None: @@ -1900,12 +1852,11 @@ def create_policy_version(policy_name, policy_document, set_as_default=None, try: ret = conn.create_policy_version(policy_arn, policy_document, **params) vid = ret.get('create_policy_version_response', {}).get('create_policy_version_result', {}).get('policy_version', {}).get('version_id') - log.info('Created {0} policy version {1}.'.format(policy_name, vid)) + log.info('Created IAM policy %s version %s.', policy_name, vid) return {'created': True, 'version_id': vid} except boto.exception.BotoServerError as e: log.debug(e) - msg = 'Failed to create {0} policy version.' - log.error(msg.format(policy_name)) + log.error('Failed to create IAM policy %s version %s.', policy_name, vid) return {'created': False, 'error': __utils__['boto.get_error'](e)} @@ -1927,12 +1878,12 @@ def delete_policy_version(policy_name, version_id, return True try: conn.delete_policy_version(policy_arn, version_id) - log.info('Deleted {0} policy version {1}.'.format(policy_name, version_id)) + log.info('Deleted IAM policy %s version %s.', policy_name, version_id) except boto.exception.BotoServerError as e: aws = __utils__['boto.get_error'](e) log.debug(aws) - msg = 'Failed to delete {0} policy version {1}: {2}' - log.error(msg.format(policy_name, version_id, aws.get('message'))) + log.error('Failed to delete IAM policy %s version %s: %s', + policy_name, version_id, aws.get('message')) return False return True @@ -1956,8 +1907,7 @@ def list_policy_versions(policy_name, return ret.get('list_policy_versions_response', {}).get('list_policy_versions_result', {}).get('versions') except boto.exception.BotoServerError as e: log.debug(e) - msg = 'Failed to list {0} policy versions.' - log.error(msg.format(policy_name)) + log.error('Failed to list versions for IAM policy %s.', policy_name) return [] @@ -1977,12 +1927,12 @@ def set_default_policy_version(policy_name, version_id, policy_arn = _get_policy_arn(policy_name, region, key, keyid, profile) try: conn.set_default_policy_version(policy_arn, version_id) - log.info('Set {0} policy to version {1}.'.format(policy_name, version_id)) + log.info('Set %s policy to version %s.', policy_name, version_id) except boto.exception.BotoServerError as e: aws = __utils__['boto.get_error'](e) log.debug(aws) - msg = 'Failed to set {0} policy to version {1}: {2}' - log.error(msg.format(policy_name, version_id, aws.get('message'))) + log.error('Failed to set %s policy to version %s: %s', + policy_name, version_id, aws.get('message')) return False return True @@ -2003,11 +1953,10 @@ def attach_user_policy(policy_name, user_name, policy_arn = _get_policy_arn(policy_name, region, key, keyid, profile) try: conn.attach_user_policy(policy_arn, user_name) - log.info('Attached {0} policy to user {1}.'.format(policy_name, user_name)) + log.info('Attached policy %s to IAM user %s.', policy_name, user_name) except boto.exception.BotoServerError as e: log.debug(e) - msg = 'Failed to attach {0} policy to user {1}.' - log.error(msg.format(policy_name, user_name)) + log.error('Failed to attach %s policy to IAM user %s.', policy_name, user_name) return False return True @@ -2028,11 +1977,10 @@ def detach_user_policy(policy_name, user_name, policy_arn = _get_policy_arn(policy_name, region, key, keyid, profile) try: conn.detach_user_policy(policy_arn, user_name) - log.info('Detached {0} policy to user {1}.'.format(policy_name, user_name)) + log.info('Detached %s policy from IAM user %s.', policy_name, user_name) except boto.exception.BotoServerError as e: log.debug(e) - msg = 'Failed to detach {0} policy to user {1}.' - log.error(msg.format(policy_name, user_name)) + log.error('Failed to detach %s policy from IAM user %s.', policy_name, user_name) return False return True @@ -2053,11 +2001,10 @@ def attach_group_policy(policy_name, group_name, policy_arn = _get_policy_arn(policy_name, region, key, keyid, profile) try: conn.attach_group_policy(policy_arn, group_name) - log.info('Attached {0} policy to group {1}.'.format(policy_name, group_name)) + log.info('Attached policy %s to IAM group %s.', policy_name, group_name) except boto.exception.BotoServerError as e: log.debug(e) - msg = 'Failed to attach {0} policy to group {1}.' - log.error(msg.format(policy_name, group_name)) + log.error('Failed to attach policy %s to IAM group %s.', policy_name, group_name) return False return True @@ -2078,11 +2025,10 @@ def detach_group_policy(policy_name, group_name, policy_arn = _get_policy_arn(policy_name, region, key, keyid, profile) try: conn.detach_group_policy(policy_arn, group_name) - log.info('Detached {0} policy to group {1}.'.format(policy_name, group_name)) + log.info('Detached policy %s from IAM group %s.', policy_name, group_name) except boto.exception.BotoServerError as e: log.debug(e) - msg = 'Failed to detach {0} policy to group {1}.' - log.error(msg.format(policy_name, group_name)) + log.error('Failed to detach policy %s from IAM group %s.', policy_name, group_name) return False return True @@ -2103,11 +2049,10 @@ def attach_role_policy(policy_name, role_name, policy_arn = _get_policy_arn(policy_name, region, key, keyid, profile) try: conn.attach_role_policy(policy_arn, role_name) - log.info('Attached {0} policy to role {1}.'.format(policy_name, role_name)) + log.info('Attached policy %s to IAM role %s.', policy_name, role_name) except boto.exception.BotoServerError as e: log.debug(e) - msg = 'Failed to attach {0} policy to role {1}.' - log.error(msg.format(policy_name, role_name)) + log.error('Failed to attach policy %s to IAM role %s.', policy_name, role_name) return False return True @@ -2128,11 +2073,10 @@ def detach_role_policy(policy_name, role_name, policy_arn = _get_policy_arn(policy_name, region, key, keyid, profile) try: conn.detach_role_policy(policy_arn, role_name) - log.info('Detached {0} policy to role {1}.'.format(policy_name, role_name)) + log.info('Detached policy %s from IAM role %s.', policy_name, role_name) except boto.exception.BotoServerError as e: log.debug(e) - msg = 'Failed to detach {0} policy to role {1}.' - log.error(msg.format(policy_name, role_name)) + log.error('Failed to detach policy %s from IAM role %s.', policy_name, role_name) return False return True @@ -2174,7 +2118,7 @@ def list_entities_for_policy(policy_name, path_prefix=None, entity_filter=None, time.sleep(5) retries -= 1 continue - log.error('Failed to list {0} policy entities: {1}'.format(policy_name, e.message)) + log.error('Failed to list entities for IAM policy %s: %s', policy_name, e.message) return {} return {} @@ -2206,8 +2150,7 @@ def list_attached_user_policies(user_name, path_prefix=None, entity_filter=None, return policies except boto.exception.BotoServerError as e: log.debug(e) - msg = 'Failed to list user {0} attached policies.' - log.error(msg.format(user_name)) + log.error('Failed to list attached policies for IAM user %s.', user_name) return [] @@ -2238,8 +2181,7 @@ def list_attached_group_policies(group_name, path_prefix=None, entity_filter=Non return policies except boto.exception.BotoServerError as e: log.debug(e) - msg = 'Failed to list group {0} attached policies.' - log.error(msg.format(group_name)) + log.error('Failed to list attached policies for IAM group %s.', group_name) return [] @@ -2270,8 +2212,7 @@ def list_attached_role_policies(role_name, path_prefix=None, entity_filter=None, return policies except boto.exception.BotoServerError as e: log.debug(e) - msg = 'Failed to list role {0} attached policies.' - log.error(msg.format(role_name)) + log.error('Failed to list attached policies for IAM role %s.', role_name) return [] @@ -2288,14 +2229,12 @@ def create_saml_provider(name, saml_metadata_document, region=None, key=None, ke conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) try: conn.create_saml_provider(saml_metadata_document, name) - msg = 'Successfully created {0} SAML provider.' - log.info(msg.format(name)) + log.info('Successfully created %s SAML provider.', name) return True except boto.exception.BotoServerError as e: aws = __utils__['boto.get_error'](e) log.debug(aws) - msg = 'Failed to create SAML provider {0}.' - log.error(msg.format(name)) + log.error('Failed to create SAML provider %s.', name) return False @@ -2319,8 +2258,7 @@ def get_saml_provider_arn(name, region=None, key=None, keyid=None, profile=None) except boto.exception.BotoServerError as e: aws = __utils__['boto.get_error'](e) log.debug(aws) - msg = 'Failed to get ARN of SAML provider {0}.' - log.error(msg.format(name)) + log.error('Failed to get ARN of SAML provider %s.', name) return False @@ -2338,18 +2276,15 @@ def delete_saml_provider(name, region=None, key=None, keyid=None, profile=None): try: saml_provider_arn = get_saml_provider_arn(name, region=region, key=key, keyid=keyid, profile=profile) if not saml_provider_arn: - msg = 'SAML provider {0} not found.' - log.info(msg.format(name)) + log.info('SAML provider %s not found.', name) return True conn.delete_saml_provider(saml_provider_arn) - msg = 'Successfully deleted {0} SAML provider.' - log.info(msg.format(name)) + log.info('Successfully deleted SAML provider %s.', name) return True except boto.exception.BotoServerError as e: aws = __utils__['boto.get_error'](e) log.debug(aws) - msg = 'Failed to delete {0} SAML provider.' - log.error(msg.format(name)) + log.error('Failed to delete SAML provider %s.', name) return False @@ -2371,10 +2306,8 @@ def list_saml_providers(region=None, key=None, keyid=None, profile=None): providers.append(arn['arn'].rsplit('/', 1)[1]) return providers except boto.exception.BotoServerError as e: - aws = __utils__['boto.get_error'](e) - log.debug(aws) - msg = 'Failed to get list of SAML providers.' - log.error(msg) + log.debug(__utils__['boto.get_error'](e)) + log.error('Failed to get list of SAML providers.') return False @@ -2393,10 +2326,8 @@ def get_saml_provider(name, region=None, key=None, keyid=None, profile=None): provider = conn.get_saml_provider(name) return provider['get_saml_provider_response']['get_saml_provider_result']['saml_metadata_document'] except boto.exception.BotoServerError as e: - aws = __utils__['boto.get_error'](e) - log.debug(aws) - msg = 'Failed to get SAML provider document.' - log.error(msg) + log.debug(__utils__['boto.get_error'](e)) + log.error('Failed to get SAML provider document %s.', name) return False @@ -2414,15 +2345,12 @@ def update_saml_provider(name, saml_metadata_document, region=None, key=None, ke try: saml_provider_arn = get_saml_provider_arn(name, region=region, key=key, keyid=keyid, profile=profile) if not saml_provider_arn: - msg = 'SAML provider {0} not found.' - log.info(msg.format(name)) + log.info('SAML provider %s not found.', name) return False if conn.update_saml_provider(name, saml_metadata_document): return True return False except boto.exception.BotoServerError as e: - aws = __utils__['boto.get_error'](e) - log.debug(aws) - msg = 'Failed to update of SAML provider.' - log.error(msg.format(name)) + log.debug(__utils__['boto.get_error'](e)) + log.error('Failed to update SAML provider %s.', name) return False diff --git a/salt/modules/boto_iot.py b/salt/modules/boto_iot.py index 05ee9256f11..b84ff0b3bcd 100644 --- a/salt/modules/boto_iot.py +++ b/salt/modules/boto_iot.py @@ -52,12 +52,12 @@ The dependencies listed above can be installed via package or pip. # Import Python libs from __future__ import absolute_import import logging -import json import datetime # Import Salt libs import salt.utils.boto3 import salt.utils.compat +import salt.utils.json from salt.utils.versions import LooseVersion as _LooseVersion log = logging.getLogger(__name__) @@ -329,7 +329,7 @@ def create_policy(policyName, policyDocument, try: conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) if not isinstance(policyDocument, string_types): - policyDocument = json.dumps(policyDocument) + policyDocument = salt.utils.json.dumps(policyDocument) policy = conn.create_policy(policyName=policyName, policyDocument=policyDocument) if policy: @@ -446,7 +446,7 @@ def create_policy_version(policyName, policyDocument, setAsDefault=False, try: conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) if not isinstance(policyDocument, string_types): - policyDocument = json.dumps(policyDocument) + policyDocument = salt.utils.json.dumps(policyDocument) policy = conn.create_policy_version(policyName=policyName, policyDocument=policyDocument, setAsDefault=setAsDefault) diff --git a/salt/modules/boto_kms.py b/salt/modules/boto_kms.py index 5f7967a8a98..d7e98dca71f 100644 --- a/salt/modules/boto_kms.py +++ b/salt/modules/boto_kms.py @@ -40,11 +40,11 @@ from __future__ import absolute_import # Import Python libs import logging -from salt.serializers import json # Import Salt libs import salt.utils.compat import salt.utils.odict as odict +import salt.serializers.json from salt.utils.versions import LooseVersion as _LooseVersion log = logging.getLogger(__name__) @@ -146,7 +146,7 @@ def create_key(policy=None, description=None, key_usage=None, region=None, conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) r = {} - _policy = json.serialize(policy) + _policy = salt.serializers.json.serialize(policy) try: key_metadata = conn.create_key( _policy, @@ -432,7 +432,7 @@ def get_key_policy(key_id, policy_name, region=None, key=None, keyid=None, r = {} try: key_policy = conn.get_key_policy(key_id, policy_name) - r['key_policy'] = json.deserialize( + r['key_policy'] = salt.serializers.json.deserialize( key_policy['Policy'], object_pairs_hook=odict.OrderedDict ) @@ -536,7 +536,7 @@ def put_key_policy(key_id, policy_name, policy, region=None, key=None, r = {} try: - conn.put_key_policy(key_id, policy_name, json.serialize(policy)) + conn.put_key_policy(key_id, policy_name, salt.serializers.json.serialize(policy)) r['result'] = True except boto.exception.BotoServerError as e: r['result'] = False diff --git a/salt/modules/boto_lambda.py b/salt/modules/boto_lambda.py index 2b0e8bb95d3..06912928140 100644 --- a/salt/modules/boto_lambda.py +++ b/salt/modules/boto_lambda.py @@ -82,7 +82,6 @@ The dependencies listed above can be installed via package or pip. # Import Python libs from __future__ import absolute_import import logging -import json import time import random @@ -90,6 +89,7 @@ import random from salt.ext import six import salt.utils.compat import salt.utils.files +import salt.utils.json from salt.utils.versions import LooseVersion as _LooseVersion from salt.exceptions import SaltInvocationError from salt.ext.six.moves import range # pylint: disable=import-error @@ -207,7 +207,7 @@ def _filedata(infile): def _resolve_vpcconfig(conf, region=None, key=None, keyid=None, profile=None): if isinstance(conf, six.string_types): - conf = json.loads(conf) + conf = salt.utils.json.loads(conf) if not conf: return None if not isinstance(conf, dict): @@ -583,7 +583,7 @@ def get_permissions(FunctionName, Qualifier=None, **kwargs) policy = policy.get('Policy', {}) if isinstance(policy, six.string_types): - policy = json.loads(policy) + policy = salt.utils.json.loads(policy) if policy is None: policy = {} permissions = {} diff --git a/salt/modules/boto_s3_bucket.py b/salt/modules/boto_s3_bucket.py index b18f92482ac..2ca071203e1 100644 --- a/salt/modules/boto_s3_bucket.py +++ b/salt/modules/boto_s3_bucket.py @@ -54,12 +54,12 @@ The dependencies listed above can be installed via package or pip. # Import Python libs from __future__ import absolute_import import logging -import json # Import Salt libs from salt.ext import six from salt.ext.six.moves import range # pylint: disable=import-error import salt.utils.compat +import salt.utils.json from salt.exceptions import SaltInvocationError from salt.utils.versions import LooseVersion as _LooseVersion @@ -227,7 +227,7 @@ def delete_objects(Bucket, Delete, MFA=None, RequestPayer=None, ''' if isinstance(Delete, six.string_types): - Delete = json.loads(Delete) + Delete = salt.utils.json.loads(Delete) if not isinstance(Delete, dict): raise SaltInvocationError("Malformed Delete request.") if 'Objects' not in Delete: @@ -483,7 +483,7 @@ def put_acl(Bucket, kwargs = {} if AccessControlPolicy is not None: if isinstance(AccessControlPolicy, six.string_types): - AccessControlPolicy = json.loads(AccessControlPolicy) + AccessControlPolicy = salt.utils.json.loads(AccessControlPolicy) kwargs['AccessControlPolicy'] = AccessControlPolicy for arg in ('ACL', 'GrantFullControl', @@ -523,7 +523,7 @@ def put_cors(Bucket, try: conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) if CORSRules is not None and isinstance(CORSRules, six.string_types): - CORSRules = json.loads(CORSRules) + CORSRules = salt.utils.json.loads(CORSRules) conn.put_bucket_cors(Bucket=Bucket, CORSConfiguration={'CORSRules': CORSRules}) return {'updated': True, 'name': Bucket} except ClientError as e: @@ -558,7 +558,7 @@ def put_lifecycle_configuration(Bucket, try: conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile) if Rules is not None and isinstance(Rules, six.string_types): - Rules = json.loads(Rules) + Rules = salt.utils.json.loads(Rules) conn.put_bucket_lifecycle_configuration(Bucket=Bucket, LifecycleConfiguration={'Rules': Rules}) return {'updated': True, 'name': Bucket} except ClientError as e: @@ -596,7 +596,7 @@ def put_logging(Bucket, else: logstatus = {} if TargetGrants is not None and isinstance(TargetGrants, six.string_types): - TargetGrants = json.loads(TargetGrants) + TargetGrants = salt.utils.json.loads(TargetGrants) conn.put_bucket_logging(Bucket=Bucket, BucketLoggingStatus=logstatus) return {'updated': True, 'name': Bucket} except ClientError as e: @@ -629,15 +629,15 @@ def put_notification_configuration(Bucket, if TopicConfigurations is None: TopicConfigurations = [] elif isinstance(TopicConfigurations, six.string_types): - TopicConfigurations = json.loads(TopicConfigurations) + TopicConfigurations = salt.utils.json.loads(TopicConfigurations) if QueueConfigurations is None: QueueConfigurations = [] elif isinstance(QueueConfigurations, six.string_types): - QueueConfigurations = json.loads(QueueConfigurations) + QueueConfigurations = salt.utils.json.loads(QueueConfigurations) if LambdaFunctionConfigurations is None: LambdaFunctionConfigurations = [] elif isinstance(LambdaFunctionConfigurations, six.string_types): - LambdaFunctionConfigurations = json.loads(LambdaFunctionConfigurations) + LambdaFunctionConfigurations = salt.utils.json.loads(LambdaFunctionConfigurations) # TODO allow the user to use simple names & substitute ARNs for those names conn.put_bucket_notification_configuration(Bucket=Bucket, NotificationConfiguration={ 'TopicConfigurations': TopicConfigurations, @@ -670,7 +670,7 @@ def put_policy(Bucket, Policy, if Policy is None: Policy = '{}' elif not isinstance(Policy, six.string_types): - Policy = json.dumps(Policy) + Policy = salt.utils.json.dumps(Policy) conn.put_bucket_policy(Bucket=Bucket, Policy=Policy) return {'updated': True, 'name': Bucket} except ClientError as e: @@ -714,7 +714,7 @@ def put_replication(Bucket, Role, Rules, if Rules is None: Rules = [] elif isinstance(Rules, six.string_types): - Rules = json.loads(Rules) + Rules = salt.utils.json.loads(Rules) conn.put_bucket_replication(Bucket=Bucket, ReplicationConfiguration={ 'Role': Role, 'Rules': Rules @@ -838,7 +838,7 @@ def put_website(Bucket, ErrorDocument=None, IndexDocument=None, val = locals()[key] if val is not None: if isinstance(val, six.string_types): - WebsiteConfiguration[key] = json.loads(val) + WebsiteConfiguration[key] = salt.utils.json.loads(val) else: WebsiteConfiguration[key] = val conn.put_bucket_website(Bucket=Bucket, diff --git a/salt/modules/boto_sqs.py b/salt/modules/boto_sqs.py index 034ba11802b..8034f67e79f 100644 --- a/salt/modules/boto_sqs.py +++ b/salt/modules/boto_sqs.py @@ -48,7 +48,9 @@ from __future__ import absolute_import # Import Python libs import logging -import json + +# Import Salt libs +import salt.utils.json # Import 3rd-party libs from salt.ext import six @@ -87,13 +89,13 @@ def _preprocess_attributes(attributes): Pre-process incoming queue attributes before setting them ''' if isinstance(attributes, six.string_types): - attributes = json.loads(attributes) + attributes = salt.utils.json.loads(attributes) def stringified(val): # Some attributes take full json policy documents, but they take them # as json strings. Convert the value back into a json string. if isinstance(val, dict): - return json.dumps(val) + return salt.utils.json.dumps(val) return val return dict( diff --git a/salt/modules/bower.py b/salt/modules/bower.py index ecf7e061347..f3157b9e049 100644 --- a/salt/modules/bower.py +++ b/salt/modules/bower.py @@ -11,11 +11,11 @@ available. from __future__ import absolute_import # Import python libs -import json import logging import shlex # Import salt libs +import salt.utils.json import salt.utils.path from salt.exceptions import CommandExecutionError from salt.utils.versions import LooseVersion as _LooseVersion @@ -129,7 +129,7 @@ def install(pkg, raise CommandExecutionError(result['stderr']) # If package is already installed, Bower will emit empty dict to STDOUT - stdout = json.loads(result['stdout']) + stdout = salt.utils.json.loads(result['stdout']) return stdout != {} @@ -174,7 +174,7 @@ def uninstall(pkg, dir, runas=None, env=None): raise CommandExecutionError(result['stderr']) # If package is not installed, Bower will emit empty dict to STDOUT - stdout = json.loads(result['stdout']) + stdout = salt.utils.json.loads(result['stdout']) return stdout != {} @@ -214,7 +214,7 @@ def list_(dir, runas=None, env=None): if result['retcode'] != 0: raise CommandExecutionError(result['stderr']) - return json.loads(result['stdout'])['dependencies'] + return salt.utils.json.loads(result['stdout'])['dependencies'] def prune(dir, runas=None, env=None): @@ -255,4 +255,4 @@ def prune(dir, runas=None, env=None): raise CommandExecutionError(result['stderr']) # Bower returns an empty dictionary if nothing was pruned - return json.loads(result['stdout']) + return salt.utils.json.loads(result['stdout']) diff --git a/salt/modules/cassandra_cql.py b/salt/modules/cassandra_cql.py index 6d2480ad5f4..71057b1a6f3 100644 --- a/salt/modules/cassandra_cql.py +++ b/salt/modules/cassandra_cql.py @@ -83,19 +83,19 @@ queries based on the internal schema of said version. # Import Python Libs from __future__ import absolute_import import logging -import json import re import ssl # Import Salt Libs +import salt.utils.json from salt.exceptions import CommandExecutionError + +# Import 3rd-party libs from salt.ext import six from salt.ext.six.moves import range SSL_VERSION = 'ssl_version' -SSL_VERSION = 'ssl_version' - log = logging.getLogger(__name__) __virtualname__ = 'cassandra_cql' @@ -728,7 +728,7 @@ def create_keyspace(keyspace, replication_strategy='SimpleStrategy', replication if replication_datacenters: if isinstance(replication_datacenters, six.string_types): try: - replication_datacenter_map = json.loads(replication_datacenters) + replication_datacenter_map = salt.utils.json.loads(replication_datacenters) replication_map.update(**replication_datacenter_map) except BaseException: # pylint: disable=W0703 log.error("Could not load json replication_datacenters.") diff --git a/salt/modules/chronos.py b/salt/modules/chronos.py index 066e0b80ca5..7b852da5f2b 100644 --- a/salt/modules/chronos.py +++ b/salt/modules/chronos.py @@ -8,9 +8,9 @@ Currently this only works when run through a proxy minion. ''' from __future__ import absolute_import -import json import logging import salt.utils.http +import salt.utils.json import salt.utils.platform from salt.exceptions import get_error_message @@ -105,7 +105,7 @@ def update_job(name, config): ''' if 'name' not in config: config['name'] = name - data = json.dumps(config) + data = salt.utils.json.dumps(config) try: response = salt.utils.http.query( "{0}/scheduler/iso8601".format(_base_url()), diff --git a/salt/modules/cmdmod.py b/salt/modules/cmdmod.py index bcf7587940d..9d4cec989b7 100644 --- a/salt/modules/cmdmod.py +++ b/salt/modules/cmdmod.py @@ -10,7 +10,6 @@ from __future__ import absolute_import, print_function, unicode_literals # Import python libs import functools import glob -import json import logging import os import shutil @@ -27,6 +26,7 @@ import tempfile import salt.utils.args import salt.utils.data import salt.utils.files +import salt.utils.json import salt.utils.path import salt.utils.platform import salt.utils.powershell @@ -3272,7 +3272,7 @@ def powershell(cmd, **kwargs) try: - return json.loads(response) + return salt.utils.json.loads(response) except Exception: log.error("Error converting PowerShell JSON return", exc_info=True) return {} @@ -3578,7 +3578,7 @@ def powershell_all(cmd, # If we fail to parse stdoutput we will raise an exception try: - result = json.loads(stdoutput) + result = salt.utils.json.loads(stdoutput) except Exception: err_msg = "cmd.powershell_all " + \ "cannot parse the Powershell output." diff --git a/salt/modules/consul.py b/salt/modules/consul.py index 15ccba9effa..f8a8158f8df 100644 --- a/salt/modules/consul.py +++ b/salt/modules/consul.py @@ -8,6 +8,12 @@ https://www.consul.io # Import Python Libs from __future__ import absolute_import +import base64 +import logging + +# Import salt libs +import salt.utils.http +import salt.utils.json # Import 3rd-party libs # pylint: disable=import-error,no-name-in-module,redefined-builtin @@ -16,13 +22,6 @@ import salt.ext.six import salt.ext.six.moves.http_client # pylint: enable=import-error,no-name-in-module -# Import salt libs -import salt.utils.http - -import base64 -import json - -import logging log = logging.getLogger(__name__) from salt.exceptions import SaltInvocationError @@ -84,7 +83,7 @@ def _query(function, if data is None: data = {} - data = json.dumps(data) + data = salt.utils.json.dumps(data) result = salt.utils.http.query( url, diff --git a/salt/modules/ddns.py b/salt/modules/ddns.py index 3cc55e25e77..a954b9fe985 100644 --- a/salt/modules/ddns.py +++ b/salt/modules/ddns.py @@ -26,7 +26,6 @@ Support for RFC 2136 dynamic DNS updates. from __future__ import absolute_import # Import python libs import logging -import json log = logging.getLogger(__name__) @@ -40,6 +39,7 @@ except ImportError as e: dns_support = False import salt.utils.files +import salt.utils.json def __virtual__(): @@ -71,7 +71,7 @@ def _get_keyring(keyfile): keyring = None if keyfile: with salt.utils.files.fopen(keyfile) as _f: - keyring = dns.tsigkeyring.from_text(json.load(_f)) + keyring = dns.tsigkeyring.from_text(salt.utils.json.load(_f)) return keyring diff --git a/salt/modules/defaults.py b/salt/modules/defaults.py index bd33f48fc27..da69030f8d9 100644 --- a/salt/modules/defaults.py +++ b/salt/modules/defaults.py @@ -6,7 +6,6 @@ Module to work with salt formula defaults files from __future__ import absolute_import import copy -import json import logging import os import yaml @@ -15,6 +14,7 @@ import salt.fileclient import salt.utils.data import salt.utils.dictupdate as dictupdate import salt.utils.files +import salt.utils.json import salt.utils.url @@ -57,9 +57,9 @@ def _load(formula): suffix = file_.rsplit('.', 1)[-1] if suffix == 'yaml': - loader = yaml + loader = yaml.safe_load elif suffix == 'json': - loader = json + loader = salt.utils.json.load else: log.debug("Failed to determine loader for %r", file_) continue @@ -67,7 +67,7 @@ def _load(formula): if os.path.exists(file_): log.debug("Reading defaults from %r", file_) with salt.utils.files.fopen(file_) as fhr: - defaults = loader.load(fhr) + defaults = loader(fhr) log.debug("Read defaults %r", defaults) return defaults or {} diff --git a/salt/modules/dockermod.py b/salt/modules/dockermod.py index ab5606485cc..b91ede416ef 100644 --- a/salt/modules/dockermod.py +++ b/salt/modules/dockermod.py @@ -212,6 +212,7 @@ from salt.ext.six.moves import map # pylint: disable=import-error,redefined-bui import salt.utils.docker.translate.container import salt.utils.docker.translate.network import salt.utils.functools +import salt.utils.json import salt.utils.path import salt.pillar import salt.exceptions @@ -357,7 +358,7 @@ def _get_client(timeout=NOTSET, **kwargs): python_shell=False) try: docker_machine_json = \ - json.loads(__utils__['stringutils.to_str'](docker_machine_json)) + salt.utils.json.loads(docker_machine_json) docker_machine_tls = \ docker_machine_json['HostOptions']['AuthOptions'] docker_machine_ip = docker_machine_json['Driver']['IPAddress'] @@ -3971,12 +3972,7 @@ def build(path=None, stream_data = [] for line in response: - stream_data.extend( - json.loads( - __utils__['stringutils.to_str'](line), - cls=DockerJSONDecoder - ) - ) + stream_data.extend(salt.utils.json.loads(line, cls=DockerJSONDecoder)) errors = [] # Iterate through API response and collect information for item in stream_data: @@ -4511,7 +4507,7 @@ def pull(image, for event in response: log.debug('pull event: %s', event) try: - event = json.loads(__utils__['stringutils.to_str'](event)) + event = salt.utils.json.loads(event) except Exception as exc: raise CommandExecutionError( 'Unable to interpret API event: \'{0}\''.format(event), @@ -4605,7 +4601,7 @@ def push(image, # Iterate through API response and collect information for event in response: try: - event = json.loads(__utils__['stringutils.to_str'](event)) + event = salt.utils.json.loads(event) except Exception as exc: raise CommandExecutionError( 'Unable to interpret API event: \'{0}\''.format(event), diff --git a/salt/modules/glassfish.py b/salt/modules/glassfish.py index 0d99e74409a..be15d79756e 100644 --- a/salt/modules/glassfish.py +++ b/salt/modules/glassfish.py @@ -12,9 +12,9 @@ except ImportError: # python3 from urllib.parse import quote, unquote try: - import json import requests import salt.defaults.exitcodes + import salt.utils.json from salt.exceptions import CommandExecutionError HAS_LIBS = True except ImportError: @@ -97,7 +97,7 @@ def _api_response(response): raise CommandExecutionError('Bad username or password') elif response.status_code == 200 or response.status_code == 500: try: - data = json.loads(response.content) + data = salt.utils.json.loads(response.content) if data['exit_code'] != 'SUCCESS': __context__['retcode'] = salt.defaults.exitcodes.SALT_BUILD_FAIL raise CommandExecutionError(data['message']) @@ -132,7 +132,7 @@ def _api_post(path, data, server=None): url=_get_url(server['ssl'], server['url'], server['port'], path), auth=_get_auth(server['user'], server['password']), headers=_get_headers(), - data=json.dumps(data), + data=salt.utils.json.dumps(data), verify=False ) return _api_response(response) diff --git a/salt/modules/grains.py b/salt/modules/grains.py index 332678a4aa4..b3f922ebee7 100644 --- a/salt/modules/grains.py +++ b/salt/modules/grains.py @@ -17,7 +17,6 @@ import random import logging import operator import collections -import json import math import yaml from functools import reduce # pylint: disable=redefined-builtin @@ -27,6 +26,7 @@ from salt.ext import six import salt.utils.compat import salt.utils.data import salt.utils.files +import salt.utils.json import salt.utils.platform import salt.utils.yamldumper from salt.defaults import DEFAULT_TARGET_DELIM @@ -116,7 +116,7 @@ def get(key, default='', delimiter=DEFAULT_TARGET_DELIM, ordered=True): if ordered is True: grains = __grains__ else: - grains = json.loads(json.dumps(__grains__)) + grains = salt.utils.json.loads(salt.utils.json.dumps(__grains__)) return salt.utils.data.traverse_dict_and_list( grains, key, diff --git a/salt/modules/heat.py b/salt/modules/heat.py index 255d5643c42..39a5019a941 100644 --- a/salt/modules/heat.py +++ b/salt/modules/heat.py @@ -43,13 +43,13 @@ Module for handling OpenStack Heat calls # Import Python libs from __future__ import absolute_import import time -import json import logging import yaml # Import Salt libs from salt.ext import six import salt.utils.files +import salt.utils.json from salt.exceptions import SaltInvocationError # pylint: disable=import-error @@ -203,7 +203,7 @@ def _parse_template(tmpl_str): ''' tmpl_str = tmpl_str.strip() if tmpl_str.startswith('{'): - tpl = json.loads(tmpl_str) + tpl = salt.utils.json.loads(tmpl_str) else: try: tpl = yaml.load(tmpl_str, Loader=YamlLoader) diff --git a/salt/modules/hipchat.py b/salt/modules/hipchat.py index 1d2bc32399e..661b8f483cd 100644 --- a/salt/modules/hipchat.py +++ b/salt/modules/hipchat.py @@ -30,7 +30,6 @@ Module for sending messages to hipchat. ''' # Import Python Libs from __future__ import absolute_import -import json import logging # Import 3rd-party Libs @@ -41,6 +40,7 @@ from salt.ext.six.moves import range import salt.ext.six.moves.http_client import salt.utils.http +import salt.utils.json # pylint: enable=import-error,no-name-in-module,redefined-builtin @@ -154,7 +154,7 @@ def _query(function, elif api_version == 'v2': headers['Authorization'] = 'Bearer {0}'.format(api_key) if data: - data = json.dumps(data) + data = salt.utils.json.dumps(data) if method == 'POST': headers['Content-Type'] = 'application/json' diff --git a/salt/modules/ifttt.py b/salt/modules/ifttt.py index e6fdb1fa198..8eef64642d2 100644 --- a/salt/modules/ifttt.py +++ b/salt/modules/ifttt.py @@ -14,12 +14,12 @@ Requires an ``api_key`` in ``/etc/salt/minion``: # Import python libs from __future__ import absolute_import, print_function -import json import logging import time # Import salt libs import salt.utils.http +import salt.utils.json log = logging.getLogger(__name__) @@ -89,7 +89,7 @@ def trigger_event(event=None, **kwargs): data['occurredat'] = time.strftime("%B %d, %Y %I:%M%p", time.localtime()) result = _query(event=event, method='POST', - data=json.dumps(data) + data=salt.utils.json.dumps(data) ) if 'status' in result: if result['status'] == 200: diff --git a/salt/modules/influx.py b/salt/modules/influx.py index 7e46d0fa00e..8d1c921f57a 100644 --- a/salt/modules/influx.py +++ b/salt/modules/influx.py @@ -36,10 +36,10 @@ except ImportError: HAS_INFLUXDB = False import collections -import json import logging # Import salt libs +import salt.utils.json from salt.state import STATE_INTERNAL_KEYWORDS as _STATE_INTERNAL_KEYWORDS log = logging.getLogger(__name__) @@ -678,7 +678,7 @@ def _pull_query_results(resultset): for _header, _values in resultset.items(): _header, _group_tags = _header if _group_tags: - _results[_header][json.dumps(_group_tags)] = [_value for _value in _values] + _results[_header][salt.utils.json.dumps(_group_tags)] = [_value for _value in _values] else: _results[_header] = [_value for _value in _values] return dict(sorted(_results.items())) diff --git a/salt/modules/ini_manage.py b/salt/modules/ini_manage.py index 0f1e8bce115..04db37a349f 100644 --- a/salt/modules/ini_manage.py +++ b/salt/modules/ini_manage.py @@ -9,22 +9,19 @@ Edit ini files (for example /etc/sysctl.conf) ''' - -from __future__ import absolute_import, print_function - # Import Python libs -from __future__ import print_function -from __future__ import absolute_import +from __future__ import absolute_import, print_function import os import re -import json # Import Salt libs -from salt.ext import six import salt.utils.files +import salt.utils.json from salt.exceptions import CommandExecutionError from salt.utils.odict import OrderedDict +# Import 3rd-party libs +from salt.ext import six __virtualname__ = 'ini' @@ -349,10 +346,10 @@ class _Section(OrderedDict): def __repr__(self, _repr_running=None): _repr_running = _repr_running or {} super_repr = super(_Section, self).__repr__(_repr_running) - return os.linesep.join((super_repr, json.dumps(self, indent=4))) + return os.linesep.join((super_repr, salt.utils.json.dumps(self, indent=4))) def __str__(self): - return json.dumps(self, indent=4) + return salt.utils.json.dumps(self, indent=4) def __eq__(self, item): return (isinstance(item, self.__class__) and diff --git a/salt/modules/junos.py b/salt/modules/junos.py index f6ae3e4dc42..3dd7822f3b7 100644 --- a/salt/modules/junos.py +++ b/salt/modules/junos.py @@ -17,7 +17,6 @@ from __future__ import absolute_import # Import python libraries import logging -import json import os try: @@ -27,6 +26,7 @@ except ImportError: # Import Salt libs import salt.utils.files +import salt.utils.json from salt.ext import six # Juniper interface libraries @@ -234,7 +234,7 @@ def rpc(cmd=None, dest=None, format='xml', **kwargs): if format == 'text': write_response = reply.text elif format == 'json': - write_response = json.dumps(reply, indent=1) + write_response = salt.utils.json.dumps(reply, indent=1) else: write_response = etree.tostring(reply) with salt.utils.files.fopen(dest, 'w') as fp: diff --git a/salt/modules/k8s.py b/salt/modules/k8s.py index f47b9f1cd15..97bcd513c92 100644 --- a/salt/modules/k8s.py +++ b/salt/modules/k8s.py @@ -18,7 +18,6 @@ from __future__ import absolute_import import os import re -import json import logging as logger import base64 from salt.ext import six @@ -28,6 +27,7 @@ from salt.ext.six.moves.urllib.parse import urlparse as _urlparse # pylint: dis import salt.utils.files import salt.utils.http as http +import salt.utils.json __virtualname__ = 'k8s' @@ -74,12 +74,15 @@ def _kpost(url, data): headers = {"Content-Type": "application/json"} # Make request log.trace("url is: {0}, data is: {1}".format(url, data)) - ret = http.query(url, method='POST', header_dict=headers, data=json.dumps(data)) + ret = http.query(url, + method='POST', + header_dict=headers, + data=salt.utils.json.dumps(data)) # Check requests status if ret.get('error'): return ret else: - return json.loads(ret.get('body')) + return salt.utils.json.loads(ret.get('body')) def _kput(url, data): @@ -88,12 +91,15 @@ def _kput(url, data): # Prepare headers headers = {"Content-Type": "application/json"} # Make request - ret = http.query(url, method='PUT', header_dict=headers, data=json.dumps(data)) + ret = http.query(url, + method='PUT', + header_dict=headers, + data=salt.utils.json.dumps(data)) # Check requests status if ret.get('error'): return ret else: - return json.loads(ret.get('body')) + return salt.utils.json.loads(ret.get('body')) def _kpatch(url, data): @@ -103,13 +109,13 @@ def _kpatch(url, data): headers = {"Content-Type": "application/json-patch+json"} # Make request ret = http.query(url, method='PATCH', header_dict=headers, - data=json.dumps(data)) + data=salt.utils.json.dumps(data)) # Check requests status if ret.get('error'): log.error("Got an error: {0}".format(ret.get("error"))) return ret else: - return json.loads(ret.get('body')) + return salt.utils.json.loads(ret.get('body')) def _kname(obj): @@ -179,7 +185,7 @@ def _get_labels(node, apiserver_url): ret = http.query(url) # Check requests status if 'body' in ret: - ret = json.loads(ret.get('body')) + ret = salt.utils.json.loads(ret.get('body')) elif ret.get('status', 0) == 404: return "Node {0} doesn't exist".format(node) else: @@ -387,7 +393,7 @@ def _get_namespaces(apiserver_url, name=""): # Make request ret = http.query(url) if ret.get("body"): - return json.loads(ret.get("body")) + return salt.utils.json.loads(ret.get("body")) else: return None @@ -493,7 +499,7 @@ def _get_secrets(namespace, name, apiserver_url): # Make request ret = http.query(url) if ret.get("body"): - return json.loads(ret.get("body")) + return salt.utils.json.loads(ret.get("body")) else: return None diff --git a/salt/modules/kapacitor.py b/salt/modules/kapacitor.py index aa09153608f..737720cd419 100644 --- a/salt/modules/kapacitor.py +++ b/salt/modules/kapacitor.py @@ -14,13 +14,12 @@ Kapacitor execution module. .. versionadded:: 2016.11.0 ''' - from __future__ import absolute_import -import json import logging import salt.utils.http +import salt.utils.json import salt.utils.path from salt.utils.decorators import memoize @@ -68,7 +67,7 @@ def get_task(name): if response['status'] == 404: return None - data = json.loads(response['body']) + data = salt.utils.json.loads(response['body']) if version() < '0.13': return { diff --git a/salt/modules/mac_brew.py b/salt/modules/mac_brew.py index a3abc178885..56018727902 100644 --- a/salt/modules/mac_brew.py +++ b/salt/modules/mac_brew.py @@ -13,21 +13,20 @@ from __future__ import absolute_import # Import python libs import copy import functools -import json import logging # Import salt libs import salt.utils.data import salt.utils.functools +import salt.utils.json import salt.utils.path import salt.utils.pkg import salt.utils.versions from salt.exceptions import CommandExecutionError, MinionError -from salt.ext import six -from salt.ext.six.moves import zip # Import third party libs -import json +from salt.ext import six +from salt.ext.six.moves import zip log = logging.getLogger(__name__) @@ -292,7 +291,7 @@ def _info(*pkgs): if brew_result['retcode']: log.error('Failed to get info about packages: {0}'.format(' '.join(pkgs))) return {} - output = json.loads(brew_result['stdout']) + output = salt.utils.json.loads(brew_result['stdout']) return dict(zip(pkgs, output)) @@ -422,7 +421,7 @@ def list_upgrades(refresh=True, **kwargs): # pylint: disable=W0613 ret = {} try: - data = json.loads(res['stdout']) + data = salt.utils.json.loads(res['stdout']) except ValueError as err: msg = 'unable to interpret output from "brew outdated": {0}'.format(err) log.error(msg) diff --git a/salt/modules/mandrill.py b/salt/modules/mandrill.py index f0df1742cba..0e2bb368026 100644 --- a/salt/modules/mandrill.py +++ b/salt/modules/mandrill.py @@ -19,10 +19,12 @@ In the minion configuration file, the following block is required: ''' from __future__ import absolute_import -# import python std lib -import json +# Import Python libs import logging +# Import Salt libs +import salt.utils.json + # import third party try: import requests @@ -115,7 +117,7 @@ def _http_request(url, log.debug('Querying {}'.format(url)) req = session.post(url, headers=headers, - data=json.dumps(data)) + data=salt.utils.json.dumps(data)) req_body = req.json() ret = _default_ret() log.debug('Status code: %d', req.status_code) diff --git a/salt/modules/marathon.py b/salt/modules/marathon.py index ad53c077472..1077823a92f 100644 --- a/salt/modules/marathon.py +++ b/salt/modules/marathon.py @@ -8,9 +8,9 @@ Currently this only works when run through a proxy minion. ''' from __future__ import absolute_import -import json import logging import salt.utils.http +import salt.utils.json import salt.utils.platform from salt.exceptions import get_error_message @@ -114,7 +114,7 @@ def update_app(id, config): # mirror marathon-ui handling for uris deprecation (see # mesosphere/marathon-ui#594 for more details) config.pop('fetch', None) - data = json.dumps(config) + data = salt.utils.json.dumps(config) try: response = salt.utils.http.query( "{0}/v2/apps/{1}?force=true".format(_base_url(), id), diff --git a/salt/modules/mattermost.py b/salt/modules/mattermost.py index 7617d863b36..f1be020aead 100644 --- a/salt/modules/mattermost.py +++ b/salt/modules/mattermost.py @@ -16,16 +16,12 @@ Module for sending messages to Mattermost ''' # Import Python libs -from __future__ import absolute_import -import json +from __future__ import absolute_import, print_function, unicode_literals import logging -# Import 3rd-party libs -# pylint: disable=import-error,no-name-in-module,redefined-builtin - +# Import Salt libs +import salt.utils.json import salt.utils.mattermost -# pylint: enable=import-error,no-name-in-module - from salt.exceptions import SaltInvocationError log = logging.getLogger(__name__) @@ -136,9 +132,8 @@ def post_message(message, if username: parameters['username'] = username parameters['text'] = '```' + message + '```' # pre-formatted, fixed-width text - log.debug('Parameters: {0}'.format(parameters)) - result = salt.utils.mattermost.query(api_url=api_url, - hook=hook, - data='payload={0}'.format(json.dumps(parameters))) + log.debug('Parameters: %s', parameters) + data = str('payload={0}').format(salt.utils.json.dumps(parameters)) # future lint: disable=blacklisted-function + result = salt.utils.mattermost.query(api_url=api_url, hook=hook, data=data) return bool(result) diff --git a/salt/modules/mongodb.py b/salt/modules/mongodb.py index 83f5430b3be..688e46d1739 100644 --- a/salt/modules/mongodb.py +++ b/salt/modules/mongodb.py @@ -17,10 +17,10 @@ from __future__ import absolute_import # Import python libs import logging -import json import re # Import salt libs +import salt.utils.json from salt.utils.versions import LooseVersion as _LooseVersion from salt.exceptions import get_error_message as _get_error_message @@ -80,7 +80,7 @@ def _to_dict(objects): """ try: if isinstance(objects, six.string_types): - objects = json.loads(objects) + objects = salt.utils.json.loads(objects) except ValueError as err: log.error("Could not parse objects: %s", err) raise err diff --git a/salt/modules/msteams.py b/salt/modules/msteams.py index 565082c2910..a171591b784 100644 --- a/salt/modules/msteams.py +++ b/salt/modules/msteams.py @@ -10,19 +10,17 @@ Module for sending messages to MS Teams msteams: hook_url: https://outlook.office.com/webhook/837 ''' - - # Import Python libs from __future__ import absolute_import -import json import logging -# Import 3rd-party libs -# pylint: disable=import-error,no-name-in-module,redefined-builtin -import salt.ext.six.moves.http_client - +# Import Salt libs +import salt.utils.json from salt.exceptions import SaltInvocationError +# Import 3rd-party libs +import salt.ext.six.moves.http_client # pylint: disable=import-error,no-name-in-module,redefined-builtin + log = logging.getLogger(__name__) __virtualname__ = 'msteams' @@ -78,7 +76,10 @@ def post_card(message, "themeColor": theme_color } - result = salt.utils.http.query(hook_url, method='POST', data=json.dumps(payload), status=True) + result = salt.utils.http.query(hook_url, + method='POST', + data=salt.utils.json.dumps(payload), + status=True) if result['status'] <= 201: return True diff --git a/salt/modules/npm.py b/salt/modules/npm.py index 21e34427b89..ae362fa73a3 100644 --- a/salt/modules/npm.py +++ b/salt/modules/npm.py @@ -9,10 +9,10 @@ except ImportError: from pipes import quote as _cmd_quote # Import python libs -import json import logging # Import salt libs +import salt.utils.json import salt.utils.path import salt.utils.user import salt.modules.cmdmod @@ -169,7 +169,7 @@ def install(pkg=None, # npm >1.2.21 is putting the output to stderr even though retcode is 0 npm_output = result['stdout'] or result['stderr'] try: - return json.loads(npm_output) + return salt.utils.json.loads(npm_output) except ValueError: pass @@ -192,7 +192,7 @@ def _extract_json(npm_output): while lines and (lines[0].startswith('[fsevents]') or lines[0].startswith('Pass ')): lines = lines[1:] try: - return json.loads(''.join(lines)) + return salt.utils.json.loads(''.join(lines)) except ValueError: pass return None @@ -322,7 +322,7 @@ def list_(pkg=None, dir=None, runas=None, env=None, depth=None): if result['retcode'] != 0 and result['stderr']: raise CommandExecutionError(result['stderr']) - return json.loads(result['stdout']).get('dependencies', {}) + return salt.utils.json.loads(result['stdout']).get('dependencies', {}) def cache_clean(path=None, runas=None, env=None, force=False): diff --git a/salt/modules/opsgenie.py b/salt/modules/opsgenie.py index acfb6d37e5d..1f161c3521c 100644 --- a/salt/modules/opsgenie.py +++ b/salt/modules/opsgenie.py @@ -22,12 +22,12 @@ Module for sending data to OpsGenie ''' # Import Python libs from __future__ import absolute_import -import json import logging import requests # Import Salt libs import salt.exceptions +import salt.utils.json API_ENDPOINT = "https://api.opsgenie.com/v1/json/saltstack?apiKey=" @@ -93,6 +93,8 @@ def post_data(api_key=None, name='OpsGenie Execution Module', reason=None, log.debug('Below data will be posted:\n' + str(data)) log.debug('API Key:' + api_key + '\t API Endpoint:' + API_ENDPOINT) - response = requests.post(url=API_ENDPOINT + api_key, data=json.dumps(data), - headers={'Content-Type': 'application/json'}) + response = requests.post( + url=API_ENDPOINT + api_key, + data=salt.utils.json.dumps(data), + headers={'Content-Type': 'application/json'}) return response.status_code, response.text diff --git a/salt/modules/osquery.py b/salt/modules/osquery.py index 403d70bb6ff..093b70b3edc 100644 --- a/salt/modules/osquery.py +++ b/salt/modules/osquery.py @@ -7,13 +7,13 @@ Support for OSQuery - https://osquery.io. from __future__ import absolute_import # Import python libs -import json +import logging # Import Salt libs +import salt.utils.json import salt.utils.path import salt.utils.platform -import logging log = logging.getLogger(__name__) @@ -41,7 +41,7 @@ def _table_attrs(table): res = __salt__['cmd.run_all'](cmd) if res['retcode'] == 0: attrs = [] - text = json.loads(res['stdout']) + text = salt.utils.json.loads(res['stdout']) for item in text: attrs.append(item['name']) return attrs @@ -62,7 +62,7 @@ def _osquery(sql, format='json'): ret['result'] = False ret['error'] = res['stderr'] else: - ret['data'] = json.loads(res['stdout']) + ret['data'] = salt.utils.json.loads(res['stdout']) return ret diff --git a/salt/modules/pagerduty.py b/salt/modules/pagerduty.py index 667f64f3ac0..5979ea98c90 100644 --- a/salt/modules/pagerduty.py +++ b/salt/modules/pagerduty.py @@ -20,10 +20,10 @@ from __future__ import absolute_import # Import python libs import yaml -import json # Import salt libs import salt.utils.functools +import salt.utils.json import salt.utils.pagerduty from salt.ext.six import string_types @@ -181,7 +181,7 @@ def create_event(service_key=None, description=None, details=None, if isinstance(details, string_types): details = {'details': details} - ret = json.loads(salt.utils.pagerduty.query( + ret = salt.utils.json.loads(salt.utils.pagerduty.query( method='POST', profile_dict=__salt__['config.option'](profile), api_key=service_key, diff --git a/salt/modules/pagerduty_util.py b/salt/modules/pagerduty_util.py index c152e8f3119..58800eb1543 100644 --- a/salt/modules/pagerduty_util.py +++ b/salt/modules/pagerduty_util.py @@ -20,8 +20,8 @@ For PagerDuty API details, see https://developer.pagerduty.com/documentation/res ''' from __future__ import absolute_import -import json import requests +import salt.utils.json def __virtual__(): @@ -164,7 +164,7 @@ def _query(method='GET', profile=None, url=None, path='api/v1', url, headers=headers, params=params, - data=json.dumps(data), + data=salt.utils.json.dumps(data), verify=verify_ssl ) @@ -185,7 +185,7 @@ def _query(method='GET', profile=None, url=None, path='api/v1', url, headers=headers, params=params, - data=data, # must not use json.dumps() or offset/limit get lost + data=data, # Already serialized above, don't do it again verify=verify_ssl).json() offset = next_page_results['offset'] limit = next_page_results['limit'] diff --git a/salt/modules/pip.py b/salt/modules/pip.py index cd9885bf761..5e52968e0f9 100644 --- a/salt/modules/pip.py +++ b/salt/modules/pip.py @@ -83,11 +83,11 @@ import shutil import logging import sys import tempfile -import json # Import Salt libs import salt.utils.data import salt.utils.files +import salt.utils.json import salt.utils.locales import salt.utils.platform import salt.utils.url @@ -1151,7 +1151,7 @@ def list_upgrades(bin_env=None, packages = {} try: - json_results = json.loads(result['stdout']) + json_results = salt.utils.json.loads(result['stdout']) for json_result in json_results: packages[json_result['name']] = json_result['latest_version'] except ValueError: diff --git a/salt/modules/rabbitmq.py b/salt/modules/rabbitmq.py index 23fe2d3ea38..db77d957c87 100644 --- a/salt/modules/rabbitmq.py +++ b/salt/modules/rabbitmq.py @@ -4,26 +4,25 @@ Module to provide RabbitMQ compatibility to Salt. Todo: A lot, need to add cluster support, logging, and minion configuration data. ''' +# Import Python libs from __future__ import absolute_import - -# Import python libs -import json -import re import logging import os -import os.path import random +import re import string -# Import salt libs +# Import Salt libs import salt.utils.itertools +import salt.utils.json import salt.utils.path import salt.utils.platform import salt.utils.user +from salt.exceptions import CommandExecutionError, SaltInvocationError + +# Import 3rd-party libs from salt.ext import six -from salt.exceptions import SaltInvocationError from salt.ext.six.moves import range -from salt.exceptions import CommandExecutionError log = logging.getLogger(__name__) @@ -866,7 +865,7 @@ def set_policy(vhost, name, pattern, definition, priority=None, apply_to=None, r if runas is None and not salt.utils.platform.is_windows(): runas = salt.utils.user.get_user() if isinstance(definition, dict): - definition = json.dumps(definition) + definition = salt.utils.json.dumps(definition) if not isinstance(definition, six.string_types): raise SaltInvocationError( 'The \'definition\' argument must be a dictionary or JSON string' diff --git a/salt/modules/rallydev.py b/salt/modules/rallydev.py index 879ff96f2fe..c0dd30075f3 100644 --- a/salt/modules/rallydev.py +++ b/salt/modules/rallydev.py @@ -15,12 +15,12 @@ Requires a ``username`` and a ``password`` in ``/etc/salt/minion``: # Import python libs from __future__ import absolute_import, print_function -import json import logging # Import salt libs from salt.exceptions import SaltInvocationError import salt.utils.http +import salt.utils.json log = logging.getLogger(__name__) @@ -202,7 +202,7 @@ def update_item(name, id_, field=None, value=None, postdata=None): action=name, command=id_, method='POST', - data=json.dumps(postdata), + data=salt.utils.json.dumps(postdata), ) return result diff --git a/salt/modules/random_org.py b/salt/modules/random_org.py index c5a787793db..e76d7dfdb4f 100644 --- a/salt/modules/random_org.py +++ b/salt/modules/random_org.py @@ -22,16 +22,15 @@ from __future__ import absolute_import import logging # Import 3rd-party libs -import json # pylint: disable=import-error,no-name-in-module,redefined-builtin import salt.ext.six import salt.ext.six.moves.http_client from salt.ext.six.moves.urllib.parse import urljoin as _urljoin +# pylint: enable=import-error,no-name-in-module,redefined-builtin # Import salt libs import salt.utils.http - -# pylint: enable=import-error,no-name-in-module,redefined-builtin +import salt.utils.json log = logging.getLogger(__name__) __virtualname__ = 'random_org' @@ -97,7 +96,7 @@ def _query(api_version=None, data=None): api_url = 'https://api.random.org/' base_url = _urljoin(api_url, 'json-rpc/' + str(api_version) + '/invoke') - data = json.dumps(data) + data = salt.utils.json.dumps(data) result = salt.utils.http.query( base_url, diff --git a/salt/modules/saltcloudmod.py b/salt/modules/saltcloudmod.py index f061cdc6363..abb4f374a11 100644 --- a/salt/modules/saltcloudmod.py +++ b/salt/modules/saltcloudmod.py @@ -2,13 +2,12 @@ ''' Control a salt cloud system ''' -from __future__ import absolute_import - # Import python libs -import json +from __future__ import absolute_import # Import salt libs import salt.utils.data +import salt.utils.json HAS_CLOUD = False try: @@ -43,7 +42,7 @@ def create(name, profile): cmd = 'salt-cloud --out json -p {0} {1}'.format(profile, name) out = __salt__['cmd.run_stdout'](cmd, python_shell=False) try: - ret = json.loads(out, object_hook=salt.utils.data.encode_dict) + ret = salt.utils.json.loads(out, object_hook=salt.utils.data.encode_dict) except ValueError: ret = {} return ret diff --git a/salt/modules/serverdensity_device.py b/salt/modules/serverdensity_device.py index fcbabcdfb38..83647d1c3bf 100644 --- a/salt/modules/serverdensity_device.py +++ b/salt/modules/serverdensity_device.py @@ -8,17 +8,18 @@ Wrapper around Server Density API # Import Python libs from __future__ import absolute_import -import json import logging import os import tempfile +# Import Salt libs +import salt.utils.json +from salt.exceptions import CommandExecutionError + # Import 3rd-party libs from salt.ext import six from salt.ext.six.moves import map # pylint: disable=import-error,no-name-in-module,redefined-builtin -from salt.exceptions import CommandExecutionError - try: import requests ENABLED = True @@ -99,7 +100,7 @@ def create(name, **params): log.debug('Server Density API Response content: {0}'.format(api_response.content)) if api_response.status_code == 200: try: - return json.loads(api_response.content) + return salt.utils.json.loads(api_response.content) except ValueError: log.error('Could not parse API Response content: {0}'.format(api_response.content)) raise CommandExecutionError( @@ -130,7 +131,7 @@ def delete(device_id): log.debug('Server Density API Response content: {0}'.format(api_response.content)) if api_response.status_code == 200: try: - return json.loads(api_response.content) + return salt.utils.json.loads(api_response.content) except ValueError: log.error('Could not parse API Response content: {0}'.format(api_response.content)) raise CommandExecutionError( @@ -172,13 +173,13 @@ def ls(**params): api_response = requests.get( 'https://api.serverdensity.io/inventory/{0}'.format(endpoint), - params={'token': get_sd_auth('api_token'), 'filter': json.dumps(params)} + params={'token': get_sd_auth('api_token'), 'filter': salt.utils.json.dumps(params)} ) log.debug('Server Density API Response: {0}'.format(api_response)) log.debug('Server Density API Response content: {0}'.format(api_response.content)) if api_response.status_code == 200: try: - return json.loads(api_response.content) + return salt.utils.json.loads(api_response.content) except ValueError: log.error( 'Could not parse Server Density API Response content: {0}' @@ -217,7 +218,7 @@ def update(device_id, **params): log.debug('Server Density API Response content: {0}'.format(api_response.content)) if api_response.status_code == 200: try: - return json.loads(api_response.content) + return salt.utils.json.loads(api_response.content) except ValueError: log.error( 'Could not parse Server Density API Response content: {0}' diff --git a/salt/modules/slack_notify.py b/salt/modules/slack_notify.py index 56c33480a49..9742f0744d9 100644 --- a/salt/modules/slack_notify.py +++ b/salt/modules/slack_notify.py @@ -18,21 +18,21 @@ Module for sending messages to Slack # Import Python libs from __future__ import absolute_import -import json import logging +# Import Salt libs +import salt.utils.json +import salt.utils.slack +from salt.exceptions import SaltInvocationError + # Import 3rd-party libs # pylint: disable=import-error,no-name-in-module,redefined-builtin from salt.ext.six.moves.urllib.parse import urlencode as _urlencode from salt.ext.six.moves.urllib.parse import urljoin as _urljoin from salt.ext.six.moves import range import salt.ext.six.moves.http_client - -import salt.utils.slack # pylint: enable=import-error,no-name-in-module -from salt.exceptions import SaltInvocationError - log = logging.getLogger(__name__) __virtualname__ = 'slack' @@ -308,7 +308,7 @@ def call_hook(message, data = _urlencode( { - 'payload': json.dumps(payload, ensure_ascii=False) + 'payload': salt.utils.json.dumps(payload) } ) result = salt.utils.http.query(url, method='POST', data=data, status=True) diff --git a/salt/modules/smartos_imgadm.py b/salt/modules/smartos_imgadm.py index 5b13ce3c00c..90ad443dbeb 100644 --- a/salt/modules/smartos_imgadm.py +++ b/salt/modules/smartos_imgadm.py @@ -6,9 +6,9 @@ from __future__ import absolute_import # Import Python libs import logging -import json # Import Salt libs +import salt.utils.json import salt.utils.path import salt.utils.platform import salt.utils.decorators as decorators @@ -144,7 +144,7 @@ def avail(search=None, verbose=False): ret['Error'] = _exit_status(retcode) return ret - for image in json.loads(res['stdout']): + for image in salt.utils.json.loads(res['stdout']): if image['manifest']['disabled'] or not image['manifest']['public']: continue if search and search not in image['manifest']['name']: @@ -178,7 +178,7 @@ def list_installed(verbose=False): ret['Error'] = _exit_status(retcode) return ret - for image in json.loads(res['stdout']): + for image in salt.utils.json.loads(res['stdout']): result[image['manifest']['uuid']] = _parse_image_meta(image, verbose) return result @@ -205,7 +205,7 @@ def show(uuid): if retcode != 0: ret['Error'] = _exit_status(retcode) return ret - ret = json.loads(res['stdout']) + ret = salt.utils.json.loads(res['stdout']) return ret @@ -230,7 +230,7 @@ def get(uuid): if retcode != 0: ret['Error'] = _exit_status(retcode) return ret - ret = json.loads(res['stdout']) + ret = salt.utils.json.loads(res['stdout']) return ret diff --git a/salt/modules/smartos_vmadm.py b/salt/modules/smartos_vmadm.py index ec5bb005840..19ad981f071 100644 --- a/salt/modules/smartos_vmadm.py +++ b/salt/modules/smartos_vmadm.py @@ -6,7 +6,6 @@ from __future__ import absolute_import # Import Python libs import logging -import json import os try: from shlex import quote as _quote_args # pylint: disable=E0611 @@ -17,8 +16,10 @@ except ImportError: import salt.utils.args import salt.utils.decorators as decorators import salt.utils.files +import salt.utils.json import salt.utils.path import salt.utils.platform +import salt.utils.stringutils from salt.utils.odict import OrderedDict # Import 3rd-party libs @@ -96,7 +97,7 @@ def _create_update_from_file(mode='create', uuid=None, path=None): ret['Error'] = _exit_status(retcode) if 'stderr' in res: if res['stderr'][0] == '{': - ret['Error'] = json.loads(res['stderr']) + ret['Error'] = salt.utils.json.loads(res['stderr']) else: ret['Error'] = res['stderr'] return ret @@ -113,7 +114,7 @@ def _create_update_from_file(mode='create', uuid=None, path=None): ret['Error'] = _exit_status(retcode) if 'stderr' in res: if res['stderr'][0] == '{': - ret['Error'] = json.loads(res['stderr']) + ret['Error'] = salt.utils.json.loads(res['stderr']) else: ret['Error'] = res['stderr'] return ret @@ -133,7 +134,7 @@ def _create_update_from_cfg(mode='create', uuid=None, vmcfg=None): # write json file vmadm_json_file = __salt__['temp.file'](prefix='vmadm-') with salt.utils.files.fopen(vmadm_json_file, 'w') as vmadm_json: - vmadm_json.write(json.dumps(vmcfg)) + salt.utils.json.dump(vmcfg, vmadm_json) # vmadm validate create|update [-f ] cmd = '{vmadm} validate {mode} {brand} -f {vmadm_json_file}'.format( @@ -148,7 +149,7 @@ def _create_update_from_cfg(mode='create', uuid=None, vmcfg=None): ret['Error'] = _exit_status(retcode) if 'stderr' in res: if res['stderr'][0] == '{': - ret['Error'] = json.loads(res['stderr']) + ret['Error'] = salt.utils.json.loads(res['stderr']) else: ret['Error'] = res['stderr'] return ret @@ -165,7 +166,7 @@ def _create_update_from_cfg(mode='create', uuid=None, vmcfg=None): ret['Error'] = _exit_status(retcode) if 'stderr' in res: if res['stderr'][0] == '{': - ret['Error'] = json.loads(res['stderr']) + ret['Error'] = salt.utils.json.loads(res['stderr']) else: ret['Error'] = res['stderr'] return ret @@ -404,7 +405,7 @@ def lookup(search=None, order=None, one=False): if one: result = res['stdout'] else: - for vm in json.loads(res['stdout']): + for vm in salt.utils.json.loads(res['stdout']): result.append(vm) return result @@ -525,7 +526,7 @@ def get(vm, key='uuid'): if retcode != 0: ret['Error'] = res['stderr'] if 'stderr' in res else _exit_status(retcode) return ret - return json.loads(res['stdout']) + return salt.utils.json.loads(res['stdout']) def info(vm, info_type='all', key='uuid'): @@ -570,7 +571,7 @@ def info(vm, info_type='all', key='uuid'): if retcode != 0: ret['Error'] = res['stderr'] if 'stderr' in res else _exit_status(retcode) return ret - return json.loads(res['stdout']) + return salt.utils.json.loads(res['stdout']) def create_snapshot(vm, name, key='uuid'): @@ -756,10 +757,10 @@ def reprovision(vm, image, key='uuid'): ret['Error'] = 'Image ({0}) is not present on this host'.format(image) return ret # vmadm reprovision [-f ] - cmd = 'echo {image} | {vmadm} reprovision {uuid}'.format( - vmadm=vmadm, - uuid=vm, - image=_quote_args(json.dumps({'image_uuid': image})) + cmd = str('echo {image} | {vmadm} reprovision {uuid}').format( # future lint: disable=blacklisted-function + vmadm=salt.utils.stringutils.to_str(vmadm), + uuid=salt.utils.stringutils.to_str(vm), + image=_quote_args(salt.utils.json.dumps({'image_uuid': image})) ) res = __salt__['cmd.run_all'](cmd, python_shell=True) retcode = res['retcode'] diff --git a/salt/modules/solr.py b/salt/modules/solr.py index 4306e77af86..1cea9465d42 100644 --- a/salt/modules/solr.py +++ b/salt/modules/solr.py @@ -61,7 +61,6 @@ verbose : True # Import python Libs from __future__ import absolute_import -import json import os # Import 3rd-party libs @@ -77,6 +76,7 @@ from salt.ext.six.moves.urllib.request import ( # pylint: enable=no-name-in-module,import-error # Import salt libs +import salt.utils.json import salt.utils.path # ######################### PRIVATE METHODS ############################## @@ -258,7 +258,7 @@ def _auth(url): def _http_request(url, request_timeout=None): ''' PRIVATE METHOD - Uses json.load to fetch the JSON results from the solr API. + Uses salt.utils.json.load to fetch the JSON results from the solr API. url : str a complete URL that can be passed to urllib.open @@ -274,10 +274,8 @@ def _http_request(url, request_timeout=None): try: request_timeout = __salt__['config.option']('solr.request_timeout') - if request_timeout is None: - data = json.load(_urlopen(url)) - else: - data = json.load(_urlopen(url, timeout=request_timeout)) + kwargs = {} if request_timeout is None else {'timeout': request_timeout} + data = salt.utils.json.load(_urlopen(url, **kwargs)) return _get_return_dict(True, data, []) except Exception as err: return _get_return_dict(False, {}, ["{0} : {1}".format(url, err)]) diff --git a/salt/modules/state.py b/salt/modules/state.py index 48263cecd22..a8285035647 100644 --- a/salt/modules/state.py +++ b/salt/modules/state.py @@ -14,7 +14,6 @@ states themselves. # Import python libs from __future__ import absolute_import, print_function import fnmatch -import json import logging import os import shutil @@ -34,8 +33,10 @@ import salt.utils.files import salt.utils.functools import salt.utils.hashutils import salt.utils.jid +import salt.utils.json import salt.utils.platform import salt.utils.state +import salt.utils.stringutils import salt.utils.url import salt.utils.versions from salt.exceptions import CommandExecutionError, SaltInvocationError @@ -1996,7 +1997,7 @@ def pkg(pkg_path, s_pkg.close() lowstate_json = os.path.join(root, 'lowstate.json') with salt.utils.files.fopen(lowstate_json, 'r') as fp_: - lowstate = json.load(fp_, object_hook=salt.utils.data.encode_dict) + lowstate = salt.utils.json.load(fp_) # Check for errors in the lowstate for chunk in lowstate: if not isinstance(chunk, dict): @@ -2004,14 +2005,14 @@ def pkg(pkg_path, pillar_json = os.path.join(root, 'pillar.json') if os.path.isfile(pillar_json): with salt.utils.files.fopen(pillar_json, 'r') as fp_: - pillar_override = json.load(fp_) + pillar_override = salt.utils.json.load(fp_) else: pillar_override = None roster_grains_json = os.path.join(root, 'roster_grains.json') if os.path.isfile(roster_grains_json): with salt.utils.files.fopen(roster_grains_json, 'r') as fp_: - roster_grains = json.load(fp_, object_hook=salt.utils.data.encode_dict) + roster_grains = salt.utils.json.load(fp_, object_hook=salt.utils.data.encode_dict) if os.path.isfile(roster_grains_json): popts['grains'] = roster_grains @@ -2237,12 +2238,15 @@ def event(tagmatch='*', if fnmatch.fnmatch(ret['tag'], tagmatch): if not quiet: - print('{0}\t{1}'.format( - ret['tag'], - json.dumps( - ret['data'], - sort_keys=pretty, - indent=None if not pretty else 4))) + salt.utils.stringutils.print_cli( + str('{0}\t{1}').format( # future lint: blacklisted-function + salt.utils.stringutils.to_str(ret['tag']), + salt.utils.json.dumps( + ret['data'], + sort_keys=pretty, + indent=None if not pretty else 4) + ) + ) sys.stdout.flush() count -= 1 diff --git a/salt/modules/swarm.py b/salt/modules/swarm.py index 2d4297531ab..7c0839146b9 100644 --- a/salt/modules/swarm.py +++ b/salt/modules/swarm.py @@ -24,7 +24,7 @@ More information: https://docker-py.readthedocs.io/en/stable/ """ # Import python libraries from __future__ import absolute_import -import json +import salt.utils.json try: import docker @@ -238,8 +238,8 @@ def swarm_service_info(service_name=str): salt_return = {} client = docker.APIClient(base_url='unix://var/run/docker.sock') service = client.inspect_service(service=service_name) - getdata = json.dumps(service) - dump = json.loads(getdata) + getdata = salt.utils.json.dumps(service) + dump = salt.utils.json.loads(getdata) version = dump['Version']['Index'] name = dump['Spec']['Name'] network_mode = dump['Spec']['EndpointSpec']['Mode'] @@ -316,8 +316,8 @@ def node_ls(server=str): salt_return = {} client = docker.APIClient(base_url='unix://var/run/docker.sock') service = client.nodes(filters=({'name': server})) - getdata = json.dumps(service) - dump = json.loads(getdata) + getdata = salt.utils.json.dumps(service) + dump = salt.utils.json.loads(getdata) for items in dump: docker_version = items['Description']['Engine']['EngineVersion'] platform = items['Description']['Platform'] diff --git a/salt/modules/telemetry.py b/salt/modules/telemetry.py index 6c98266f8d8..2694fee718f 100644 --- a/salt/modules/telemetry.py +++ b/salt/modules/telemetry.py @@ -23,11 +23,16 @@ https://github.com/mongolab/mongolab-telemetry-api-docs/blob/master/alerts.md :depends: requests ''' +# Import Python libs from __future__ import absolute_import -from salt._compat import string_types -import json import logging +# Import Salt libs +import salt.utils.json + +# Import 3rd-party libs +from salt.ext import six + log = logging.getLogger(__name__) # Import third party libs @@ -57,7 +62,7 @@ def _auth(api_key=None, profile='telemetry'): if api_key is None and profile is None: raise Exception("Missing api_key and profile") if profile: - if isinstance(profile, string_types): + if isinstance(profile, six.string_types): _profile = __salt__['config.option'](profile) elif isinstance(profile, dict): _profile = profile @@ -184,7 +189,7 @@ def get_notification_channel_id(notify_channel, profile="telemetry"): "name": notify_channel[:notify_channel.find('@')] + 'EscalationPolicy', "email": notify_channel } - response = requests.post(post_url, data=json.dumps(data), headers=auth) + response = requests.post(post_url, data=salt.utils.json.dumps(data), headers=auth) if response.status_code == 200: log.info("Successfully created EscalationPolicy {0} with EmailNotificationChannel {1}" .format(data.get('name'), notify_channel)) @@ -224,7 +229,7 @@ def get_alarms(deployment_id, profile="telemetry"): return 'No alarms defined for deployment: {0}'.format(deployment_id) else: # Non 200 response, sent back the error response' - return {'err_code': response.status_code, 'err_msg': json.loads(response.text).get('err', '')} + return {'err_code': response.status_code, 'err_msg': salt.utils.json.loads(response.text).get('err', '')} def create_alarm(deployment_id, metric_name, data, api_key=None, profile="telemetry"): @@ -259,19 +264,24 @@ def create_alarm(deployment_id, metric_name, data, api_key=None, profile="teleme } try: - response = requests.post(request_uri, data=json.dumps(post_body), headers=auth) + response = requests.post(request_uri, data=salt.utils.json.dumps(post_body), headers=auth) except requests.exceptions.RequestException as e: # TODO: May be we should retry? log.error(str(e)) if response.status_code >= 200 and response.status_code < 300: # update cache - log.info("Created alarm on metric: {0} in deployment: {1}".format(metric_name, deployment_id)) - log.debug("Updating cache for metric {0} in deployment {1}: {2}".format(metric_name, deployment_id, response.json())) + log.info('Created alarm on metric: %s in deployment: %s', metric_name, deployment_id) + log.debug('Updating cache for metric %s in deployment %s: %s', + metric_name, deployment_id, response.json()) _update_cache(deployment_id, metric_name, response.json()) else: - log.error("Failed to create alarm on metric: {0} in deployment {1}: payload: {2}". - format(metric_name, deployment_id, json.dumps(post_body))) + log.error( + str('Failed to create alarm on metric: %s in deployment %s: payload: %s'), # future lint: disable=blacklisted-function + salt.utils.stringutils.to_str(metric_name), + salt.utils.stringutils.to_str(deployment_id), + salt.utils.json.dumps(post_body) + ) return response.status_code >= 200 and response.status_code < 300, response.json() @@ -308,19 +318,26 @@ def update_alarm(deployment_id, metric_name, data, api_key=None, profile="teleme } try: - response = requests.put(request_uri, data=json.dumps(post_body), headers=auth) + response = requests.put(request_uri, data=salt.utils.json.dumps(post_body), headers=auth) except requests.exceptions.RequestException as e: - log.error("Update failed {0}" .format(str(e))) + log.error('Update failed: %s', e) return False, str(e) if response.status_code >= 200 and response.status_code < 300: # Also update cache - log.debug("Updating cache for metric {0} in deployment {1}: {2}".format(metric_name, deployment_id, response.json())) + log.debug('Updating cache for metric %s in deployment %s: %s', + metric_name, deployment_id, response.json()) _update_cache(deployment_id, metric_name, response.json()) - log.info("Updated alarm on metric: {0} in deployment: {1}".format(metric_name, deployment_id)) + log.info('Updated alarm on metric: %s in deployment: %s', metric_name, deployment_id) return True, response.json() - err_msg = "Failed to create alarm on metric: {0} in deployment {1}: payload: {2}".format(metric_name, deployment_id, json.dumps(post_body)) + err_msg = str( # future lint: disable=blacklisted-function + 'Failed to create alarm on metric: {0} in deployment: {1} ' + 'payload: {2}').format( + salt.utils.stringutils.to_str(metric_name), + salt.utils.stringutils.to_str(deployment_id), + salt.utils.json.dumps(post_body) + ) log.error(err_msg) return False, err_msg diff --git a/salt/modules/travisci.py b/salt/modules/travisci.py index 3657c590064..885b4c12579 100644 --- a/salt/modules/travisci.py +++ b/salt/modules/travisci.py @@ -8,8 +8,6 @@ Commands for working with travisci. # Import python libraries from __future__ import absolute_import import base64 -import json - try: import OpenSSL @@ -19,6 +17,7 @@ except ImportError: HAS_OPENSSL = False # Import Salt libraries +import salt.utils.json from salt.utils.versions import LooseVersion as _LooseVersion from salt.ext.six.moves.urllib.parse import parse_qs # pylint: disable=import-error,no-name-in-module @@ -65,7 +64,7 @@ def verify_webhook(signature, body): signature = base64.b64decode(signature) # parse the urlencoded payload from travis - payload = json.loads(parse_qs(body)['payload'][0]) + payload = salt.utils.json.loads(parse_qs(body)['payload'][0]) try: OpenSSL.crypto.verify(certificate, signature, payload, str('sha1')) diff --git a/salt/modules/uwsgi.py b/salt/modules/uwsgi.py index ec361cf69bf..8563c079cb8 100644 --- a/salt/modules/uwsgi.py +++ b/salt/modules/uwsgi.py @@ -6,12 +6,11 @@ uWSGI stats server https://uwsgi-docs.readthedocs.io/en/latest/StatsServer.html :maturity: new :platform: all ''' +# Import Python libs from __future__ import absolute_import -# Import Python libs -import json - # Import Salt libs +import salt.utils.json import salt.utils.path @@ -43,4 +42,4 @@ def stats(socket): cmd = ['uwsgi', '--connect-and-read', '{0}'.format(socket)] out = __salt__['cmd.run'](cmd, python_shell=False) - return json.loads(out) + return salt.utils.json.loads(out) diff --git a/salt/modules/victorops.py b/salt/modules/victorops.py index b57282cc73a..558d345a9cf 100644 --- a/salt/modules/victorops.py +++ b/salt/modules/victorops.py @@ -15,13 +15,13 @@ Requires an ``api_key`` in ``/etc/salt/minion``: # Import python libs from __future__ import absolute_import, print_function import datetime -import json import logging import time # Import salt libs from salt.exceptions import SaltInvocationError import salt.utils.http +import salt.utils.json log = logging.getLogger(__name__) @@ -205,7 +205,7 @@ def create_event(message_type=None, routing_key='everybody', **kwargs): status, result = _query(action='alert', routing_key=routing_key, - data=json.dumps(data), + data=salt.utils.json.dumps(data), method='POST' ) return result diff --git a/salt/modules/win_dsc.py b/salt/modules/win_dsc.py index 64fec5d1b7e..cdd76c61804 100644 --- a/salt/modules/win_dsc.py +++ b/salt/modules/win_dsc.py @@ -19,10 +19,10 @@ from __future__ import absolute_import # Import Python libs import logging -import json import os # Import Salt libs +import salt.utils.json import salt.utils.platform import salt.utils.versions from salt.exceptions import CommandExecutionError, SaltInvocationError @@ -77,7 +77,7 @@ def _pshell(cmd, cwd=None, json_depth=2): 'Issue executing PowerShell {0}'.format(cmd), info=results) try: - ret = json.loads(results['stdout'], strict=False) + ret = salt.utils.json.loads(results['stdout'], strict=False) except ValueError: raise CommandExecutionError( 'No JSON results from PowerShell', info=results) diff --git a/salt/modules/win_iis.py b/salt/modules/win_iis.py index 02b75016b2b..ba71228db39 100644 --- a/salt/modules/win_iis.py +++ b/salt/modules/win_iis.py @@ -9,15 +9,14 @@ Microsoft IIS site management via WebAdministration powershell module .. versionadded:: 2016.3.0 ''' - # Import python libs from __future__ import absolute_import, print_function, unicode_literals -import json +import decimal import logging import os -import decimal # Import salt libs +import salt.utils.json import salt.utils.platform from salt.ext.six.moves import range from salt.exceptions import SaltInvocationError, CommandExecutionError @@ -93,7 +92,7 @@ def _list_certs(certificate_store='My'): cmd_ret = _srvmgr(cmd=ps_cmd, return_json=True) try: - items = json.loads(cmd_ret['stdout'], strict=False) + items = salt.utils.json.loads(cmd_ret['stdout'], strict=False) except ValueError: raise CommandExecutionError('Unable to parse return data as Json.') @@ -118,7 +117,7 @@ def _iisVersion(): cmd_ret = _srvmgr(pscmd, return_json=True) try: - items = json.loads(cmd_ret['stdout'], strict=False) + items = salt.utils.json.loads(cmd_ret['stdout'], strict=False) except ValueError: log.error('Unable to parse return data as Json.') return -1 @@ -181,7 +180,7 @@ def list_sites(): cmd_ret = _srvmgr(cmd=ps_cmd, return_json=True) try: - items = json.loads(cmd_ret['stdout'], strict=False) + items = salt.utils.json.loads(cmd_ret['stdout'], strict=False) except ValueError: raise CommandExecutionError('Unable to parse return data as Json.') @@ -979,7 +978,7 @@ def list_apppools(): cmd_ret = _srvmgr(cmd=ps_cmd, return_json=True) try: - items = json.loads(cmd_ret['stdout'], strict=False) + items = salt.utils.json.loads(cmd_ret['stdout'], strict=False) except ValueError: raise CommandExecutionError('Unable to parse return data as Json.') @@ -1223,7 +1222,7 @@ def get_container_setting(name, container, settings): cmd_ret = _srvmgr(cmd=ps_cmd, return_json=True) try: - items = json.loads(cmd_ret['stdout'], strict=False) + items = salt.utils.json.loads(cmd_ret['stdout'], strict=False) if isinstance(items, list): ret.update(items[0]) @@ -1351,7 +1350,7 @@ def list_apps(site): cmd_ret = _srvmgr(cmd=ps_cmd, return_json=True) try: - items = json.loads(cmd_ret['stdout'], strict=False) + items = salt.utils.json.loads(cmd_ret['stdout'], strict=False) except ValueError: raise CommandExecutionError('Unable to parse return data as Json.') @@ -1515,7 +1514,7 @@ def list_vdirs(site, app=_DEFAULT_APP): cmd_ret = _srvmgr(cmd=ps_cmd, return_json=True) try: - items = json.loads(cmd_ret['stdout'], strict=False) + items = salt.utils.json.loads(cmd_ret['stdout'], strict=False) except ValueError: raise CommandExecutionError('Unable to parse return data as Json.') @@ -1672,7 +1671,7 @@ def list_backups(): cmd_ret = _srvmgr(cmd=ps_cmd, return_json=True) try: - items = json.loads(cmd_ret['stdout'], strict=False) + items = salt.utils.json.loads(cmd_ret['stdout'], strict=False) except ValueError: raise CommandExecutionError('Unable to parse return data as Json.') @@ -1786,7 +1785,7 @@ def list_worker_processes(apppool): cmd_ret = _srvmgr(cmd=ps_cmd, return_json=True) try: - items = json.loads(cmd_ret['stdout'], strict=False) + items = salt.utils.json.loads(cmd_ret['stdout'], strict=False) except ValueError: raise CommandExecutionError('Unable to parse return data as Json.') @@ -1854,7 +1853,7 @@ def get_webapp_settings(name, site, settings): # Update dict var to return data try: - items = json.loads(cmd_ret['stdout'], strict=False) + items = salt.utils.json.loads(cmd_ret['stdout'], strict=False) if isinstance(items, list): ret.update(items[0]) diff --git a/salt/modules/win_pki.py b/salt/modules/win_pki.py index 2c5e28e7116..966b4ad31d2 100644 --- a/salt/modules/win_pki.py +++ b/salt/modules/win_pki.py @@ -18,11 +18,11 @@ https://technet.microsoft.com/en-us/library/hh848636(v=wps.620).aspx # Import Python libs from __future__ import absolute_import import ast -import json import logging import os # Import salt libs +import salt.utils.json import salt.utils.platform import salt.utils.powershell import salt.utils.versions @@ -80,7 +80,7 @@ def _cmd_run(cmd, as_json=False): if as_json: try: - items = json.loads(cmd_ret['stdout'], strict=False) + items = salt.utils.json.loads(cmd_ret['stdout'], strict=False) return items except ValueError: _LOG.error('Unable to parse return data as Json.') diff --git a/salt/modules/win_psget.py b/salt/modules/win_psget.py index 672f16d1d47..d0bb94515e0 100644 --- a/salt/modules/win_psget.py +++ b/salt/modules/win_psget.py @@ -13,9 +13,9 @@ from __future__ import absolute_import, print_function, unicode_literals # Import Python libs import copy import logging -import json # Import Salt libs +import salt.utils.json import salt.utils.platform import salt.utils.versions from salt.exceptions import CommandExecutionError @@ -68,7 +68,7 @@ def _pshell(cmd, cwd=None, json_depth=2): raise CommandExecutionError('Issue executing powershell {0}'.format(cmd), info=results) try: - ret = json.loads(results['stdout'], strict=False) + ret = salt.utils.json.loads(results['stdout'], strict=False) except ValueError: raise CommandExecutionError('No JSON results from powershell', info=results) diff --git a/salt/modules/win_servermanager.py b/salt/modules/win_servermanager.py index 46c3b1d9535..703d586eea9 100644 --- a/salt/modules/win_servermanager.py +++ b/salt/modules/win_servermanager.py @@ -10,7 +10,6 @@ available and installed roles/features. Can install and remove roles/features. # Import Python libs from __future__ import absolute_import -import json import logging try: @@ -19,6 +18,7 @@ except ImportError: from pipes import quote as _cmd_quote # Import Salt libs +import salt.utils.json import salt.utils.platform import salt.utils.powershell import salt.utils.versions @@ -59,7 +59,7 @@ def _pshell_json(cmd, cwd=None): log.debug('PowerShell: {0}'.format(cmd)) ret = __salt__['cmd.shell'](cmd, shell='powershell', cwd=cwd) try: - ret = json.loads(ret, strict=False) + ret = salt.utils.json.loads(ret, strict=False) except ValueError: log.debug('Json not returned') return ret diff --git a/salt/modules/zabbix.py b/salt/modules/zabbix.py index a73c5ae1260..79bc6f09bc5 100644 --- a/salt/modules/zabbix.py +++ b/salt/modules/zabbix.py @@ -27,10 +27,10 @@ from __future__ import absolute_import # Import python libs import logging import socket -import json # Import salt libs import salt.utils.http +import salt.utils.json import salt.utils.path from salt.utils.versions import LooseVersion as _LooseVersion from salt.ext.six.moves.urllib.error import HTTPError, URLError # pylint: disable=import-error,no-name-in-module @@ -96,7 +96,7 @@ def _query(method, params, url, auth=None): if method not in unauthenticated_methods: data['auth'] = auth - data = json.dumps(data) + data = salt.utils.json.dumps(data) try: result = salt.utils.http.query(url, diff --git a/salt/modules/zenoss.py b/salt/modules/zenoss.py index deeab114c0f..14ca53dab26 100644 --- a/salt/modules/zenoss.py +++ b/salt/modules/zenoss.py @@ -21,7 +21,6 @@ Module for working with the Zenoss API from __future__ import absolute_import import re -import json import logging try: @@ -30,6 +29,7 @@ try: except ImportError: HAS_LIBS = False +import salt.utils.json # Disable INFO level logs from requests/urllib3 urllib3_logger = logging.getLogger('urllib3') @@ -84,7 +84,7 @@ def _router_request(router, method, data=None): if router not in ROUTERS: return False - req_data = json.dumps([dict( + req_data = salt.utils.json.dumps([dict( action=router, method=method, data=data, @@ -103,7 +103,7 @@ def _router_request(router, method, data=None): log.error('Request failed. Bad username/password.') raise Exception('Request failed. Bad username/password.') - return json.loads(response.content).get('result', None) + return salt.utils.json.loads(response.content).get('result', None) def _determine_device_class(): diff --git a/salt/netapi/rest_cherrypy/app.py b/salt/netapi/rest_cherrypy/app.py index 3f0dc4d5644..2c9638e4efa 100644 --- a/salt/netapi/rest_cherrypy/app.py +++ b/salt/netapi/rest_cherrypy/app.py @@ -585,7 +585,6 @@ import collections import itertools import functools import logging -import json import os import signal import tarfile @@ -615,6 +614,7 @@ import salt import salt.auth import salt.exceptions import salt.utils.event +import salt.utils.json import salt.utils.stringutils import salt.utils.versions from salt.ext import six @@ -835,7 +835,7 @@ def cors_tool(): # Maps Content-Type to serialization functions; this is a tuple of tuples to # preserve order of preference. ct_out_map = ( - ('application/json', json.dumps), + ('application/json', salt.utils.json.dumps), ('application/x-yaml', functools.partial( yaml.safe_dump, default_flow_style=False)), ) @@ -975,7 +975,7 @@ def json_processor(entity): body = salt.utils.stringutils.to_unicode(contents.read()) del contents try: - cherrypy.serving.request.unserialized_data = json.loads(body) + cherrypy.serving.request.unserialized_data = salt.utils.json.loads(body) except ValueError: raise cherrypy.HTTPError(400, 'Invalid JSON document') @@ -1024,7 +1024,7 @@ def text_processor(entity): contents.seek(0) body = salt.utils.stringutils.to_unicode(contents.read()) try: - cherrypy.serving.request.unserialized_data = json.loads(body) + cherrypy.serving.request.unserialized_data = salt.utils.json.loads(body) except ValueError: cherrypy.serving.request.unserialized_data = body @@ -2334,12 +2334,12 @@ class Events(object): listen=True) stream = event.iter_events(full=True, auto_reconnect=True) - yield u'retry: {0}\n'.format(400) + yield str('retry: 400\n') # future lint: disable=blacklisted-function while True: data = next(stream) - yield u'tag: {0}\n'.format(data.get('tag', '')) - yield u'data: {0}\n\n'.format(json.dumps(data)) + yield str('tag: {0}\n').format(data.get('tag', '')) # future lint: disable=blacklisted-function + yield str('data: {0}\n\n').format(salt.utils.json.dumps(data)) # future lint: disable=blacklisted-function return listen() @@ -2520,8 +2520,10 @@ class WebsocketEndpoint(object): if 'format_events' in kwargs: SaltInfo.process(data, salt_token, self.opts) else: - handler.send('data: {0}\n\n'.format( - json.dumps(data)), False) + handler.send( + str('data: {0}\n\n').format(salt.utils.json.dumps(data)), # future lint: disable=blacklisted-function + False + ) except UnicodeDecodeError: logger.error( "Error: Salt event has non UTF-8 data:\n{0}" diff --git a/salt/netapi/rest_cherrypy/event_processor.py b/salt/netapi/rest_cherrypy/event_processor.py index 58ec010b00c..b7aaff1972c 100644 --- a/salt/netapi/rest_cherrypy/event_processor.py +++ b/salt/netapi/rest_cherrypy/event_processor.py @@ -2,7 +2,6 @@ # Import python libs from __future__ import absolute_import -import json import logging # Import 3rd-party libs @@ -10,6 +9,7 @@ from salt.ext import six # Import Salt libs import salt.netapi +import salt.utils.json logger = logging.getLogger(__name__) @@ -45,14 +45,14 @@ class SaltInfo(object): minions.append(curr_minion) ret = {'minions': minions} - self.handler.send(json.dumps(ret), False) + self.handler.send(salt.utils.json.dumps(ret), False) def publish(self, key, data): ''' Publishes the data to the event stream. ''' publish_data = {key: data} - self.handler.send(json.dumps(publish_data), False) + self.handler.send(salt.utils.json.dumps(publish_data), False) def process_minion_update(self, event_data): ''' diff --git a/salt/netapi/rest_tornado/event_processor.py b/salt/netapi/rest_tornado/event_processor.py index c644f123617..27455284736 100644 --- a/salt/netapi/rest_tornado/event_processor.py +++ b/salt/netapi/rest_tornado/event_processor.py @@ -1,11 +1,11 @@ # encoding: utf-8 from __future__ import absolute_import -import json import logging import threading from salt.ext import six import salt.netapi +import salt.utils.json logger = logging.getLogger(__name__) @@ -46,14 +46,15 @@ class SaltInfo(object): minions[minion] = curr_minion logger.debug('ended loop') ret = {'minions': minions} - self.handler.write_message(u'{0}\n\n'.format(json.dumps(ret))) + self.handler.write_message( + salt.utils.json.dumps(ret) + str('\n\n')) # future lint: disable=blacklisted-function def publish(self, key, data): ''' Publishes the data to the event stream. ''' publish_data = {key: data} - pub = u'{0}\n\n'.format(json.dumps(publish_data)) + pub = salt.utils.json.dumps(publish_data) + str('\n\n') # future lint: disable=blacklisted-function self.handler.write_message(pub) def process_minion_update(self, event_data): @@ -63,7 +64,7 @@ class SaltInfo(object): tag = event_data['tag'] event_info = event_data['data'] - _, _, _, _, mid = tag.split('/') + mid = tag.split('/')[-1] if not self.minions.get(mid, None): self.minions[mid] = {} diff --git a/salt/netapi/rest_tornado/saltnado.py b/salt/netapi/rest_tornado/saltnado.py index 4021632d79c..85296687be8 100644 --- a/salt/netapi/rest_tornado/saltnado.py +++ b/salt/netapi/rest_tornado/saltnado.py @@ -1,6 +1,4 @@ # encoding: utf-8 -from __future__ import absolute_import, print_function - ''' A non-blocking REST API for Salt ================================ @@ -186,8 +184,8 @@ a return like:: .. |401| replace:: authentication required .. |406| replace:: requested Content-Type not available .. |500| replace:: internal server error -''' # pylint: disable=W0105 -# pylint: disable=W0232 +''' +from __future__ import absolute_import, print_function # Import Python libs import time @@ -226,6 +224,15 @@ from salt.exceptions import EauthAuthenticationError json = salt.utils.json.import_json() logger = logging.getLogger() + +def _json_dumps(obj, **kwargs): + ''' + Invoke salt.utils.json.dumps using the alternate json module loaded using + salt.utils.json.import_json(). This ensures that we properly encode any + strings in the object before we perform the serialization. + ''' + return salt.utils.json.dumps(obj, _json_module=json, **kwargs) + # The clients rest_cherrypi supports. We want to mimic the interface, but not # necessarily use the same API under the hood # # all of these require coordinating minion stuff @@ -391,7 +398,7 @@ class EventListener(object): class BaseSaltAPIHandler(tornado.web.RequestHandler, SaltClientsMixIn): # pylint: disable=W0223 ct_out_map = ( - ('application/json', json.dumps), + ('application/json', _json_dumps), ('application/x-yaml', yaml.safe_dump), ) @@ -515,11 +522,11 @@ class BaseSaltAPIHandler(tornado.web.RequestHandler, SaltClientsMixIn): # pylin ''' ct_in_map = { 'application/x-www-form-urlencoded': self._form_loader, - 'application/json': json.loads, + 'application/json': salt.utils.json.loads, 'application/x-yaml': yaml.safe_load, 'text/yaml': yaml.safe_load, # because people are terrible and don't mean what they say - 'text/plain': json.loads + 'text/plain': salt.utils.json.loads } try: @@ -1485,8 +1492,8 @@ class EventsSaltAPIHandler(SaltAPIHandler): # pylint: disable=W0223 while True: try: event = yield self.application.event_listener.get_event(self) - self.write(u'tag: {0}\n'.format(event.get('tag', ''))) - self.write(u'data: {0}\n\n'.format(json.dumps(event))) + self.write('tag: {0}\n'.format(event.get('tag', ''))) + self.write(str('data: {0}\n\n').format(_json_dumps(event))) # future lint: disable=blacklisted-function self.flush() except TimeoutException: break diff --git a/salt/netapi/rest_tornado/saltnado_websockets.py b/salt/netapi/rest_tornado/saltnado_websockets.py index dfc8e1719a2..8d64e27e43f 100644 --- a/salt/netapi/rest_tornado/saltnado_websockets.py +++ b/salt/netapi/rest_tornado/saltnado_websockets.py @@ -300,7 +300,7 @@ import tornado.gen import salt.utils.json import salt.netapi -json = salt.utils.json.import_json() +_json = salt.utils.json.import_json() import logging logger = logging.getLogger() @@ -354,7 +354,8 @@ class AllEventsHandler(tornado.websocket.WebSocketHandler): # pylint: disable=W while True: try: event = yield self.application.event_listener.get_event(self) - self.write_message(json.dumps(event)) + self.write_message( + salt.utils.json.dumps(event, _json_module=_json)) except Exception as err: logger.info('Error! Ending server side websocket connection. Reason = {0}'.format(str(err))) break @@ -417,7 +418,7 @@ class FormattedEventsHandler(AllEventsHandler): # pylint: disable=W0223,W0232 try: event = yield self.application.event_listener.get_event(self) evt_processor.process(event, self.token, self.application.opts) - # self.write_message(u'data: {0}\n\n'.format(json.dumps(event))) + # self.write_message(u'data: {0}\n\n'.format(salt.utils.json.dumps(event, _json_module=_json))) except Exception as err: logger.debug('Error! Ending server side websocket connection. Reason = {0}'.format(str(err))) break diff --git a/salt/netapi/rest_wsgi.py b/salt/netapi/rest_wsgi.py index 495adb2e875..ab7831c451c 100644 --- a/salt/netapi/rest_wsgi.py +++ b/salt/netapi/rest_wsgi.py @@ -123,13 +123,13 @@ Usage examples ''' from __future__ import absolute_import import errno -import json import logging import os # Import salt libs import salt import salt.netapi +import salt.utils.json # HTTP response codes to response headers map H = { @@ -198,7 +198,7 @@ def get_json(environ): raise HTTPError(406, 'JSON required') try: - return json.loads(read_body(environ)) + return salt.utils.json.loads(read_body(environ)) except ValueError as exc: raise HTTPError(400, exc) @@ -283,10 +283,10 @@ def application(environ, start_response): # Convert the response to JSON try: - ret = json.dumps({'return': resp}) + ret = salt.utils.json.dumps({'return': resp}) except TypeError as exc: code = 500 - ret = str(exc) + ret = str(exc) # future lint: disable=blacklisted-function # Return the response start_response(H[code], get_headers(ret, { diff --git a/salt/output/json_out.py b/salt/output/json_out.py index 4522ca1a78c..e40c0baadc0 100644 --- a/salt/output/json_out.py +++ b/salt/output/json_out.py @@ -32,9 +32,11 @@ single-line output format and to parse each line individually. Example output from __future__ import absolute_import, print_function, unicode_literals # Import python libs -import json import logging +# Import Salt libs +import salt.utils.json + # Import 3rd-party libs from salt.ext import six @@ -57,7 +59,7 @@ def output(data, **kwargs): # pylint: disable=unused-argument ''' try: if 'output_indent' not in __opts__: - return json.dumps(data, default=repr, indent=4) + return salt.utils.json.dumps(data, default=repr, indent=4) indent = __opts__.get('output_indent') sort_keys = False @@ -75,13 +77,16 @@ def output(data, **kwargs): # pylint: disable=unused-argument else: indent = None - return json.dumps(data, default=repr, indent=indent, sort_keys=sort_keys) + return salt.utils.json.dumps(data, default=repr, indent=indent, sort_keys=sort_keys) except UnicodeDecodeError as exc: log.error('Unable to serialize output to json') - return json.dumps({'error': 'Unable to serialize output to json', 'message': six.text_type(exc)}) + return salt.utils.json.dumps( + {'error': 'Unable to serialize output to json', + 'message': six.text_type(exc)} + ) except TypeError: log.debug('An error occurred while outputting JSON', exc_info=True) # Return valid JSON for unserializable objects - return json.dumps({}) + return salt.utils.json.dumps({}) diff --git a/salt/pillar/cmd_json.py b/salt/pillar/cmd_json.py index 1a75d819bea..13f59151da8 100644 --- a/salt/pillar/cmd_json.py +++ b/salt/pillar/cmd_json.py @@ -7,9 +7,11 @@ from __future__ import absolute_import # Don't "fix" the above docstring to put it on two lines, as the sphinx # autosummary pulls only the first line for its description. -# Import python libs +# Import Python libs import logging -import json + +# Import Salt libs +import salt.utils.json # Set up logging log = logging.getLogger(__name__) @@ -23,7 +25,7 @@ def ext_pillar(minion_id, # pylint: disable=W0613 ''' try: command = command.replace('%s', minion_id) - return json.loads(__salt__['cmd.run'](command)) + return salt.utils.json.loads(__salt__['cmd.run'](command)) except Exception: log.critical( 'JSON data from {0} failed to parse'.format(command) diff --git a/salt/pillar/redismod.py b/salt/pillar/redismod.py index 5a997bfb4ae..398161af4b4 100644 --- a/salt/pillar/redismod.py +++ b/salt/pillar/redismod.py @@ -30,10 +30,11 @@ Configuring the Redis ext_pillar - redis: {function: key_value} ''' +# Import Python libs from __future__ import absolute_import -# Import python libs -import json +# Import Salt libs +import salt.utils.json __virtualname__ = 'redis' @@ -107,7 +108,7 @@ def key_json(minion_id, if not key_data: return {} - data = json.loads(key_data) + data = salt.utils.json.loads(key_data) # Return as requested if isinstance(data, dict) and not pillar_key: return data diff --git a/salt/proxy/philips_hue.py b/salt/proxy/philips_hue.py index 9420963d540..a012ca32cb2 100644 --- a/salt/proxy/philips_hue.py +++ b/salt/proxy/philips_hue.py @@ -27,7 +27,7 @@ import salt.ext.six.moves.http_client as http_client # Import python libs import logging import time -import json +import salt.utils.json from salt.exceptions import (CommandExecutionError, MinionError) from salt.ext import six @@ -108,13 +108,13 @@ def _query(lamp_id, state, action='', method='GET'): + (action and "/{0}".format(action) or '') conn = http_client.HTTPConnection(CONFIG['host']) if method == 'PUT': - conn.request(method, url, json.dumps(state)) + conn.request(method, url, salt.utils.json.dumps(state)) else: conn.request(method, url) resp = conn.getresponse() if resp.status == http_client.OK: - res = json.loads(resp.read()) + res = salt.utils.json.loads(resp.read()) else: err = "HTTP error: {0}, {1}".format(resp.status, resp.reason) conn.close() diff --git a/salt/proxy/ssh_sample.py b/salt/proxy/ssh_sample.py index fa8db0c5ce2..9fd70843402 100644 --- a/salt/proxy/ssh_sample.py +++ b/salt/proxy/ssh_sample.py @@ -8,10 +8,10 @@ from __future__ import absolute_import, print_function, unicode_literals # Import python libs -import json import logging -# Import Salt's libs +# Import Salt libs +import salt.utils.json from salt.utils.vt_helper import SSHConnection from salt.utils.vt import TerminalException @@ -128,7 +128,7 @@ def parse(out): jsonret.append(ln_) if '}' in ln_: in_json = False - return json.loads('\n'.join(jsonret)) + return salt.utils.json.loads('\n'.join(jsonret)) def package_list(): diff --git a/salt/queues/pgjsonb_queue.py b/salt/queues/pgjsonb_queue.py index 57d0617e57a..00d8eaba770 100644 --- a/salt/queues/pgjsonb_queue.py +++ b/salt/queues/pgjsonb_queue.py @@ -42,10 +42,10 @@ Use the following Pg database schema: # Import python libs from __future__ import absolute_import from contextlib import contextmanager -import json import sys # import salt libs +import salt.utils.json from salt.ext import six from salt.exceptions import SaltInvocationError, SaltMasterError @@ -189,27 +189,23 @@ def insert(queue, items): with _conn(commit=True) as cur: if isinstance(items, dict): - items = json.dumps(items) - cmd = '''INSERT INTO {0}(data) VALUES('{1}')'''.format(queue, items) + items = salt.utils.json.dumps(items) + cmd = str('''INSERT INTO {0}(data) VALUES('{1}')''').format(queue, items) # future lint: disable=blacklisted-function log.debug('SQL Query: {0}'.format(cmd)) try: cur.execute(cmd) except psycopg2.IntegrityError as esc: - return('Item already exists in this queue. ' - 'sqlite error: {0}'.format(esc)) + return ('Item already exists in this queue. ' + 'postgres error: {0}'.format(esc)) if isinstance(items, list): - items = [json.dumps(el) for el in items] - cmd = "INSERT INTO {0}(data) VALUES (%s)".format(queue) + items = [(salt.utils.json.dumps(el),) for el in items] + cmd = str("INSERT INTO {0}(data) VALUES (%s)").format(queue) # future lint: disable=blacklisted-function log.debug('SQL Query: {0}'.format(cmd)) - newitems = [] - for item in items: - newitems.append((item,)) - # we need a list of one item tuples here try: - cur.executemany(cmd, newitems) + cur.executemany(cmd, items) except psycopg2.IntegrityError as esc: - return('One or more items already exists in this queue. ' - 'sqlite error: {0}'.format(esc)) + return ('One or more items already exists in this queue. ' + 'postgres error: {0}'.format(esc)) return True @@ -219,19 +215,17 @@ def delete(queue, items): ''' with _conn(commit=True) as cur: if isinstance(items, dict): - cmd = """DELETE FROM {0} WHERE data = '{1}'""".format(queue, json.dumps(items)) + cmd = str("""DELETE FROM {0} WHERE data = '{1}'""").format( # future lint: disable=blacklisted-function + queue, + salt.utils.json.dumps(items)) log.debug('SQL Query: {0}'.format(cmd)) cur.execute(cmd) return True if isinstance(items, list): - items = [json.dumps(el) for el in items] + items = [(salt.utils.json.dumps(el),) for el in items] cmd = 'DELETE FROM {0} WHERE data = %s'.format(queue) log.debug('SQL Query: {0}'.format(cmd)) - newitems = [] - for item in items: - newitems.append((item,)) - # we need a list of one item tuples here - cur.executemany(cmd, newitems) + cur.executemany(cmd, items) return True diff --git a/salt/queues/sqlite_queue.py b/salt/queues/sqlite_queue.py index 2e298579a79..196bd042933 100644 --- a/salt/queues/sqlite_queue.py +++ b/salt/queues/sqlite_queue.py @@ -12,16 +12,14 @@ to another location:: sqlite_queue_dir: /home/myuser/salt/master/queues ''' - # Import python libs -from __future__ import print_function -from __future__ import absolute_import +from __future__ import absolute_import, print_function import glob import logging import os -import json import re -import sqlite3 as lite +import sqlite3 +import salt.utils.json from salt.exceptions import SaltInvocationError # Import 3rd-party libs @@ -47,7 +45,7 @@ def _conn(queue): db = os.path.join(queue_dir, '{0}.db'.format(queue)) log.debug('Connecting to: {0}'.format(db)) - con = lite.connect(db) + con = sqlite3.connect(db) tables = _list_tables(con) if queue not in tables: _create_table(con, queue) @@ -149,7 +147,7 @@ def insert(queue, items): log.debug('SQL Query: {0}'.format(cmd)) try: cur.execute(cmd) - except lite.IntegrityError as esc: + except sqlite3.IntegrityError as esc: return('Item already exists in this queue. ' 'sqlite error: {0}'.format(esc)) if isinstance(items, list): @@ -162,17 +160,17 @@ def insert(queue, items): # we need a list of one item tuples here try: cur.executemany(cmd, newitems) - except lite.IntegrityError as esc: + except sqlite3.IntegrityError as esc: return('One or more items already exists in this queue. ' 'sqlite error: {0}'.format(esc)) if isinstance(items, dict): - items = json.dumps(items).replace('"', "'") + items = salt.utils.json.dumps(items).replace('"', "'") items = _quote_escape(items) - cmd = '''INSERT INTO {0}(name) VALUES('{1}')'''.format(queue, items) + cmd = str('''INSERT INTO {0}(name) VALUES('{1}')''').format(queue, items) # future lint: disable=blacklisted-function log.debug('SQL Query: {0}'.format(cmd)) try: cur.execute(cmd) - except lite.IntegrityError as esc: + except sqlite3.IntegrityError as esc: return('Item already exists in this queue. ' 'sqlite error: {0}'.format(esc)) return True @@ -201,9 +199,9 @@ def delete(queue, items): # we need a list of one item tuples here cur.executemany(cmd, newitems) if isinstance(items, dict): - items = json.dumps(items).replace('"', "'") + items = salt.utils.json.dumps(items).replace('"', "'") items = _quote_escape(items) - cmd = """DELETE FROM {0} WHERE name = '{1}'""".format(queue, items) + cmd = ("""DELETE FROM {0} WHERE name = '{1}'""").format(queue, items) # future lint: disable=blacklisted-function log.debug('SQL Query: {0}'.format(cmd)) cur.execute(cmd) return True @@ -241,6 +239,6 @@ def pop(queue, quantity=1, is_runner=False): cur.execute(del_cmd) con.commit() if is_runner: - items = [json.loads(item[0].replace("'", '"')) for item in result] + items = [salt.utils.json.loads(item[0].replace("'", '"')) for item in result] log.info(items) return items diff --git a/salt/returners/cassandra_cql_return.py b/salt/returners/cassandra_cql_return.py index 3fb0c09ad22..8e92e321479 100644 --- a/salt/returners/cassandra_cql_return.py +++ b/salt/returners/cassandra_cql_return.py @@ -120,7 +120,6 @@ some stacktraces that appear to come from the Datastax Python driver. from __future__ import absolute_import, print_function, unicode_literals # Import python libs -import json import logging import uuid import time @@ -128,6 +127,7 @@ import time # Import salt libs import salt.returners import salt.utils.jid +import salt.utils.json import salt.exceptions from salt.exceptions import CommandExecutionError from salt.ext import six @@ -195,8 +195,8 @@ def returner(ret): statement_arguments.append('{0}'.format(ret['id'])) statement_arguments.append('{0}'.format(ret['fun'])) statement_arguments.append(int(time.time() * 1000)) - statement_arguments.append('{0}'.format(json.dumps(ret).replace("'", "''"))) - statement_arguments.append('{0}'.format(json.dumps(ret['return']).replace("'", "''"))) + statement_arguments.append(salt.utils.json.dumps(ret).replace("'", "''")) + statement_arguments.append(salt.utils.json.dumps(ret['return']).replace("'", "''")) statement_arguments.append(ret.get('success', False)) # cassandra_cql.cql_query may raise a CommandExecutionError @@ -262,7 +262,7 @@ def event_return(events): ''' statement_arguments = [six.text_type(uuid.uuid1()), int(time.time() * 1000), - json.dumps(data).replace("'", "''"), + salt.utils.json.dumps(data).replace("'", "''"), __opts__['id'], tag] @@ -286,14 +286,14 @@ def save_load(jid, load, minions=None): ''' # Load is being stored as a text datatype. Single quotes are used in the # VALUES list. Therefore, all single quotes contained in the results from - # json.dumps(load) must be escaped Cassandra style. + # salt.utils.json.dumps(load) must be escaped Cassandra style. query = '''INSERT INTO salt.jids ( jid, load ) VALUES (?, ?)''' statement_arguments = [ jid, - json.dumps(load).replace("'", "''") + salt.utils.json.dumps(load).replace("'", "''") ] # cassandra_cql.cql_query may raise a CommandExecutionError @@ -331,7 +331,7 @@ def get_load(jid): if data: load = data[0].get('load') if load: - ret = json.loads(load) + ret = salt.utils.json.loads(load) except CommandExecutionError: log.critical('Could not get load from jids table.') raise @@ -359,7 +359,7 @@ def get_jid(jid): minion = row.get('minion_id') full_ret = row.get('full_ret') if minion and full_ret: - ret[minion] = json.loads(full_ret) + ret[minion] = salt.utils.json.loads(full_ret) except CommandExecutionError: log.critical('Could not select job specific information.') raise @@ -417,7 +417,9 @@ def get_jids(): jid = row.get('jid') load = row.get('load') if jid and load: - ret[jid] = salt.utils.jid.format_jid_instance(jid, json.loads(load)) + ret[jid] = salt.utils.jid.format_jid_instance( + jid, + salt.utils.json.loads(load)) except CommandExecutionError: log.critical('Could not get a list of all job ids.') raise diff --git a/salt/returners/couchbase_return.py b/salt/returners/couchbase_return.py index fb108a1a816..2ab12b53c23 100644 --- a/salt/returners/couchbase_return.py +++ b/salt/returners/couchbase_return.py @@ -49,8 +49,6 @@ return: return_data full_ret: full load of job return ''' from __future__ import absolute_import, print_function, unicode_literals - -import json import logging try: @@ -78,14 +76,18 @@ COUCHBASE_CONN = None DESIGN_NAME = 'couchbase_returner' VERIFIED_VIEWS = False +_json = salt.utils.json.import_json() + + +def _json_dumps(obj, **kwargs): + return salt.utils.json.dumps(obj, _json_module=_json) + def __virtual__(): if not HAS_DEPS: return False, 'Could not import couchbase returner; couchbase is not installed.' - # try to load some faster json libraries. In order of fastest to slowest - json = salt.utils.json.import_json() - couchbase.set_json_converters(json.dumps, json.loads) + couchbase.set_json_converters(_json_dumps, salt.utils.json.loads) return __virtualname__ @@ -192,7 +194,7 @@ def returner(load): hn_key = '{0}/{1}'.format(load['jid'], load['id']) try: ret_doc = {'return': load['return'], - 'full_ret': json.dumps(load)} + 'full_ret': salt.utils.json.dumps(load)} cb_.add(hn_key, ret_doc, diff --git a/salt/returners/couchdb_return.py b/salt/returners/couchdb_return.py index 499085c420d..65d227937fe 100644 --- a/salt/returners/couchdb_return.py +++ b/salt/returners/couchdb_return.py @@ -56,7 +56,6 @@ otherwise multi-minion targeting can lead to losing output: from __future__ import absolute_import, print_function, unicode_literals import logging import time -import json # Import 3rd-party libs # pylint: disable=no-name-in-module,import-error @@ -70,6 +69,7 @@ from salt.ext.six.moves.urllib.request import ( # Import Salt libs import salt.utils.jid +import salt.utils.json import salt.returners log = logging.getLogger(__name__) @@ -136,7 +136,7 @@ def _request(method, url, content_type=None, _data=None): handler = opener.open(request) except HTTPError as exc: return {'error': '{0}'.format(exc)} - return json.loads(handler.read()) + return salt.utils.json.loads(handler.read()) def returner(ret): @@ -168,7 +168,7 @@ def returner(ret): _response = _request("PUT", options['url'] + options['db'] + "/" + doc['_id'], 'application/json', - json.dumps(doc)) + salt.utils.json.dumps(doc)) # Sanity check regarding the response.. if 'ok' not in _response or _response['ok'] is not True: @@ -355,7 +355,7 @@ def set_salt_view(): # Make the request to update the design doc. _response = _request("PUT", options['url'] + options['db'] + "/_design/salt", - "application/json", json.dumps(new_doc)) + "application/json", salt.utils.json.dumps(new_doc)) if 'error' in _response: log.warning( 'Unable to set the salt design document: %s', diff --git a/salt/returners/elasticsearch_return.py b/salt/returners/elasticsearch_return.py index 076b06f3280..b12e5c0a4bb 100644 --- a/salt/returners/elasticsearch_return.py +++ b/salt/returners/elasticsearch_return.py @@ -100,11 +100,11 @@ import datetime from datetime import tzinfo, timedelta import uuid import logging -import json # Import Salt libs import salt.returners import salt.utils.jid +import salt.utils.json # Import 3rd-party libs from salt.ext import six @@ -329,7 +329,7 @@ def returner(ret): # Post the payload ret = __salt__['elasticsearch.document_create'](index=index, doc_type=options['doc_type'], - body=json.dumps(data)) + body=salt.utils.json.dumps(data)) def event_return(events): @@ -358,7 +358,7 @@ def event_return(events): ret = __salt__['elasticsearch.document_create'](index=index, doc_type=doc_type, id=uuid.uuid4(), - body=json.dumps(data)) + body=salt.utils.json.dumps(data)) def prep_jid(nocache=False, passed_jid=None): # pylint: disable=unused-argument @@ -389,7 +389,7 @@ def save_load(jid, load, minions=None): ret = __salt__['elasticsearch.document_create'](index=index, doc_type=doc_type, id=jid, - body=json.dumps(data)) + body=salt.utils.json.dumps(data)) def get_load(jid): @@ -407,5 +407,5 @@ def get_load(jid): id=jid, doc_type=doc_type) if data: - return json.loads(data) + return salt.utils.json.loads(data) return {} diff --git a/salt/returners/etcd_return.py b/salt/returners/etcd_return.py index 744c5b530b0..525eaea69b9 100644 --- a/salt/returners/etcd_return.py +++ b/salt/returners/etcd_return.py @@ -67,11 +67,11 @@ create the profiles as specified above. Then add: from __future__ import absolute_import, print_function, unicode_literals # Import python libs -import json import logging # Import salt libs import salt.utils.jid +import salt.utils.json try: import salt.utils.etcd_util HAS_LIBS = True @@ -134,7 +134,7 @@ def returner(ret): ret['id'], field )) - client.set(dest, json.dumps(ret[field]), ttl=ttl) + client.set(dest, salt.utils.json.dumps(ret[field]), ttl=ttl) def save_load(jid, load, minions=None): @@ -149,7 +149,7 @@ def save_load(jid, load, minions=None): ttl = __opts__.get('etcd.ttl') client.set( '/'.join((path, 'jobs', jid, '.load.p')), - json.dumps(load), + salt.utils.json.dumps(load), ttl=ttl, ) @@ -167,7 +167,7 @@ def get_load(jid): ''' read_profile = __opts__.get('etcd.returner_read_profile') client, path = _get_conn(__opts__, read_profile) - return json.loads(client.get('/'.join((path, 'jobs', jid, '.load.p')))) + return salt.utils.json.loads(client.get('/'.join((path, 'jobs', jid, '.load.p')))) def get_jid(jid): @@ -203,7 +203,7 @@ def get_jids(): if item.dir is True: jid = six.text_type(item.key).split('/')[-1] load = client.get('/'.join((item.key, '.load.p'))).value - ret[jid] = salt.utils.jid.format_jid_instance(jid, json.loads(load)) + ret[jid] = salt.utils.jid.format_jid_instance(jid, salt.utils.json.loads(load)) return ret diff --git a/salt/returners/highstate_return.py b/salt/returners/highstate_return.py index c6d1f7afc30..46e9b43c89a 100644 --- a/salt/returners/highstate_return.py +++ b/salt/returners/highstate_return.py @@ -80,7 +80,6 @@ the time of execution. from __future__ import absolute_import, print_function, unicode_literals import logging -import json import smtplib import cgi from email.mime.text import MIMEText @@ -91,6 +90,7 @@ from salt.ext.six.moves import StringIO from salt.ext import six import salt.utils.files +import salt.utils.json import salt.utils.stringutils import salt.returners @@ -424,7 +424,7 @@ def _produce_output(report, failed, setup): log.debug('highstate output format: %s', report_format) if report_format == 'json': - report_text = json.dumps(report) + report_text = salt.utils.json.dumps(report) elif report_format == 'yaml': string_file = StringIO() yaml.safe_dump(report, string_file, default_flow_style=False) diff --git a/salt/returners/hipchat_return.py b/salt/returners/hipchat_return.py index 5aa7bb98299..a777a732280 100644 --- a/salt/returners/hipchat_return.py +++ b/salt/returners/hipchat_return.py @@ -96,7 +96,6 @@ To override individual configuration items, append --return_kwargs '{"key:": "va from __future__ import absolute_import, print_function, unicode_literals # Import Python libs -import json import pprint import logging @@ -109,6 +108,7 @@ import salt.ext.six.moves.http_client # Import Salt Libs import salt.returners +import salt.utils.json log = logging.getLogger(__name__) @@ -241,7 +241,7 @@ def _query(function, headers['Content-Type'] = 'application/json' headers['Authorization'] = 'Bearer {0}'.format(api_key) if data: - data = json.dumps(data) + data = salt.utils.json.dumps(data) else: log.error('Unsupported HipChat API version') return False diff --git a/salt/returners/influxdb_return.py b/salt/returners/influxdb_return.py index 2264adea5a5..69aaf0d3289 100644 --- a/salt/returners/influxdb_return.py +++ b/salt/returners/influxdb_return.py @@ -53,7 +53,6 @@ To override individual configuration items, append --return_kwargs '{"key:": "va from __future__ import absolute_import, print_function, unicode_literals # Import python libs -import json import logging import requests @@ -155,9 +154,9 @@ def returner(ret): serv = _get_serv(ret) # strip the 'return' key to avoid data duplication in the database - json_return = json.dumps(ret['return']) + json_return = salt.utils.json.dumps(ret['return']) del ret['return'] - json_full_ret = json.dumps(ret) + json_full_ret = salt.utils.json.dumps(ret) # create legacy request in case an InfluxDB 0.8.x version is used if "influxdb08" in serv.__module__: @@ -206,7 +205,7 @@ def save_load(jid, load, minions=None): 'name': 'jids', 'columns': ['jid', 'load'], 'points': [ - [jid, json.dumps(load)] + [jid, salt.utils.json.dumps(load)] ], } ] @@ -219,7 +218,7 @@ def save_load(jid, load, minions=None): 'jid': jid }, 'fields': { - 'load': json.dumps(load) + 'load': salt.utils.json.dumps(load) } } ] @@ -265,7 +264,7 @@ def get_jid(jid): if data: points = data[0]['points'] for point in points: - ret[point[3]] = json.loads(point[2]) + ret[point[3]] = salt.utils.json.loads(point[2]) return ret @@ -287,7 +286,7 @@ def get_fun(fun): if data: points = data[0]['points'] for point in points: - ret[point[1]] = json.loads(point[2]) + ret[point[1]] = salt.utils.json.loads(point[2]) return ret @@ -307,7 +306,7 @@ def get_jids(): ret = {} if data: for _, jid, load in data[0]['points']: - ret[jid] = salt.utils.jid.format_jid_instance(jid, json.loads(load)) + ret[jid] = salt.utils.jid.format_jid_instance(jid, salt.utils.json.loads(load)) return ret diff --git a/salt/returners/kafka_return.py b/salt/returners/kafka_return.py index b63da11c696..d0bcef44641 100644 --- a/salt/returners/kafka_return.py +++ b/salt/returners/kafka_return.py @@ -26,8 +26,8 @@ To use the kafka returner, append '--return kafka' to the Salt command, eg; # Import Python libs from __future__ import absolute_import, print_function, unicode_literals -import json import logging +import salt.utils.json # Import third-party libs try: @@ -74,7 +74,7 @@ def returner(ret): conn = _get_conn(ret) producer = SimpleProducer(conn) - producer.send_messages(topic, json.dumps(ret)) + producer.send_messages(topic, salt.utils.json.dumps(ret)) _close_conn(conn) else: diff --git a/salt/returners/mattermost_returner.py b/salt/returners/mattermost_returner.py index 4f258acfa08..09015b43911 100644 --- a/salt/returners/mattermost_returner.py +++ b/salt/returners/mattermost_returner.py @@ -47,7 +47,6 @@ from __future__ import absolute_import, print_function, unicode_literals # Import Python libs import logging -import json # Import 3rd-party libs from salt.ext import six @@ -57,6 +56,7 @@ import salt.ext.six.moves.http_client # Import Salt Libs import salt.returners +import salt.utils.json import salt.utils.mattermost log = logging.getLogger(__name__) @@ -184,9 +184,10 @@ def post_message(channel, parameters['username'] = username parameters['text'] = '```' + message + '```' # pre-formatted, fixed-width text log.debug('Parameters: %s', parameters) - result = salt.utils.mattermost.query(api_url=api_url, - hook=hook, - data='payload={0}'.format(json.dumps(parameters))) + result = salt.utils.mattermost.query( + api_url=api_url, + hook=hook, + data=str('payload={0}').format(salt.utils.json.dumps(parameters))) # future lint: disable=blacklisted-function log.debug('result %s', result) return bool(result) diff --git a/salt/returners/memcache_return.py b/salt/returners/memcache_return.py index 9a9397708dc..03802aa76b3 100644 --- a/salt/returners/memcache_return.py +++ b/salt/returners/memcache_return.py @@ -48,10 +48,10 @@ To override individual configuration items, append --return_kwargs '{"key:": "va from __future__ import absolute_import, print_function, unicode_literals # Import python libs -import json import logging import salt.utils.jid +import salt.utils.json import salt.returners from salt.ext import six @@ -145,7 +145,7 @@ def returner(ret): minion = ret['id'] jid = ret['jid'] fun = ret['fun'] - rets = json.dumps(ret) + rets = salt.utils.json.dumps(ret) serv.set('{0}:{1}'.format(jid, minion), rets) # cache for get_jid serv.set('{0}:{1}'.format(fun, minion), rets) # cache for get_fun @@ -160,7 +160,7 @@ def save_load(jid, load, minions=None): Save the load to the specified jid ''' serv = _get_serv(ret=None) - serv.set(jid, json.dumps(load)) + serv.set(jid, salt.utils.json.dumps(load)) _append_list(serv, 'jids', jid) @@ -178,7 +178,7 @@ def get_load(jid): serv = _get_serv(ret=None) data = serv.get(jid) if data: - return json.loads(data) + return salt.utils.json.loads(data) return {} @@ -192,7 +192,7 @@ def get_jid(jid): # returns = {minion: return, minion: return, ...} ret = {} for minion, data in six.iteritems(returns): - ret[minion] = json.loads(data) + ret[minion] = salt.utils.json.loads(data) return ret @@ -206,7 +206,7 @@ def get_fun(fun): # returns = {minion: return, minion: return, ...} ret = {} for minion, data in six.iteritems(returns): - ret[minion] = json.loads(data) + ret[minion] = salt.utils.json.loads(data) return ret @@ -219,7 +219,7 @@ def get_jids(): loads = serv.get_multi(jids) # {jid: load, jid: load, ...} ret = {} for jid, load in six.iteritems(loads): - ret[jid] = salt.utils.jid.format_jid_instance(jid, json.loads(load)) + ret[jid] = salt.utils.jid.format_jid_instance(jid, salt.utils.json.loads(load)) return ret diff --git a/salt/returners/mysql.py b/salt/returners/mysql.py index a5cfa5d4717..af6698142b8 100644 --- a/salt/returners/mysql.py +++ b/salt/returners/mysql.py @@ -145,12 +145,12 @@ from __future__ import absolute_import, print_function, unicode_literals # Import python libs from contextlib import contextmanager import sys -import json import logging # Import salt libs import salt.returners import salt.utils.jid +import salt.utils.json import salt.exceptions # Import 3rd-party libs @@ -285,14 +285,14 @@ def returner(ret): try: with _get_serv(ret, commit=True) as cur: sql = '''INSERT INTO `salt_returns` - (`fun`, `jid`, `return`, `id`, `success`, `full_ret` ) - VALUES (%s, %s, %s, %s, %s, %s)''' + (`fun`, `jid`, `return`, `id`, `success`, `full_ret`) + VALUES (%s, %s, %s, %s, %s, %s)''' cur.execute(sql, (ret['fun'], ret['jid'], - json.dumps(ret['return']), + salt.utils.json.dumps(ret['return']), ret['id'], ret.get('success', False), - json.dumps(ret))) + salt.utils.json.dumps(ret))) except salt.exceptions.SaltMasterError as exc: log.critical(exc) log.critical('Could not store return with MySQL returner. MySQL server unavailable.') @@ -309,9 +309,9 @@ def event_return(events): for event in events: tag = event.get('tag', '') data = event.get('data', '') - sql = '''INSERT INTO `salt_events` (`tag`, `data`, `master_id` ) + sql = '''INSERT INTO `salt_events` (`tag`, `data`, `master_id`) VALUES (%s, %s, %s)''' - cur.execute(sql, (tag, json.dumps(data), __opts__['id'])) + cur.execute(sql, (tag, salt.utils.json.dumps(data), __opts__['id'])) def save_load(jid, load, minions=None): @@ -320,12 +320,10 @@ def save_load(jid, load, minions=None): ''' with _get_serv(commit=True) as cur: - sql = '''INSERT INTO `jids` - (`jid`, `load`) - VALUES (%s, %s)''' + sql = '''INSERT INTO `jids` (`jid`, `load`) VALUES (%s, %s)''' try: - cur.execute(sql, (jid, json.dumps(load))) + cur.execute(sql, (jid, salt.utils.json.dumps(load))) except MySQLdb.IntegrityError: # https://github.com/saltstack/salt/issues/22171 # Without this try:except: we get tons of duplicate entry errors @@ -350,7 +348,7 @@ def get_load(jid): cur.execute(sql, (jid,)) data = cur.fetchone() if data: - return json.loads(data[0]) + return salt.utils.json.loads(data[0]) return {} @@ -368,7 +366,7 @@ def get_jid(jid): ret = {} if data: for minion, full_ret in data: - ret[minion] = json.loads(full_ret) + ret[minion] = salt.utils.json.loads(full_ret) return ret @@ -392,7 +390,7 @@ def get_fun(fun): ret = {} if data: for minion, _, full_ret in data: - ret[minion] = json.loads(full_ret) + ret[minion] = salt.utils.json.loads(full_ret) return ret @@ -409,8 +407,9 @@ def get_jids(): data = cur.fetchall() ret = {} for jid in data: - ret[jid[0]] = salt.utils.jid.format_jid_instance(jid[0], - json.loads(jid[1])) + ret[jid[0]] = salt.utils.jid.format_jid_instance( + jid[0], + salt.utils.json.loads(jid[1])) return ret @@ -434,8 +433,9 @@ def get_jids_filter(count, filter_find_job=True): data = cur.fetchall() ret = [] for jid in data: - ret.append(salt.utils.jid.format_jid_instance_ext(jid[0], - json.loads(jid[1]))) + ret.append(salt.utils.jid.format_jid_instance_ext( + jid[0], + salt.utils.json.loads(jid[1]))) return ret diff --git a/salt/returners/odbc.py b/salt/returners/odbc.py index 0f0c431f43f..f2d3695e052 100644 --- a/salt/returners/odbc.py +++ b/salt/returners/odbc.py @@ -123,15 +123,12 @@ To override individual configuration items, append --return_kwargs '{"key:": "va salt '*' test.ping --return odbc --return_kwargs '{"dsn": "dsn-name"}' ''' -from __future__ import absolute_import, print_function, unicode_literals -# Let's not allow PyLint complain about string substitution -# pylint: disable=W1321,E1321 - # Import python libs -import json +from __future__ import absolute_import, print_function, unicode_literals # Import Salt libs import salt.utils.jid +import salt.utils.json import salt.returners # FIXME We'll need to handle this differently for Windows. @@ -205,10 +202,10 @@ def returner(ret): sql, ( ret['fun'], ret['jid'], - json.dumps(ret['return']), + salt.utils.json.dumps(ret['return']), ret['id'], ret['success'], - json.dumps(ret) + salt.utils.json.dumps(ret) ) ) _close_conn(conn) @@ -222,7 +219,7 @@ def save_load(jid, load, minions=None): cur = conn.cursor() sql = '''INSERT INTO jids (jid, load) VALUES (?, ?)''' - cur.execute(sql, (jid, json.dumps(load))) + cur.execute(sql, (jid, salt.utils.json.dumps(load))) _close_conn(conn) @@ -244,7 +241,7 @@ def get_load(jid): cur.execute(sql, (jid,)) data = cur.fetchone() if data: - return json.loads(data) + return salt.utils.json.loads(data) _close_conn(conn) return {} @@ -262,7 +259,7 @@ def get_jid(jid): ret = {} if data: for minion, full_ret in data: - ret[minion] = json.loads(full_ret) + ret[minion] = salt.utils.json.loads(full_ret) _close_conn(conn) return ret @@ -286,7 +283,7 @@ def get_fun(fun): ret = {} if data: for minion, _, retval in data: - ret[minion] = json.loads(retval) + ret[minion] = salt.utils.json.loads(retval) _close_conn(conn) return ret @@ -303,7 +300,7 @@ def get_jids(): data = cur.fetchall() ret = {} for jid, load in data: - ret[jid] = salt.utils.jid.format_jid_instance(jid, json.loads(load)) + ret[jid] = salt.utils.jid.format_jid_instance(jid, salt.utils.json.loads(load)) _close_conn(conn) return ret diff --git a/salt/returners/postgres.py b/salt/returners/postgres.py index 8ccbacbf79b..573de68ecf8 100644 --- a/salt/returners/postgres.py +++ b/salt/returners/postgres.py @@ -127,17 +127,15 @@ To override individual configuration items, append --return_kwargs '{"key:": "va ''' from __future__ import absolute_import, print_function, unicode_literals -# Let's not allow PyLint complain about string substitution -# pylint: disable=W1321,E1321 # Import python libs -import json import sys import logging from contextlib import contextmanager # Import Salt libs import salt.utils.jid +import salt.utils.json import salt.returners import salt.exceptions @@ -235,10 +233,10 @@ def returner(ret): sql, ( ret['fun'], ret['jid'], - json.dumps(ret['return']), + salt.utils.json.dumps(ret['return']), ret['id'], ret.get('success', False), - json.dumps(ret))) + salt.utils.json.dumps(ret))) except salt.exceptions.SaltMasterError: log.critical('Could not store return with postgres returner. PostgreSQL server unavailable.') @@ -257,7 +255,7 @@ def event_return(events): sql = '''INSERT INTO salt_events (tag, data, master_id) VALUES (%s, %s, %s)''' cur.execute(sql, (tag, - json.dumps(data), + salt.utils.json.dumps(data), __opts__['id'])) @@ -273,7 +271,7 @@ def save_load(jid, load, minions=None): # pylint: disable=unused-argument try: cur.execute(sql, (jid, - json.dumps(load))) + salt.utils.json.dumps(load))) except psycopg2.IntegrityError: # https://github.com/saltstack/salt/issues/22171 # Without this try:except: we get tons of duplicate entry errors @@ -297,7 +295,7 @@ def get_load(jid): cur.execute(sql, (jid,)) data = cur.fetchone() if data: - return json.loads(data[0]) + return salt.utils.json.loads(data[0]) return {} @@ -315,7 +313,7 @@ def get_jid(jid): ret = {} if data: for minion, full_ret in data: - ret[minion] = json.loads(full_ret) + ret[minion] = salt.utils.json.loads(full_ret) return ret @@ -339,7 +337,7 @@ def get_fun(fun): ret = {} if data: for minion, _, full_ret in data: - ret[minion] = json.loads(full_ret) + ret[minion] = salt.utils.json.loads(full_ret) return ret @@ -357,7 +355,7 @@ def get_jids(): ret = {} for jid, load in data: ret[jid] = salt.utils.jid.format_jid_instance(jid, - json.loads(load)) + salt.utils.json.loads(load)) return ret diff --git a/salt/returners/postgres_local_cache.py b/salt/returners/postgres_local_cache.py index 95c239a7b51..d20ceac9520 100644 --- a/salt/returners/postgres_local_cache.py +++ b/salt/returners/postgres_local_cache.py @@ -109,13 +109,13 @@ Required python modules: psycopg2 # Import python libs from __future__ import absolute_import, print_function, unicode_literals -import json import logging import re import sys # Import salt libs import salt.utils.jid +import salt.utils.json from salt.ext import six # Import third party libs @@ -166,7 +166,7 @@ def _format_job_instance(job): Format the job instance correctly ''' ret = {'Function': job.get('fun', 'unknown-function'), - 'Arguments': json.loads(job.get('arg', '[]')), + 'Arguments': salt.utils.json.loads(job.get('arg', '[]')), # unlikely but safeguard from invalid returns 'Target': job.get('tgt', 'unknown-target'), 'Target-type': job.get('tgt_type', 'list'), @@ -241,7 +241,7 @@ def returner(load): sql, ( load['fun'], load['jid'], - json.dumps(job_ret), + salt.utils.json.dumps(job_ret), load['id'], load.get('success'), ) @@ -266,7 +266,7 @@ def event_return(events): sql = '''INSERT INTO salt_events (tag, data, master_id) VALUES (%s, %s, %s)''' - cur.execute(sql, (tag, json.dumps(data), __opts__['id'])) + cur.execute(sql, (tag, salt.utils.json.dumps(data), __opts__['id'])) _close_conn(conn) @@ -294,7 +294,7 @@ def save_load(jid, clear_load, minions=None): six.text_type(clear_load.get("kwargs")), six.text_type(clear_load.get("ret")), six.text_type(clear_load.get("user")), - six.text_type(json.dumps(clear_load.get("arg"))), + six.text_type(salt.utils.json.dumps(clear_load.get("arg"))), six.text_type(clear_load.get("fun")), ) ) @@ -371,7 +371,7 @@ def get_jid(jid): ret = {} if data: for minion, full_ret in data: - ret_data = json.loads(full_ret) + ret_data = salt.utils.json.loads(full_ret) if not isinstance(ret_data, dict) or 'return' not in ret_data: # Convert the old format in which the return contains the only return data to the # new that is dict containing 'return' and optionally 'retcode' and 'success'. diff --git a/salt/returners/rawfile_json.py b/salt/returners/rawfile_json.py index 77cbb09db52..7f78d20b130 100644 --- a/salt/returners/rawfile_json.py +++ b/salt/returners/rawfile_json.py @@ -20,10 +20,10 @@ to restrict the events that are written. # Import python libs from __future__ import absolute_import, print_function, with_statement, unicode_literals import logging -import json import salt.returners import salt.utils.files +import salt.utils.json log = logging.getLogger(__name__) @@ -58,7 +58,8 @@ def returner(ret): opts = _get_options({}) # Pass in empty ret, since this is a list of events try: with salt.utils.files.flopen(opts['filename'], 'a') as logfile: - logfile.write(json.dumps(ret)+'\n') + salt.utils.json.dump(ret, logfile) + logfile.write(str('\n')) # future lint: disable=blacklisted-function except: log.error('Could not write to rawdata_json file %s', opts['filename']) raise @@ -76,8 +77,8 @@ def event_return(events): try: with salt.utils.files.flopen(opts['filename'], 'a') as logfile: for event in events: - json.dump(event, logfile) - logfile.write('\n') + salt.utils.json.dump(event, logfile) + logfile.write(str('\n')) # future lint: disable=blacklisted-function except: log.error('Could not write to rawdata_json file %s', opts['filename']) raise diff --git a/salt/returners/redis_return.py b/salt/returners/redis_return.py index 52a304768c7..c51e754350e 100644 --- a/salt/returners/redis_return.py +++ b/salt/returners/redis_return.py @@ -84,15 +84,14 @@ cluster.skip_full_coverage_check: ``False`` ''' - # Import Python libs from __future__ import absolute_import, print_function, unicode_literals -import json import logging # Import Salt libs import salt.returners import salt.utils.jid +import salt.utils.json import salt.utils.platform # Import 3rd-party libs @@ -198,7 +197,7 @@ def returner(ret): serv = _get_serv(ret) pipeline = serv.pipeline(transaction=False) minion, jid = ret['id'], ret['jid'] - pipeline.hset('ret:{0}'.format(jid), minion, json.dumps(ret)) + pipeline.hset('ret:{0}'.format(jid), minion, salt.utils.json.dumps(ret)) pipeline.expire('ret:{0}'.format(jid), _get_ttl()) pipeline.set('{0}:{1}'.format(minion, ret['fun']), jid) pipeline.sadd('minions', minion) @@ -210,7 +209,7 @@ def save_load(jid, load, minions=None): Save the load to the specified jid ''' serv = _get_serv(ret=None) - serv.setex('load:{0}'.format(jid), _get_ttl(), json.dumps(load)) + serv.setex('load:{0}'.format(jid), _get_ttl(), salt.utils.json.dumps(load)) def save_minions(jid, minions, syndic_id=None): # pylint: disable=unused-argument @@ -227,7 +226,7 @@ def get_load(jid): serv = _get_serv(ret=None) data = serv.get('load:{0}'.format(jid)) if data: - return json.loads(data) + return salt.utils.json.loads(data) return {} @@ -239,7 +238,7 @@ def get_jid(jid): ret = {} for minion, data in six.iteritems(serv.hgetall('ret:{0}'.format(jid))): if data: - ret[minion] = json.loads(data) + ret[minion] = salt.utils.json.loads(data) return ret @@ -259,7 +258,7 @@ def get_fun(fun): continue data = serv.get('{0}:{1}'.format(minion, jid)) if data: - ret[minion] = json.loads(data) + ret[minion] = salt.utils.json.loads(data) return ret @@ -272,7 +271,7 @@ def get_jids(): for s in serv.mget(serv.keys('load:*')): if s is None: continue - load = json.loads(s) + load = salt.utils.json.loads(s) jid = load['jid'] ret[jid] = salt.utils.jid.format_jid_instance(jid, load) return ret diff --git a/salt/returners/splunk.py b/salt/returners/splunk.py index c949dcbecae..7ddcc0c8715 100644 --- a/salt/returners/splunk.py +++ b/salt/returners/splunk.py @@ -17,17 +17,17 @@ Run a test by using ``salt-call test.ping --return splunk`` Written by Scott Pack (github.com/scottjpack) ''' +# Import Python libs from __future__ import absolute_import, print_function, unicode_literals - - -import socket - -#Imports for http event forwarder -import requests -import json -import time import logging +import requests +import socket +import time +# Import salt libs +import salt.utils.json + +# Import 3rd-party libs from salt.ext import six _max_content_bytes = 100000 @@ -74,17 +74,18 @@ def _send_splunk(event, index_override=None, sourcetype_override=None): This is available on Splunk Enterprise version 6.3 or higher. ''' - #Get Splunk Options + # Get Splunk Options opts = _get_options() - logging.info('Options: %s', json.dumps(opts)) + log.info(str('Options: %s'), # future lint: disable=blacklisted-function + salt.utils.json.dumps(opts)) http_event_collector_key = opts['token'] http_event_collector_host = opts['indexer'] - #Set up the collector + # Set up the collector splunk_event = http_event_collector(http_event_collector_key, http_event_collector_host) - #init the payload + # init the payload payload = {} - #Set up the event metadata + # Set up the event metadata if index_override is None: payload.update({"index": opts['index']}) else: @@ -94,10 +95,11 @@ def _send_splunk(event, index_override=None, sourcetype_override=None): else: payload.update({"index": sourcetype_override}) - #Add the event + # Add the event payload.update({"event": event}) - logging.info('Payload: %s', json.dumps(payload)) - #fire it off + log.info(str('Payload: %s'), # future lint: disable=blacklisted-function + salt.utils.json.dumps(payload)) + # Fire it off splunk_event.sendEvent(payload) return True @@ -158,7 +160,10 @@ class http_event_collector(object): data.update(payload) # send event to http event collector - r = requests.post(self.server_uri, data=json.dumps(data), headers=headers, verify=http_event_collector_SSL_verify) + r = requests.post(self.server_uri, + data=salt.utils.json.dumps(data), + headers=headers, + verify=http_event_collector_SSL_verify) # Print debug info if flag set if http_event_collector_debug: @@ -172,9 +177,10 @@ class http_event_collector(object): if 'host' not in payload: payload.update({"host": self.host}) - payloadLength = len(json.dumps(payload)) + serialized_payload = salt.utils.json.dumps(payload) + payloadLength = len(serialized_payload) - if (self.currentByteLength+payloadLength) > self.maxByteLength: + if (self.currentByteLength + payloadLength) > self.maxByteLength: self.flushBatch() # Print debug info if flag set if http_event_collector_debug: @@ -190,7 +196,7 @@ class http_event_collector(object): data = {"time": eventtime} data.update(payload) - self.batchEvents.append(json.dumps(data)) + self.batchEvents.append(serialized_payload) def flushBatch(self): # Method to flush the batch list of events diff --git a/salt/returners/sqlite3_return.py b/salt/returners/sqlite3_return.py index 2458b559117..bf1918b838b 100644 --- a/salt/returners/sqlite3_return.py +++ b/salt/returners/sqlite3_return.py @@ -84,11 +84,11 @@ from __future__ import absolute_import, print_function, unicode_literals # Import python libs import logging -import json import datetime # Import Salt libs import salt.utils.jid +import salt.utils.json import salt.returners # Import 3rd-party libs @@ -174,7 +174,7 @@ def returner(ret): 'id': ret['id'], 'fun_args': six.text_type(ret['fun_args']) if ret.get('fun_args') else None, 'date': six.text_type(datetime.datetime.now()), - 'full_ret': json.dumps(ret['return']), + 'full_ret': salt.utils.json.dumps(ret['return']), 'success': ret.get('success', '')}) _close_conn(conn) @@ -189,7 +189,7 @@ def save_load(jid, load, minions=None): sql = '''INSERT INTO jids (jid, load) VALUES (:jid, :load)''' cur.execute(sql, {'jid': jid, - 'load': json.dumps(load)}) + 'load': salt.utils.json.dumps(load)}) _close_conn(conn) @@ -212,7 +212,7 @@ def get_load(jid): {'jid': jid}) data = cur.fetchone() if data: - return json.loads(data[0].encode()) + return salt.utils.json.loads(data[0].encode()) _close_conn(conn) return {} @@ -231,7 +231,7 @@ def get_jid(jid): log.debug('query result: %s', data) ret = {} if data and len(data) > 1: - ret = {six.text_type(data[0]): {u'return': json.loads(data[1])}} + ret = {six.text_type(data[0]): {'return': salt.utils.json.loads(data[1])}} log.debug('ret: %s', ret) _close_conn(conn) return ret @@ -260,7 +260,7 @@ def get_fun(fun): # pylint score :-) data.pop() for minion, ret in data: - ret[minion] = json.loads(ret) + ret[minion] = salt.utils.json.loads(ret) _close_conn(conn) return ret @@ -277,7 +277,7 @@ def get_jids(): data = cur.fetchall() ret = {} for jid, load in data: - ret[jid] = salt.utils.jid.format_jid_instance(jid, json.loads(load)) + ret[jid] = salt.utils.jid.format_jid_instance(jid, salt.utils.json.loads(load)) _close_conn(conn) return ret diff --git a/salt/returners/syslog_return.py b/salt/returners/syslog_return.py index 3fbca3e5f12..736df2fd6f7 100644 --- a/salt/returners/syslog_return.py +++ b/salt/returners/syslog_return.py @@ -90,7 +90,6 @@ from __future__ import absolute_import, print_function, unicode_literals import logging # Import python libs -import json try: import syslog HAS_SYSLOG = True @@ -99,6 +98,7 @@ except ImportError: # Import Salt libs import salt.utils.jid +import salt.utils.json import salt.returners from salt.ext import six @@ -202,7 +202,7 @@ def returner(ret): syslog.openlog(_options.get('tag', 'salt-minion'), logoption) # Send log of given level and facility - syslog.syslog(facility | level, '{0}'.format(json.dumps(ret))) + syslog.syslog(facility | level, salt.utils.json.dumps(ret)) # Close up to reset syslog to defaults syslog.closelog() diff --git a/salt/roster/ansible.py b/salt/roster/ansible.py index d97b1068841..d92c8461dd0 100644 --- a/salt/roster/ansible.py +++ b/salt/roster/ansible.py @@ -92,12 +92,12 @@ from __future__ import absolute_import import os import re import fnmatch -import json import subprocess # Import Salt libs import salt.utils.args import salt.utils.files +import salt.utils.json import salt.utils.stringutils from salt.roster import get_roster_file @@ -247,7 +247,7 @@ class Script(Target): self.tgt = tgt self.tgt_type = tgt_type inventory, error = subprocess.Popen([inventory_file], shell=True, stdout=subprocess.PIPE).communicate() - self.inventory = json.loads(salt.utils.stringutils.to_str(inventory)) + self.inventory = salt.utils.json.loads(salt.utils.stringutils.to_str(inventory)) self.meta = self.inventory.get('_meta', {}) self.groups = dict() self.hostvars = dict() diff --git a/salt/runners/ddns.py b/salt/runners/ddns.py index c8738e73255..cf3c86e801a 100644 --- a/salt/runners/ddns.py +++ b/salt/runners/ddns.py @@ -15,7 +15,6 @@ from __future__ import absolute_import, print_function, unicode_literals # Import python libs import os import logging -import json # Import third party libs HAS_LIBS = False @@ -29,6 +28,7 @@ except ImportError: # Import salt libs import salt.utils.files +import salt.utils.json log = logging.getLogger(__name__) @@ -48,7 +48,7 @@ def _get_keyring(keyfile): keyring = None if keyfile and os.path.isfile(os.path.expanduser(keyfile)): with salt.utils.files.fopen(keyfile) as _f: - keyring = dns.tsigkeyring.from_text(json.load(_f)) + keyring = dns.tsigkeyring.from_text(salt.utils.json.load(_f)) return keyring diff --git a/salt/runners/digicertapi.py b/salt/runners/digicertapi.py index 5b7e293fc38..15e2658fadf 100644 --- a/salt/runners/digicertapi.py +++ b/salt/runners/digicertapi.py @@ -41,12 +41,12 @@ import logging import tempfile import subprocess import collections -import json import re import salt.syspaths as syspaths import salt.cache import salt.utils.files import salt.utils.http +import salt.utils.json from salt.ext import six from salt.ext.six.moves import range from salt.exceptions import (CommandExecutionError, SaltRunnerError) @@ -445,7 +445,7 @@ def order_certificate(minion_id, common_name, organization_id, validity_years, body['renewal_of_order_id'] = renewal_of_order_id body['certificate'] = certificate - encoded_body = json.dumps(body) + encoded_body = salt.utils.json.dumps(body) qdata = salt.utils.http.query( '{0}/order/certificate/ssl'.format(_base_url()), diff --git a/salt/runners/mattermost.py b/salt/runners/mattermost.py index 3d176f6cc8a..357e2f68292 100644 --- a/salt/runners/mattermost.py +++ b/salt/runners/mattermost.py @@ -17,9 +17,11 @@ Module for sending messages to Mattermost # Import Python libs from __future__ import absolute_import, print_function, unicode_literals -import json import logging +# Import Salt libs +import salt.utils.json + # Import 3rd-party libs from salt.ext import six @@ -129,9 +131,11 @@ def post_message(message, parameters['username'] = username parameters['text'] = '```' + message + '```' # pre-formatted, fixed-width text log.debug('Parameters: %s', parameters) - result = salt.utils.mattermost.query(api_url=api_url, - hook=hook, - data='payload={0}'.format(json.dumps(parameters))) + data = salt.utils.json.dumps(parameters) + result = salt.utils.mattermost.query( + api_url=api_url, + hook=hook, + data=str('payload={0}').format(data)) # future lint: blacklisted-function if result: return True diff --git a/salt/runners/pagerduty.py b/salt/runners/pagerduty.py index c3bb94d8f88..7095c9836c5 100644 --- a/salt/runners/pagerduty.py +++ b/salt/runners/pagerduty.py @@ -19,10 +19,10 @@ from __future__ import absolute_import, print_function, unicode_literals # Import python libs import yaml -import json # Import salt libs import salt.utils.functools +import salt.utils.json import salt.utils.pagerduty from salt.ext import six @@ -180,7 +180,7 @@ def create_event(service_key=None, description=None, details=None, if isinstance(details, six.string_types): details = {'details': details} - ret = json.loads(salt.utils.pagerduty.query( + ret = salt.utils.json.loads(salt.utils.pagerduty.query( method='POST', profile_dict=__salt__['config.option'](profile), api_key=service_key, diff --git a/salt/runners/venafiapi.py b/salt/runners/venafiapi.py index 95b52aad0aa..dc4a5726fc2 100644 --- a/salt/runners/venafiapi.py +++ b/salt/runners/venafiapi.py @@ -30,7 +30,6 @@ file and set the ``api_key`` to it: api_key: abcdef01-2345-6789-abcd-ef0123456789 ''' from __future__ import absolute_import, print_function, unicode_literals -import json import logging import os import tempfile @@ -40,6 +39,7 @@ from Crypto.PublicKey import RSA import salt.cache import salt.syspaths as syspaths import salt.utils.files +import salt.utils.json import salt.utils.stringutils from salt.exceptions import CommandExecutionError @@ -305,7 +305,7 @@ def request( password=password, ) - pdata = json.dumps({ + pdata = salt.utils.json.dumps({ 'zoneId': zone_id, 'certificateSigningRequest': csr, }) @@ -379,7 +379,7 @@ def register(email): data = __utils__['http.query']( '{0}/useraccounts'.format(_base_url()), method='POST', - data=json.dumps({ + data=salt.utils.json.dumps({ 'username': email, 'userAccountType': 'API', }), diff --git a/salt/runners/vistara.py b/salt/runners/vistara.py index 609d1ee1229..32774949727 100644 --- a/salt/runners/vistara.py +++ b/salt/runners/vistara.py @@ -23,7 +23,6 @@ For example ``/etc/salt/master.d/_vistara.conf``: from __future__ import absolute_import, print_function, unicode_literals # Import Python libs -import json import logging # Import Salt libs @@ -144,7 +143,7 @@ def _search_devices(query_string, client_id, access_token): if not respbody: return False - respbodydict = json.loads(resp['body']) + respbodydict = salt.utils.json.loads(resp['body']) deviceresults = respbodydict['results'] return deviceresults @@ -174,7 +173,7 @@ def _delete_resource(device_id, client_id, access_token): if not respbody: return False - respbodydict = json.loads(resp['body']) + respbodydict = salt.utils.json.loads(resp['body']) return respbodydict @@ -217,5 +216,5 @@ def _get_oath2_access_token(client_key, client_secret): if not respbody: return False - access_token = json.loads(respbody)['access_token'] + access_token = salt.utils.json.loads(respbody)['access_token'] return access_token diff --git a/salt/sdb/tism.py b/salt/sdb/tism.py index 8843190386c..fab12d0ab93 100644 --- a/salt/sdb/tism.py +++ b/salt/sdb/tism.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- ''' -tISM - the Immutalbe Secrets Manager SDB Module +tISM - the Immutable Secrets Manager SDB Module :maintainer: tISM :maturity: New @@ -16,7 +16,9 @@ This module will decrypt PGP encrypted secrets against a tISM server. sdb://tism/hQEMAzJ+GfdAB3KqAQf9E3cyvrPEWR1sf1tMvH0nrJ0bZa9kDFLPxvtwAOqlRiNp0F7IpiiVRF+h+sW5Mb4ffB1TElMzQ+/G5ptd6CjmgBfBsuGeajWmvLEi4lC6/9v1rYGjjLeOCCcN4Dl5AHlxUUaSrxB8akTDvSAnPvGhtRTZqDlltl5UEHsyYXM8RaeCrBw5Or1yvC9Ctx2saVp3xmALQvyhzkUv5pTb1mH0I9Z7E0ian07ZUOD+pVacDAf1oQcPpqkeNVTQQ15EP0fDuvnW+a0vxeLhkbFLfnwqhqEsvFxVFLHVLcs2ffE5cceeOMtVo7DS9fCtkdZr5hR7a+86n4hdKfwDMFXiBwSIPMkmY980N/H30L/r50+CBkuI/u4M2pXDcMYsvvt4ajCbJn91qaQ7BDI= -A profile must be setup in the minion configuration or pillar. If you want to use sdb in a runner or pillar you must also place a profile in the master configuration. +A profile must be setup in the minion configuration or pillar. If you want to +use sdb in a runner or pillar you must also place a profile in the master +configuration. .. code-block:: yaml @@ -29,9 +31,9 @@ A profile must be setup in the minion configuration or pillar. If you want to u # Import Python libs from __future__ import absolute_import import logging -import json # Import Salt libs +import salt.utils.json import salt.utils.http as http from salt.exceptions import SaltConfigurationError @@ -60,7 +62,7 @@ def get(key, service=None, profile=None): # pylint: disable=W0613 result = http.query( profile['url'], method='POST', - data=json.dumps(request), + data=salt.utils.json.dumps(request), ) decrypted = result.get('body') diff --git a/salt/serializers/json.py b/salt/serializers/json.py index 64a46657b1e..84c6a6c323f 100644 --- a/salt/serializers/json.py +++ b/salt/serializers/json.py @@ -11,11 +11,12 @@ from __future__ import absolute_import, print_function, unicode_literals try: - import simplejson as json + import simplejson as _json except ImportError: - import json + import json as _json # pylint: disable=blacklisted-import # Import Salt libs +import salt.utils.json from salt.serializers import DeserializationError, SerializationError # Import 3rd-party libs @@ -36,12 +37,13 @@ def deserialize(stream_or_string, **options): try: if not isinstance(stream_or_string, (bytes, six.string_types)): - return json.load(stream_or_string, **options) + return salt.utils.json.load( + stream_or_string, _json_module=_json, **options) if isinstance(stream_or_string, bytes): stream_or_string = stream_or_string.decode('utf-8') - return json.loads(stream_or_string) + return salt.utils.json.loads(stream_or_string, _json_module=_json) except Exception as error: raise DeserializationError(error) @@ -56,8 +58,8 @@ def serialize(obj, **options): try: if 'fp' in options: - return json.dump(obj, **options) + return salt.utils.json.dump(obj, _json_module=_json, **options) else: - return json.dumps(obj, **options) + return salt.utils.json.dumps(obj, _json_module=_json, **options) except Exception as error: raise SerializationError(error) diff --git a/salt/serializers/python.py b/salt/serializers/python.py index 89c993127e9..7272413d898 100644 --- a/salt/serializers/python.py +++ b/salt/serializers/python.py @@ -7,15 +7,15 @@ Implements a Python serializer (via pprint.format) ''' - -from __future__ import absolute_import, print_function, unicode_literals +from __future__ import absolute_import, unicode_literals +import pprint try: - import simplejson as json + import simplejson as _json except ImportError: - import json + import json as _json # pylint: disable=blacklisted-import -import pprint +import salt.utils.json __all__ = ['serialize', 'available'] @@ -34,4 +34,10 @@ def serialize(obj, **options): # there's probably a more performant way to do this... # TODO remove json round-trip when all dataset will use # serializers - return pprint.pformat(json.loads(json.dumps(obj)), **options) + return pprint.pformat( + salt.utils.json.loads( + salt.utils.json.dumps(obj, _json_module=_json), + _json_module=_json + ), + **options + ) diff --git a/salt/states/bigip.py b/salt/states/bigip.py index 11247b3618e..4c6495e317b 100644 --- a/salt/states/bigip.py +++ b/salt/states/bigip.py @@ -8,7 +8,8 @@ A state module designed to enforce load-balancing configurations for F5 Big-IP e # Import Python libs from __future__ import absolute_import -import json +# Import Salt libs +import salt.utils.json # Import 3rd-party libs from salt.ext import six @@ -123,7 +124,7 @@ def _test_output(ret, action, params): ret['comment'] += 'The modify action will attempt to modify an existing entity only if it exists.\n' ret['comment'] += 'An iControl REST Request will be made using the parameters:\n' - ret['comment'] += json.dumps(params, indent=4) + ret['comment'] += salt.utils.json.dumps(params, indent=4) ret['changes'] = {} # Return ``None`` when running with ``test=true``. diff --git a/salt/states/boto3_sns.py b/salt/states/boto3_sns.py index 720277e5384..8b7cf78874e 100644 --- a/salt/states/boto3_sns.py +++ b/salt/states/boto3_sns.py @@ -58,8 +58,9 @@ from __future__ import absolute_import import re import logging -import json import copy + +import salt.utils.json from salt.ext import six log = logging.getLogger(__name__) @@ -157,7 +158,7 @@ def topic_present(name, subscriptions=None, attributes=None, ret['comment'] += ' Attribute {0} would be updated on topic {1}.'.format(attr, TopicArn) ret['result'] = None continue - want_val = want_val if isinstance(want_val, six.string_types) else json.dumps(want_val) + want_val = want_val if isinstance(want_val, six.string_types) else salt.utils.json.dumps(want_val) if __salt__['boto3_sns.set_topic_attributes'](TopicArn, attr, want_val, region=region, key=key, keyid=keyid, profile=profile): ret['comment'] += ' Attribute {0} set to {1} on topic {2}.'.format(attr, want_val, @@ -307,6 +308,12 @@ def topic_absent(name, unsubscribe=False, region=None, key=None, keyid=None, pro def _json_objs_equal(left, right): - left = __utils__['boto3.ordered'](json.loads(left) if isinstance(left, six.string_types) else left) - right = __utils__['boto3.ordered'](json.loads(right) if isinstance(right, six.string_types) else right) + left = __utils__['boto3.ordered']( + salt.utils.json.loads(left) + if isinstance(left, six.string_types) + else left) + right = __utils__['boto3.ordered']( + salt.utils.json.loads(right) + if isinstance(right, six.string_types) + else right) return left == right diff --git a/salt/states/boto_apigateway.py b/salt/states/boto_apigateway.py index 4146fb772e1..ee0bcc6c154 100644 --- a/salt/states/boto_apigateway.py +++ b/salt/states/boto_apigateway.py @@ -48,18 +48,20 @@ config: # Import Python Libs from __future__ import absolute_import +import hashlib import logging import os -import os.path -import hashlib import re -import json import yaml + # Import Salt Libs -from salt.ext import six import salt.utils.files +import salt.utils.json from salt.utils.yamlloader import SaltYamlSafeLoader +# Import 3rd-party libs +from salt.ext import six + log = logging.getLogger(__name__) @@ -447,7 +449,7 @@ def _dict_to_json_pretty(d, sort_keys=True): ''' helper function to generate pretty printed json output ''' - return json.dumps(d, indent=4, separators=(',', ': '), sort_keys=sort_keys) + return salt.utils.json.dumps(d, indent=4, separators=(',', ': '), sort_keys=sort_keys) # Heuristic on whether or not the property name loosely matches given set of 'interesting' factors diff --git a/salt/states/boto_cfn.py b/salt/states/boto_cfn.py index 3db51d75792..65bbcf4be00 100644 --- a/salt/states/boto_cfn.py +++ b/salt/states/boto_cfn.py @@ -40,10 +40,10 @@ Connection module for Amazon Cloud Formation # Import Python libs from __future__ import absolute_import import logging -import json # Import Salt libs import salt.utils.compat +import salt.utils.json from salt.ext import six # Import 3rd-party libs @@ -159,8 +159,8 @@ def present(name, template_body=None, template_url=None, parameters=None, notifi if __salt__['boto_cfn.exists'](name, region, key, keyid, profile): template = __salt__['boto_cfn.get_template'](name, region, key, keyid, profile) template = template['GetTemplateResponse']['GetTemplateResult']['TemplateBody'].encode('ascii', 'ignore') - template = json.loads(template) - _template_body = json.loads(template_body) + template = salt.utils.json.loads(template) + _template_body = salt.utils.json.loads(template_body) compare = salt.utils.compat.cmp(template, _template_body) if compare != 0: log.debug('Templates are not the same. Compare value is {0}'.format(compare)) diff --git a/salt/states/boto_cloudwatch_event.py b/salt/states/boto_cloudwatch_event.py index 3faddf7347a..054cc89e299 100644 --- a/salt/states/boto_cloudwatch_event.py +++ b/salt/states/boto_cloudwatch_event.py @@ -52,14 +52,15 @@ config: ''' -# Import Python Libs +# Import Python libs from __future__ import absolute_import import logging import os -import os.path -import json -# Import Salt Libs +# Import Salt libs +import salt.utils.json + +# Import 3rd-party libs from salt.ext import six log = logging.getLogger(__name__) @@ -132,7 +133,7 @@ def present(name, Name=None, Name = Name if Name else name if isinstance(Targets, six.string_types): - Targets = json.loads(Targets) + Targets = salt.utils.json.loads(Targets) if Targets is None: Targets = [] diff --git a/salt/states/boto_datapipeline.py b/salt/states/boto_datapipeline.py index f29c3bcbc45..65ef3aa7fbc 100644 --- a/salt/states/boto_datapipeline.py +++ b/salt/states/boto_datapipeline.py @@ -54,9 +54,9 @@ from __future__ import absolute_import import copy import datetime import difflib -import json # Import Salt lobs +import salt.utils.json from salt.ext import six from salt.ext.six.moves import zip @@ -399,10 +399,10 @@ def _diff(old_pipeline_definition, new_pipeline_definition): new_pipeline_definition.pop('ResponseMetadata', None) diff = difflib.unified_diff( - json.dumps(old_pipeline_definition, indent=4).splitlines(1), - json.dumps(new_pipeline_definition, indent=4).splitlines(1), + salt.utils.json.dumps(old_pipeline_definition, indent=4).splitlines(True), + salt.utils.json.dumps(new_pipeline_definition, indent=4).splitlines(True), ) - return ''.join(diff) + return str('').join(diff) # future lint: disable=blacklisted-function def _standardize(structure): diff --git a/salt/states/boto_elasticsearch_domain.py b/salt/states/boto_elasticsearch_domain.py index 2e544a655b8..7fd5a780be9 100644 --- a/salt/states/boto_elasticsearch_domain.py +++ b/salt/states/boto_elasticsearch_domain.py @@ -78,14 +78,15 @@ config: ''' -# Import Python Libs +# Import Python libs from __future__ import absolute_import import logging import os -import os.path -import json -# Import Salt Libs +# Import Salt libs +import salt.utils.json + +# Import 3rd-party libs from salt.ext import six log = logging.getLogger(__name__) @@ -224,7 +225,7 @@ def present(name, DomainName, Tags = {} if AccessPolicies is not None and isinstance(AccessPolicies, six.string_types): try: - AccessPolicies = json.loads(AccessPolicies) + AccessPolicies = salt.utils.json.loads(AccessPolicies) except ValueError as e: ret['result'] = False ret['comment'] = 'Failed to create domain: {0}.'.format(e.message) @@ -275,7 +276,7 @@ def present(name, DomainName, _describe = __salt__['boto_elasticsearch_domain.describe'](DomainName=DomainName, region=region, key=key, keyid=keyid, profile=profile)['domain'] - _describe['AccessPolicies'] = json.loads(_describe['AccessPolicies']) + _describe['AccessPolicies'] = salt.utils.json.loads(_describe['AccessPolicies']) # When EBSEnabled is false, describe returns extra values that can't be set if not _describe.get('EBSOptions', {}).get('EBSEnabled'): diff --git a/salt/states/boto_iam.py b/salt/states/boto_iam.py index 23f01a22053..ff127e57f4c 100644 --- a/salt/states/boto_iam.py +++ b/salt/states/boto_iam.py @@ -134,12 +134,12 @@ passed in as a dict, or as a string to pull from pillars or minion config: # Import Python Libs from __future__ import absolute_import import logging -import json import os # Import Salt Libs import salt.utils.data import salt.utils.files +import salt.utils.json import salt.utils.odict as odict import salt.utils.dictupdate as dictupdate from salt.ext import six @@ -159,8 +159,8 @@ __virtualname__ = 'boto_iam' if six.PY2: def _byteify(thing): - # Note that we intentionally don't treat odicts here - they won't compare equal - # in many circumstances where AWS treats them the same... + # Note that we intentionally don't treat odicts here - they won't + # compare equal in many circumstances where AWS treats them the same... if isinstance(thing, dict): return dict([(_byteify(k), _byteify(v)) for k, v in six.iteritems(thing)]) elif isinstance(thing, list): @@ -567,7 +567,7 @@ def _user_policies_present(name, policies=None, region=None, key=None, keyid=Non policies_to_delete = [] for policy_name, policy in six.iteritems(policies): if isinstance(policy, six.string_types): - dict_policy = _byteify(json.loads(policy, object_pairs_hook=odict.OrderedDict)) + dict_policy = _byteify(salt.utils.json.loads(policy, object_pairs_hook=odict.OrderedDict)) else: dict_policy = _byteify(policy) _policy = _byteify(__salt__['boto_iam.get_user_policy'](name, policy_name, region, key, keyid, profile)) @@ -1030,7 +1030,7 @@ def _group_policies_present( policies_to_delete = [] for policy_name, policy in six.iteritems(policies): if isinstance(policy, six.string_types): - dict_policy = _byteify(json.loads(policy, object_pairs_hook=odict.OrderedDict)) + dict_policy = _byteify(salt.utils.json.loads(policy, object_pairs_hook=odict.OrderedDict)) else: dict_policy = _byteify(policy) _policy = _byteify(__salt__['boto_iam.get_group_policy'](name, policy_name, region, key, keyid, profile)) @@ -1528,12 +1528,12 @@ def policy_present(name, policy_document, path=None, description=None, _describe = __salt__['boto_iam.get_policy_version'](name, policy.get('default_version_id'), region, key, keyid, profile).get('policy_version', {}) if isinstance(_describe['document'], six.string_types): - describeDict = json.loads(_describe['document']) + describeDict = salt.utils.json.loads(_describe['document']) else: describeDict = _describe['document'] if isinstance(policy_document, six.string_types): - policy_document = json.loads(policy_document) + policy_document = salt.utils.json.loads(policy_document) r = salt.utils.data.compare_dicts(describeDict, policy_document) @@ -1545,7 +1545,7 @@ def policy_present(name, policy_document, path=None, description=None, return ret ret['comment'] = ' '.join([ret['comment'], 'Policy to be modified']) - policy_document = json.dumps(policy_document) + policy_document = salt.utils.json.dumps(policy_document) r = __salt__['boto_iam.create_policy_version'](policy_name=name, policy_document=policy_document, diff --git a/salt/states/boto_iot.py b/salt/states/boto_iot.py index b8d8f732551..3d7ab3aae92 100644 --- a/salt/states/boto_iot.py +++ b/salt/states/boto_iot.py @@ -70,19 +70,19 @@ config: - key: askdjghsdfjkghWupUjasdflkdfklgjsdfjajkghs ''' - -# Import Python Libs +# Import Python libs from __future__ import absolute_import +import datetime import logging import os -import os.path -import datetime import time -import json -# Import Salt Libs +# Import Salt libs import salt.utils.data -from salt.ext.six import string_types +import salt.utils.json + +# Import 3rd-party libs +from salt.ext import six log = logging.getLogger(__name__) @@ -363,13 +363,13 @@ def policy_present(name, policyName, policyDocument, _describe = __salt__['boto_iot.describe_policy'](policyName=policyName, region=region, key=key, keyid=keyid, profile=profile)['policy'] - if isinstance(_describe['policyDocument'], string_types): - describeDict = json.loads(_describe['policyDocument']) + if isinstance(_describe['policyDocument'], six.string_types): + describeDict = salt.utils.json.loads(_describe['policyDocument']) else: describeDict = _describe['policyDocument'] - if isinstance(policyDocument, string_types): - policyDocument = json.loads(policyDocument) + if isinstance(policyDocument, six.string_types): + policyDocument = salt.utils.json.loads(policyDocument) r = salt.utils.data.compare_dicts(describeDict, policyDocument) if bool(r): @@ -380,7 +380,7 @@ def policy_present(name, policyName, policyDocument, return ret ret['comment'] = os.linesep.join([ret['comment'], 'Policy to be modified']) - policyDocument = json.dumps(policyDocument) + policyDocument = salt.utils.json.dumps(policyDocument) r = __salt__['boto_iot.create_policy_version'](policyName=policyName, policyDocument=policyDocument, @@ -706,8 +706,8 @@ def topic_rule_present(name, ruleName, sql, actions, description='', _describe = __salt__['boto_iot.describe_topic_rule'](ruleName=ruleName, region=region, key=key, keyid=keyid, profile=profile)['rule'] - if isinstance(actions, string_types): - actions = json.loads(actions) + if isinstance(actions, six.string_types): + actions = salt.utils.json.loads(actions) need_update = False # cmp() function is deprecated in Python 3: use the following as a substitute for 'r'. diff --git a/salt/states/boto_lambda.py b/salt/states/boto_lambda.py index 5fd54d1c0f8..9a048c46acb 100644 --- a/salt/states/boto_lambda.py +++ b/salt/states/boto_lambda.py @@ -64,15 +64,14 @@ config: from __future__ import absolute_import import logging import os -import os.path import hashlib -import json # Import Salt Libs from salt.ext import six import salt.utils.data import salt.utils.dictupdate as dictupdate import salt.utils.files +import salt.utils.json from salt.exceptions import SaltInvocationError log = logging.getLogger(__name__) @@ -208,7 +207,7 @@ def function_present(name, FunctionName, Runtime, Role, Handler, ZipFile=None, if Permissions is not None: if isinstance(Permissions, six.string_types): - Permissions = json.loads(Permissions) + Permissions = salt.utils.json.loads(Permissions) required_keys = set(('Action', 'Principal')) optional_keys = set(('SourceArn', 'SourceAccount', 'Qualifier')) for sid, permission in six.iteritems(Permissions): @@ -321,7 +320,7 @@ def _get_role_arn(name, region=None, key=None, keyid=None, profile=None): def _resolve_vpcconfig(conf, region=None, key=None, keyid=None, profile=None): if isinstance(conf, six.string_types): - conf = json.loads(conf) + conf = salt.utils.json.loads(conf) if not conf: # if the conf is None, we should explicitly set the VpcConfig to # {'SubnetIds': [], 'SecurityGroupIds': []} to take the lambda out of diff --git a/salt/states/boto_route53.py b/salt/states/boto_route53.py index 1686ae7e8ee..eaa8129a1b6 100644 --- a/salt/states/boto_route53.py +++ b/salt/states/boto_route53.py @@ -73,13 +73,14 @@ passed in as a dict, or as a string to pull from pillars or minion config: # Import Python Libs from __future__ import absolute_import -import json +import logging import uuid # Import Salt Libs -from salt.utils import exactly_one +import salt.utils.data +import salt.utils.json from salt.exceptions import SaltInvocationError -import logging + log = logging.getLogger(__name__) @@ -406,7 +407,7 @@ def hosted_zone_present(name, domain_name=None, private_zone=False, caller_ref=N # First translaste vpc_name into a vpc_id if possible if private_zone: - if not exactly_one((vpc_name, vpc_id)): + if not salt.utils.data.exactly_one((vpc_name, vpc_id)): raise SaltInvocationError('Either vpc_name or vpc_id is required when creating a ' 'private zone.') vpcs = __salt__['boto_vpc.describe_vpcs']( @@ -440,7 +441,7 @@ def hosted_zone_present(name, domain_name=None, private_zone=False, caller_ref=N if not deets: create = True else: # Something exists - now does it match our criteria? - if (json.loads(deets['HostedZone']['Config']['PrivateZone']) != + if (salt.utils.json.loads(deets['HostedZone']['Config']['PrivateZone']) != private_zone): create = True else: diff --git a/salt/states/boto_s3_bucket.py b/salt/states/boto_s3_bucket.py index da2c83b2926..eb59881304e 100644 --- a/salt/states/boto_s3_bucket.py +++ b/salt/states/boto_s3_bucket.py @@ -138,13 +138,15 @@ config: ''' -# Import Python Libs +# Import Python libs from __future__ import absolute_import +import copy import logging -from copy import deepcopy -import json # Import Salt libs +import salt.utils.json + +# Import 3rd-party libs from salt.ext import six log = logging.getLogger(__name__) @@ -158,7 +160,7 @@ def __virtual__(): def _normalize_user(user_dict): - ret = deepcopy(user_dict) + ret = copy.deepcopy(user_dict) # 'Type' is required as input to the AWS API, but not returned as output. So # we ignore it everywhere. if 'Type' in ret: @@ -177,7 +179,7 @@ def _prep_acl_for_compare(ACL): ''' Prepares the ACL returned from the AWS API for comparison with a given one. ''' - ret = deepcopy(ACL) + ret = copy.deepcopy(ACL) ret['Owner'] = _normalize_user(ret['Owner']) for item in ret.get('Grants', ()): item['Grantee'] = _normalize_user(item.get('Grantee')) @@ -186,13 +188,13 @@ def _prep_acl_for_compare(ACL): def _acl_to_grant(ACL, owner_canonical_id): if 'AccessControlPolicy' in ACL: - ret = deepcopy(ACL['AccessControlPolicy']) + ret = copy.deepcopy(ACL['AccessControlPolicy']) ret['Owner'] = _normalize_user(ret['Owner']) for item in ACL.get('Grants', ()): item['Grantee'] = _normalize_user(item.get('Grantee')) # If AccessControlPolicy is set, other options are not allowed return ret - owner_canonical_grant = deepcopy(owner_canonical_id) + owner_canonical_grant = copy.deepcopy(owner_canonical_id) owner_canonical_grant.update({'Type': 'CanonicalUser'}) ret = { 'Grants': [], @@ -326,7 +328,7 @@ def _compare_replication(current, desired, region, key, keyid, profile): Replication accepts a non-ARN role name, but always returns an ARN ''' if desired is not None and desired.get('Role'): - desired = deepcopy(desired) + desired = copy.deepcopy(desired) desired['Role'] = _get_role_arn(desired['Role'], region=region, key=key, keyid=keyid, profile=profile) return __utils__['boto3.json_objs_equal'](current, desired) @@ -423,7 +425,7 @@ def present(name, Bucket, RequestPayment = {'Payer': 'BucketOwner'} if Policy: if isinstance(Policy, six.string_types): - Policy = json.loads(Policy) + Policy = salt.utils.json.loads(Policy) Policy = __utils__['boto3.ordered'](Policy) r = __salt__['boto_s3_bucket.exists'](Bucket=Bucket, @@ -551,7 +553,9 @@ def present(name, Bucket, # Policy description is always returned as a JSON string. # Convert it to JSON now for ease of comparisons later. if isinstance(temp, six.string_types): - current = __utils__['boto3.ordered']({'Policy': json.loads(temp)}) + current = __utils__['boto3.ordered']( + {'Policy': salt.utils.json.loads(temp)} + ) if not comparator(current, desired, region, key, keyid, profile): update = True if varname == 'ACL': diff --git a/salt/states/boto_sqs.py b/salt/states/boto_sqs.py index e9b142f8643..1c385a09b95 100644 --- a/salt/states/boto_sqs.py +++ b/salt/states/boto_sqs.py @@ -61,10 +61,12 @@ from __future__ import absolute_import # Import Python libs import difflib -import json import logging import yaml +# Import Salt libs +import salt.utils.json + # Import 3rd-party libs from salt.ext import six @@ -183,12 +185,12 @@ def present( if attr == 'Policy': # Normalize by brute force if isinstance(_val, six.string_types): - _val = json.loads(_val) + _val = salt.utils.json.loads(_val) if isinstance(val, six.string_types): - val = json.loads(val) + val = salt.utils.json.loads(val) if _val != val: log.debug('Policies differ:\n{0}\n{1}'.format(_val, val)) - attrs_to_set[attr] = json.dumps(val, sort_keys=True) + attrs_to_set[attr] = salt.utils.json.dumps(val, sort_keys=True) elif str(_val) != str(val): log.debug('Attributes differ:\n{0}\n{1}'.format(_val, val)) attrs_to_set[attr] = val diff --git a/salt/states/ceph.py b/salt/states/ceph.py index 23194b19725..c91a1203d9d 100644 --- a/salt/states/ceph.py +++ b/salt/states/ceph.py @@ -7,9 +7,9 @@ Manage ceph with salt. # Import Python Libs from __future__ import absolute_import import logging -import json # Import Salt Libs +import salt.utils.json from salt.exceptions import CommandExecutionError, CommandNotFoundError @@ -48,7 +48,7 @@ def _ordereddict2dict(input_ordered_dict): ''' Convert ordered dictionary to a dictionary ''' - return json.loads(json.dumps(input_ordered_dict)) + return salt.utils.json.loads(salt.utils.json.dumps(input_ordered_dict)) def quorum(name, **kwargs): diff --git a/salt/states/cmd.py b/salt/states/cmd.py index d02e59b2f2e..30dabb6f12e 100644 --- a/salt/states/cmd.py +++ b/salt/states/cmd.py @@ -236,12 +236,12 @@ from __future__ import absolute_import, print_function, unicode_literals import os import copy -import json import logging # Import salt libs import salt.utils.args import salt.utils.functools +import salt.utils.json from salt.exceptions import CommandExecutionError, SaltRenderError from salt.ext import six @@ -264,7 +264,7 @@ def _reinterpreted_state(state): is_json = False try: - data = json.loads(out) + data = salt.utils.json.loads(out) if not isinstance(data, dict): return _failout( state, diff --git a/salt/states/elasticsearch.py b/salt/states/elasticsearch.py index c5f297db59b..6eada7eb27d 100644 --- a/salt/states/elasticsearch.py +++ b/salt/states/elasticsearch.py @@ -8,9 +8,10 @@ State module to manage Elasticsearch. # Import python libs from __future__ import absolute_import import logging -import json # Import salt libs +import salt.utils.json + log = logging.getLogger(__name__) @@ -273,7 +274,7 @@ def index_template_present(name, definition, check_definition=False): ret['comment'] = 'Cannot create index template {0}, {1}'.format(name, output) else: if check_definition: - definition_parsed = json.loads(definition) + definition_parsed = salt.utils.json.loads(definition) current_template = __salt__['elasticsearch.index_template_get'](name=name)[name] diff = __utils__['dictdiffer.deep_diff'](current_template, definition_parsed) if len(diff) != 0: @@ -407,13 +408,13 @@ def search_template_absent(name): if template: if __opts__['test']: ret['comment'] = 'Search template {0} will be removed'.format(name) - ret['changes']['old'] = json.loads(template["template"]) + ret['changes']['old'] = salt.utils.json.loads(template["template"]) ret['result'] = None else: ret['result'] = __salt__['elasticsearch.search_template_delete'](id=name) if ret['result']: ret['comment'] = 'Successfully removed search template {0}'.format(name) - ret['changes']['old'] = json.loads(template["template"]) + ret['changes']['old'] = salt.utils.json.loads(template["template"]) else: ret['comment'] = 'Failed to remove search template {0} for unknown reasons'.format(name) else: @@ -452,7 +453,7 @@ def search_template_present(name, definition): old = {} if template: - old = json.loads(template["template"]) + old = salt.utils.json.loads(template["template"]) ret['changes'] = __utils__['dictdiffer.deep_diff'](old, definition) diff --git a/salt/states/glassfish.py b/salt/states/glassfish.py index f413048e023..651ef5e1947 100644 --- a/salt/states/glassfish.py +++ b/salt/states/glassfish.py @@ -17,7 +17,7 @@ You can setup connection parameters like this from __future__ import absolute_import try: - import json + import salt.utils.json from salt.ext import six from salt.exceptions import CommandExecutionError import requests @@ -56,8 +56,8 @@ def _is_updated(old_conf, new_conf): changed = {} # Dirty json hacking to get parameters in the same format - new_conf = _json_to_unicode(json.loads(json.dumps(new_conf, ensure_ascii=False))) - old_conf = json.loads(json.dumps(old_conf, ensure_ascii=False)) + new_conf = _json_to_unicode(salt.utils.json.loads(salt.utils.json.dumps(new_conf, ensure_ascii=False))) + old_conf = salt.utils.json.loads(salt.utils.json.dumps(old_conf, ensure_ascii=False)) for key, value in old_conf.items(): oldval = str(value).lower() diff --git a/salt/states/grafana.py b/salt/states/grafana.py index a8818e9ab8e..ad4f9a93805 100644 --- a/salt/states/grafana.py +++ b/salt/states/grafana.py @@ -166,13 +166,18 @@ add rows if they do not exist in existing dashboards, and to update rows if they exist in dashboards. The module will not manage rows that are not defined, allowing users to manage their own custom rows. ''' +# Import Python libs from __future__ import absolute_import -from salt.exceptions import SaltInvocationError -from salt.ext.six import string_types -from salt.utils.dictdiffer import DictDiffer -import json import copy +# Import Salt libs +import salt.utils.json +from salt.exceptions import SaltInvocationError +from salt.utils.dictdiffer import DictDiffer + +# Import 3rd-party +from salt.ext.six import string_types + def __virtual__(): ''' @@ -281,7 +286,7 @@ def dashboard_present( index=index, id=name, doc_type='dashboard', hosts=hosts ) _dashboard = _dashboard.get('_source', {}).get('dashboard') - _dashboard = json.loads(_dashboard) + _dashboard = salt.utils.json.loads(_dashboard) else: if not dashboard: raise SaltInvocationError('Grafana dashboard does not exist and no' @@ -341,7 +346,7 @@ def dashboard_present( "user": "guest", "group": "guest", "title": name, - "dashboard": json.dumps(_dashboard) + "dashboard": salt.utils.json.dumps(_dashboard) } updated = __salt__['elasticsearch.index']( index=index, doc_type='dashboard', body=body, id=name, diff --git a/salt/states/grafana4_dashboard.py b/salt/states/grafana4_dashboard.py index 548e3e17afa..7296ed7bea5 100644 --- a/salt/states/grafana4_dashboard.py +++ b/salt/states/grafana4_dashboard.py @@ -41,9 +41,9 @@ allowing users to manage their own custom rows. # Import Python libs from __future__ import absolute_import import copy -import json # Import Salt libs +import salt.utils.json from salt.ext import six from salt.utils.dictdiffer import DictDiffer @@ -160,16 +160,18 @@ def present(name, if updated_needed: if __opts__['test']: ret['result'] = None - ret['comment'] = ('Dashboard {0} is set to be updated, ' - 'changes={1}').format( - name, - json.dumps( - _dashboard_diff( - _cleaned(new_dashboard), - _cleaned(old_dashboard) - ), - indent=4 - )) + ret['comment'] = ( + str('Dashboard {0} is set to be updated, changes={1}').format( # future lint: blacklisted-function + name, + salt.utils.json.dumps( + _dashboard_diff( + _cleaned(new_dashboard), + _cleaned(old_dashboard) + ), + indent=4 + ) + ) + ) return ret response = __salt__['grafana4.create_update_dashboard']( diff --git a/salt/states/grafana_dashboard.py b/salt/states/grafana_dashboard.py index 6e1ef15aab9..70d3e45d2c6 100644 --- a/salt/states/grafana_dashboard.py +++ b/salt/states/grafana_dashboard.py @@ -41,10 +41,10 @@ allowing users to manage their own custom rows. # Import Python libs from __future__ import absolute_import import copy -import json import requests # Import Salt libs +import salt.utils.json from salt.ext import six from salt.utils.dictdiffer import DictDiffer @@ -158,16 +158,18 @@ def present(name, if updated_needed: if __opts__['test']: ret['result'] = None - ret['comment'] = ('Dashboard {0} is set to be updated, ' - 'changes={1}').format( - name, - json.dumps( - _dashboard_diff( - _cleaned(new_dashboard), - _cleaned(old_dashboard) - ), - indent=4 - )) + ret['comment'] = ( + str('Dashboard {0} is set to be updated, changes={1}').format( # future lint: blacklisted-function + name, + salt.utils.json.dumps( + _dashboard_diff( + _cleaned(new_dashboard), + _cleaned(old_dashboard) + ), + indent=4 + ) + ) + ) return ret response = _update(new_dashboard, profile) diff --git a/salt/states/heat.py b/salt/states/heat.py index b6ee6b737a5..0bc32d6a2f4 100644 --- a/salt/states/heat.py +++ b/salt/states/heat.py @@ -34,16 +34,19 @@ mysql: - rollback: True ''' +# Import Python libs from __future__ import absolute_import -import json import logging - -# Import third party libs -from salt.ext import six -import salt.utils.files -import salt.exceptions import yaml -# Import python libs + +# Import Salt libs +import salt.utils.files +import salt.utils.json +import salt.exceptions + +# Import 3rd-party libs +from salt.ext import six + # pylint: disable=import-error HAS_OSLO = False try: @@ -102,7 +105,7 @@ def _parse_template(tmpl_str): ''' tmpl_str = tmpl_str.strip() if tmpl_str.startswith('{'): - tpl = json.loads(tmpl_str) + tpl = salt.utils.json.loads(tmpl_str) else: try: tpl = yaml.load(tmpl_str, Loader=YamlLoader) diff --git a/salt/states/netyang.py b/salt/states/netyang.py index e2d673d2414..5295062743d 100644 --- a/salt/states/netyang.py +++ b/salt/states/netyang.py @@ -22,9 +22,8 @@ Please check Installation_ for complete details. .. _Installation: https://napalm.readthedocs.io/en/latest/installation.html ''' from __future__ import absolute_import - -import json import logging + log = logging.getLogger(__file__) # Import third party libs @@ -39,6 +38,7 @@ except ImportError: # Import salt modules import salt.utils.files +import salt.utils.json import salt.utils.napalm # ------------------------------------------------------------------------------ @@ -157,7 +157,7 @@ def managed(name, data = {'to_dict': data} data = [data] with salt.utils.files.fopen(temp_file, 'w') as file_handle: - yaml.safe_dump(json.loads(json.dumps(data)), file_handle, encoding='utf-8', allow_unicode=True) + yaml.safe_dump(salt.utils.json.loads(salt.utils.json.dumps(data)), file_handle, encoding='utf-8', allow_unicode=True) device_config = __salt__['napalm_yang.parse'](*models, config=True, profiles=profiles) diff --git a/salt/states/serverdensity_device.py b/salt/states/serverdensity_device.py index a2427e2de8d..909c24191a2 100644 --- a/salt/states/serverdensity_device.py +++ b/salt/states/serverdensity_device.py @@ -49,12 +49,13 @@ Example: # Import python libs from __future__ import absolute_import -import json import logging +# Import Salt libs +import salt.utils.json + # Import 3rd-party libs from salt.ext import six -import json # TODO: # @@ -82,12 +83,12 @@ def _get_salt_params(): sd_os = {'code': 'mac', 'name': 'Mac'} else: sd_os = {'code': all_grains['kernel'].lower(), 'name': all_grains['kernel']} - params['os'] = json.dumps(sd_os) + params['os'] = salt.utils.json.dumps(sd_os) params['cpuCores'] = all_stats['cpuinfo']['cpu cores'] params['installedRAM'] = str(int(all_stats['meminfo']['MemTotal']['value']) / 1024) params['swapSpace'] = str(int(all_stats['meminfo']['SwapTotal']['value']) / 1024) - params['privateIPs'] = json.dumps(all_grains['fqdn_ip4']) - params['privateDNS'] = json.dumps(all_grains['fqdn']) + params['privateIPs'] = salt.utils.json.dumps(all_grains['fqdn_ip4']) + params['privateDNS'] = salt.utils.json.dumps(all_grains['fqdn']) except KeyError: pass diff --git a/salt/states/solrcloud.py b/salt/states/solrcloud.py index 6fc969af279..5a2800b2b63 100644 --- a/salt/states/solrcloud.py +++ b/salt/states/solrcloud.py @@ -8,7 +8,9 @@ States for solrcloud alias and collection configuration # Import Python libs from __future__ import absolute_import -import json + +# Import Salt libs +import salt.utils.json # Import 3rd party libs from salt.ext import six @@ -136,8 +138,8 @@ def collection(name, options=None, **kwargs): if __opts__['test']: ret['comment'] = 'Collection options "{0}" will be changed.'.format(name) ret['pchanges'] = { - 'old': json.dumps(current_options, sort_keys=True, indent=4, separators=(',', ': ')), - 'new': json.dumps(options, sort_keys=True, indent=4, separators=(',', ': ')) + 'old': salt.utils.json.dumps(current_options, sort_keys=True, indent=4, separators=(',', ': ')), + 'new': salt.utils.json.dumps(options, sort_keys=True, indent=4, separators=(',', ': ')) } ret['result'] = None @@ -148,18 +150,19 @@ def collection(name, options=None, **kwargs): ret['comment'] = 'Parameters were updated for collection "{0}".'.format(name) ret['result'] = True ret['changes'] = { - 'old': json.dumps(current_options, sort_keys=True, indent=4, separators=(',', ': ')), - 'new': json.dumps(options, sort_keys=True, indent=4, separators=(',', ': ')) + 'old': salt.utils.json.dumps(current_options, sort_keys=True, indent=4, separators=(',', ': ')), + 'new': salt.utils.json.dumps(options, sort_keys=True, indent=4, separators=(',', ': ')) } return ret else: + new_changes = salt.utils.json.dumps(options, sort_keys=True, indent=4, separators=(',', ': ')) if __opts__['test']: ret['comment'] = 'The collection "{0}" will be created.'.format(name) ret['pchanges'] = { 'old': None, - 'new': 'options='+json.dumps(options, sort_keys=True, indent=4, separators=(',', ': ')) + 'new': str('options=') + new_changes # future lint: disable=blacklisted-function } ret['result'] = None else: @@ -167,7 +170,7 @@ def collection(name, options=None, **kwargs): ret['comment'] = 'The collection "{0}" has been created.'.format(name) ret['changes'] = { 'old': None, - 'new': 'options='+json.dumps(options, sort_keys=True, indent=4, separators=(',', ': ')) + 'new': str('options=') + new_changes # future lint: disable=blacklisted-function } ret['result'] = True diff --git a/salt/states/win_lgpo.py b/salt/states/win_lgpo.py index fcedaae0106..780b0ce80f2 100644 --- a/salt/states/win_lgpo.py +++ b/salt/states/win_lgpo.py @@ -106,10 +106,10 @@ Multiple policy configuration # Import python libs from __future__ import absolute_import import logging -import json # Import salt libs import salt.utils.dictdiffer +import salt.utils.json # Import 3rd party libs from salt.ext import six @@ -265,8 +265,9 @@ def set_(name, # compare log.debug('need to compare {0} from current/requested policy'.format(policy_name)) changes = False - if json.dumps(policy_data['requested_policy'][policy_name], sort_keys=True).lower() != \ - json.dumps(current_policy[policy_data['output_section']][pol_id], sort_keys=True).lower(): + requested_policy_json = salt.utils.json.dumps(policy_data['requested_policy'][policy_name], sort_keys=True).lower() + current_policy_json = salt.utils.json.dumps(current_policy[policy_data['output_section']][pol_id], sort_keys=True).lower() + if requested_policy_json != current_policy_json: if policy_data['policy_lookup'][policy_name]['rights_assignment'] and cumulative_rights_assignments: for user in policy_data['requested_policy'][policy_name]: if user not in current_policy[policy_data['output_section']][pol_id]: @@ -275,14 +276,14 @@ def set_(name, changes = True if changes: log.debug('{0} current policy != requested policy'.format(policy_name)) - log.debug('we compared {0} to {1}'.format( - json.dumps(policy_data['requested_policy'][policy_name], sort_keys=True).lower(), - json.dumps(current_policy[policy_data['output_section']][pol_id], sort_keys=True).lower())) + log.debug( + 'we compared %s to %s', + requested_policy_json, current_policy_json + ) policy_changes.append(policy_name) else: log.debug('{0} current setting matches the requested setting'.format(policy_name)) - ret['comment'] = ' '.join(['"{0}" is already set.'.format(policy_name), - ret['comment']]) + ret['comment'] = '"{0}" is already set.'.format(policy_name) + ret['comment'] else: policy_changes.append(policy_name) log.debug('policy {0} is not set, we will configure it'.format(policy_name)) diff --git a/salt/template.py b/salt/template.py index 579674b60e0..97cb5939e09 100644 --- a/salt/template.py +++ b/salt/template.py @@ -167,7 +167,7 @@ def template_shebang(template, renderers, default, blacklist, whitelist, input_d line = input_data.split()[0] else: with salt.utils.files.fopen(template, 'r') as ifile: - line = ifile.readline() + line = salt.utils.stringutils.to_unicode(ifile.readline()) # Check if it starts with a shebang and not a path if line.startswith('#!') and not line.startswith('#!/'): diff --git a/salt/thorium/file.py b/salt/thorium/file.py index 6520ade46e8..a114daa081e 100644 --- a/salt/thorium/file.py +++ b/salt/thorium/file.py @@ -41,11 +41,11 @@ that can be re-imported into Python. # import python libs from __future__ import absolute_import, print_function, unicode_literals import os -import json # Import salt libs import salt.utils.data import salt.utils.files +import salt.utils.json def save(name, filter=False): @@ -80,9 +80,7 @@ def save(name, filter=False): os.makedirs(tgt_dir) with salt.utils.files.fopen(fn_, 'w+') as fp_: if filter is True: - fp_.write(json.dumps( - salt.utils.data.simple_types_filter(__reg__)) - ) + salt.utils.json.dump(salt.utils.data.simple_types_filter(__reg__), fp_) else: - fp_.write(json.dumps(__reg__)) + salt.utils.json.dump(__reg__, fp_) return ret diff --git a/salt/utils/data.py b/salt/utils/data.py index 94ed47978c7..84ed7ec4508 100644 --- a/salt/utils/data.py +++ b/salt/utils/data.py @@ -67,78 +67,145 @@ def compare_lists(old=None, new=None): return ret -def decode_dict(data): +def decode(data, preserve_dict_class=False, preserve_tuples=False): ''' - Decode all values to Unicode + Generic function which will decode whichever type is passed, if necessary ''' - rv = {} + if isinstance(data, six.string_types): + if six.PY2 and isinstance(data, str): + return data.decode('utf-8') + else: + return data + elif isinstance(data, collections.Mapping): + return decode_dict(data, preserve_dict_class, preserve_tuples) + elif isinstance(data, list): + return decode_list(data, preserve_dict_class, preserve_tuples) + elif isinstance(data, tuple): + return decode_tuple(data, preserve_dict_class) if preserve_tuples \ + else decode_list(data, preserve_dict_class, preserve_tuples) + else: + return data + + +def decode_dict(data, preserve_dict_class=False, preserve_tuples=False): + ''' + Decode all string values to Unicode + ''' + # Make sure we preserve OrderedDicts + rv = data.__class__() if preserve_dict_class else {} for key, value in six.iteritems(data): if six.PY2 and isinstance(key, str): - key = key.decode(__salt_system_encoding__) + key = key.decode('utf-8') if six.PY2 and isinstance(value, str): - value = value.decode(__salt_system_encoding__) + value = value.decode('utf-8') elif isinstance(value, list): - value = decode_list(value) - elif isinstance(value, dict): - value = decode_dict(value) + value = decode_list(value, preserve_dict_class, preserve_tuples) + elif isinstance(value, tuple): + value = decode_tuple(value, preserve_dict_class) if preserve_tuples \ + else decode_list(value, preserve_dict_class, preserve_tuples) + elif isinstance(value, collections.Mapping): + value = decode_dict(value, preserve_dict_class, preserve_tuples) rv[key] = value return rv -def decode_list(data): +def decode_list(data, preserve_dict_class=False, preserve_tuples=False): ''' - Decode all values to Unicode + Decode all string values to Unicode ''' rv = [] for item in data: - if six.PY2 and isinstance(item, six.text_type): - item = item.decode(__salt_system_encoding__) + if six.PY2 and isinstance(item, str): + item = item.decode('utf-8') elif isinstance(item, list): - item = decode_list(item) - elif isinstance(item, dict): - item = decode_dict(item) + item = decode_list(item, preserve_dict_class, preserve_tuples) + elif isinstance(item, tuple): + item = decode_tuple(item, preserve_dict_class) if preserve_tuples \ + else decode_list(item, preserve_dict_class, preserve_tuples) + elif isinstance(item, collections.Mapping): + item = decode_dict(item, preserve_dict_class, preserve_tuples) rv.append(item) return rv +def decode_tuple(data, preserve_dict_class=False): + ''' + Decode all string values to Unicode + ''' + return tuple(decode_list(data, preserve_dict_class, True)) + + +def encode(data, preserve_dict_class=False, preserve_tuples=False): + ''' + Generic function which will encode whichever type is passed, if necessary + ''' + if isinstance(data, six.string_types): + if six.PY2 and isinstance(data, six.text_type): + return data.encode('utf-8') + else: + return data + elif isinstance(data, collections.Mapping): + return encode_dict(data, preserve_dict_class, preserve_tuples) + elif isinstance(data, list): + return encode_list(data, preserve_dict_class, preserve_tuples) + elif isinstance(data, tuple): + return encode_tuple(data, preserve_dict_class) if preserve_tuples \ + else encode_list(data, preserve_dict_class, preserve_tuples) + else: + return data + + @jinja_filter('json_decode_dict') # Remove this for Neon @jinja_filter('json_encode_dict') -def encode_dict(data): +def encode_dict(data, preserve_dict_class=False, preserve_tuples=False): ''' - Encode all values to bytes + Encode all string values to bytes ''' - rv = {} + rv = data.__class__() if preserve_dict_class else {} for key, value in six.iteritems(data): if six.PY2 and isinstance(key, six.text_type): - key = key.encode(__salt_system_encoding__) + key = key.encode('utf-8') if six.PY2 and isinstance(value, six.text_type): - value = value.encode(__salt_system_encoding__) + value = value.encode('utf-8') elif isinstance(value, list): - value = encode_list(value) - elif isinstance(value, dict): - value = encode_dict(value) + value = encode_list(value, preserve_dict_class, preserve_tuples) + elif isinstance(value, tuple): + value = encode_tuple(value, preserve_dict_class) if preserve_tuples \ + else encode_list(value, preserve_dict_class, preserve_tuples) + elif isinstance(value, collections.Mapping): + value = encode_dict(value, preserve_dict_class, preserve_tuples) rv[key] = value return rv @jinja_filter('json_decode_list') # Remove this for Neon @jinja_filter('json_encode_list') -def encode_list(data): +def encode_list(data, preserve_dict_class=False, preserve_tuples=False): ''' - Encode all values to bytes + Encode all string values to bytes ''' rv = [] for item in data: - if isinstance(item, six.text_type) and six.PY2: - item = item.encode(__salt_system_encoding__) + if six.PY2 and isinstance(item, six.text_type): + item = item.encode('utf-8') elif isinstance(item, list): - item = encode_list(item) - elif isinstance(item, dict): - item = encode_dict(item) + item = encode_list(item, preserve_dict_class, preserve_tuples) + elif isinstance(item, tuple): + item = encode_tuple(item, preserve_dict_class) if preserve_tuples \ + else encode_list(item, preserve_dict_class, preserve_tuples) + elif isinstance(item, collections.Mapping): + item = encode_dict(item, preserve_dict_class, preserve_tuples) rv.append(item) return rv +def encode_tuple(data, preserve_dict_class=False): + ''' + Encode all string values to Unicode + ''' + return tuple(encode_list(data, preserve_dict_class, True)) + + @jinja_filter('exactly_n_true') def exactly_n(l, n=1): ''' diff --git a/salt/utils/github.py b/salt/utils/github.py index 6db18c72e21..f6b44bc2084 100644 --- a/salt/utils/github.py +++ b/salt/utils/github.py @@ -5,14 +5,15 @@ Connection library for GitHub # Import Python libs from __future__ import absolute_import -import json -import salt.utils.http import logging +# Import Salt libs +import salt.utils.json +import salt.utils.http + # Import 3rd-party libs from salt.ext import six - log = logging.getLogger(__name__) @@ -57,7 +58,7 @@ def get_user_pubkeys(users): text=True, ) - keys = json.loads(result['text']) + keys = salt.utils.json.loads(result['text']) ret[user] = {} for key in keys: diff --git a/salt/utils/http.py b/salt/utils/http.py index 64f8565f932..e0b12cc85a9 100644 --- a/salt/utils/http.py +++ b/salt/utils/http.py @@ -9,9 +9,8 @@ and the like, but also useful for basic HTTP testing. # Import python libs from __future__ import absolute_import import cgi -import json import logging -import os.path +import os import pprint import socket import yaml @@ -44,6 +43,7 @@ import salt.loader import salt.syspaths import salt.utils.args import salt.utils.files +import salt.utils.json import salt.utils.network import salt.utils.platform import salt.utils.stringutils @@ -648,7 +648,7 @@ def query(url, return ret if decode_type == 'json': - ret['dict'] = json.loads(salt.utils.stringutils.to_str(result_text)) + ret['dict'] = salt.utils.json.loads(result_text) elif decode_type == 'xml': ret['dict'] = [] items = ET.fromstring(result_text) diff --git a/salt/utils/jinja.py b/salt/utils/jinja.py index 649f63a0c6a..f1fcb10df52 100644 --- a/salt/utils/jinja.py +++ b/salt/utils/jinja.py @@ -6,7 +6,6 @@ Jinja loading utils to enable a more powerful backend for jinja templates # Import python libs from __future__ import absolute_import import collections -import json import logging import os.path import pipes @@ -31,6 +30,7 @@ from salt.exceptions import TemplateError import salt.fileclient import salt.utils.data import salt.utils.files +import salt.utils.json import salt.utils.url import salt.utils.yamldumper from salt.utils.decorators.jinja import jinja_filter, jinja_test, jinja_global @@ -782,7 +782,7 @@ class SerializerExtension(Extension, object): return explore(data) def format_json(self, value, sort_keys=True, indent=None): - return Markup(json.dumps(value, sort_keys=sort_keys, indent=indent).strip()) + return Markup(salt.utils.json.dumps(value, sort_keys=sort_keys, indent=indent).strip()) def format_yaml(self, value, flow_style=True): yaml_txt = salt.utils.yamldumper.safe_dump( @@ -853,7 +853,7 @@ class SerializerExtension(Extension, object): if isinstance(value, TemplateModule): value = str(value) try: - return json.loads(value) + return salt.utils.json.loads(value) except (ValueError, TypeError, AttributeError): raise TemplateRuntimeError( 'Unable to load json from {0}'.format(value)) diff --git a/salt/utils/json.py b/salt/utils/json.py index e9f5e0969a7..15e76a7ec31 100644 --- a/salt/utils/json.py +++ b/salt/utils/json.py @@ -6,11 +6,15 @@ Functions to work with JSON from __future__ import absolute_import # Import Python libs -import json +import json # future lint: blacklisted-module import logging # Import Salt libs import salt.utils.data +import salt.utils.stringutils + +# Import 3rd-party libs +from salt.ext import six log = logging.getLogger(__name__) @@ -24,7 +28,7 @@ def find_json(raw): for ind, _ in enumerate(raw): working = '\n'.join(raw.splitlines()[ind:]) try: - ret = json.loads(working, object_hook=salt.utils.data.decode_dict) + ret = json.loads(working, object_hook=salt.utils.data.decode_dict) # future lint: blacklisted-function except ValueError: continue if ret: @@ -45,3 +49,77 @@ def import_json(): return mod except ImportError: continue + + +def load(fp, **kwargs): + ''' + .. versionadded:: Oxygen + + Wraps json.load + + You can pass an alternate json module (loaded via import_json() above) + using the _json_module argument) + ''' + return kwargs.pop('_json_module', json).load(fp, **kwargs) + + +def loads(s, **kwargs): + ''' + .. versionadded:: Oxygen + + Wraps json.loads and prevents a traceback in the event that a bytestring is + passed to the function. (Python < 3.6 cannot load bytestrings) + + You can pass an alternate json module (loaded via import_json() above) + using the _json_module argument) + ''' + json_module = kwargs.pop('_json_module', json) + try: + return json_module.loads(s, **kwargs) + except TypeError as exc: + # json.loads cannot load bytestrings in Python < 3.6 + if six.PY3 and isinstance(s, bytes): + return json_module.loads(s.decode(__salt_system_encoding__), **kwargs) + else: + raise exc + + +def dump(obj, fp, **kwargs): + ''' + .. versionadded:: Oxygen + + Wraps json.dump and encodes the result to the system encoding. Also assumes + that ensure_ascii is False (unless explicitly passed as True) for unicode + compatibility. Note that setting it to True will mess up any unicode + characters, as they will be dumped as the string literal version of the + unicode code point. + + You can pass an alternate json module (loaded via import_json() above) + using the _json_module argument) + ''' + json_module = kwargs.pop('_json_module', json) + if 'ensure_ascii' not in kwargs: + kwargs['ensure_ascii'] = False + obj = salt.utils.data.encode(obj) + return json.dump(obj, fp, **kwargs) # future lint: blacklisted-function + + +def dumps(obj, **kwargs): + ''' + .. versionadded:: Oxygen + + Wraps json.dumps and encodes the result to the system encoding. Also + assumes that ensure_ascii is False (unless explicitly passed as True) for + unicode compatibility. Note that setting it to True will mess up any + unicode characters, as they will be dumped as the string literal version of + the unicode code point. + + You can pass an alternate json module (loaded via import_json() above) + using the _json_module argument) + ''' + import sys + json_module = kwargs.pop('_json_module', json) + if 'ensure_ascii' not in kwargs: + kwargs['ensure_ascii'] = False + obj = salt.utils.data.encode(obj) + return json_module.dumps(obj, **kwargs) # future lint: blacklisted-function diff --git a/salt/utils/pagerduty.py b/salt/utils/pagerduty.py index 7b8fa33f621..cb40f4e0ff0 100644 --- a/salt/utils/pagerduty.py +++ b/salt/utils/pagerduty.py @@ -18,9 +18,9 @@ Library for interacting with PagerDuty API ''' from __future__ import absolute_import -import json import logging import salt.utils.http +import salt.utils.json from salt.version import __version__ log = logging.getLogger(__name__) @@ -99,7 +99,7 @@ def query(method='GET', profile_dict=None, url=None, path='api/v1', method, params=params, header_dict=headers, - data=json.dumps(data), + data=salt.utils.json.dumps(data), decode=False, text=True, opts=opts, @@ -113,7 +113,7 @@ def list_items(action, key, profile_dict=None, api_key=None, opts=None): List items belonging to an API call. Used for list_services() and list_incidents() ''' - items = json.loads(query( + items = salt.utils.json.loads(query( profile_dict=profile_dict, api_key=api_key, action=action, diff --git a/salt/utils/pkg/win.py b/salt/utils/pkg/win.py index 89ad3c048bc..ff2090c51a9 100644 --- a/salt/utils/pkg/win.py +++ b/salt/utils/pkg/win.py @@ -1300,7 +1300,7 @@ def __main(): version_only = True if str(sys.argv[2]) == 'system+user': user_pkgs = True - import json + import salt.utils.json import timeit def run(): @@ -1308,7 +1308,7 @@ def __main(): Main run code, when this module is run directly ''' pkg_list = WinSoftware(user_pkgs=user_pkgs, version_only=version_only) - print(json.dumps(pkg_list.data, sort_keys=True, indent=4)) # pylint: disable=superfluous-parens + print(salt.utils.json.dumps(pkg_list.data, sort_keys=True, indent=4)) # pylint: disable=superfluous-parens print('Total: {}'.format(len(pkg_list))) # pylint: disable=superfluous-parens print('Time Taken: {}'.format(timeit.timeit(run, number=1))) # pylint: disable=superfluous-parens diff --git a/salt/utils/schema.py b/salt/utils/schema.py index 7ce3371ee88..2d8984cd458 100644 --- a/salt/utils/schema.py +++ b/salt/utils/schema.py @@ -77,7 +77,7 @@ ('x-ordering', ['host', 'port']), ('additionalProperties', True)] ) - >>> print(json.dumps(HostConfig.serialize(), indent=2)) + >>> print(salt.utils.json.dumps(HostConfig.serialize(), indent=2)) { "$schema": "http://json-schema.org/draft-04/schema#", "title": "Host Configuration", @@ -170,7 +170,7 @@ .. code-block:: python - >>> print json.dumps(MyConfig.serialize(), indent=4) + >>> print salt.utils.json.dumps(MyConfig.serialize(), indent=4) { "$schema": "http://json-schema.org/draft-04/schema#", "title": "My Config", @@ -282,7 +282,7 @@ .. code-block:: python - >>> print(json.dumps(MyConfig, indent=4)) + >>> print(salt.utils.json.dumps(MyConfig, indent=4)) { "$schema": "http://json-schema.org/draft-04/schema#", "title": "My Config", diff --git a/salt/utils/ssdp.py b/salt/utils/ssdp.py index e748d4d84c7..d7b547370dd 100644 --- a/salt/utils/ssdp.py +++ b/salt/utils/ssdp.py @@ -27,10 +27,11 @@ import random import socket from collections import OrderedDict -from salt.utils import json -json = json.import_json() -if not hasattr(json, 'dumps'): - json = None +import salt.utils.json + +_json = salt.utils.json.import_json() +if not hasattr(_json, 'dumps'): + _json = None try: import asyncio @@ -80,7 +81,7 @@ class SSDPBase(object): Return True if the USSDP dependencies are satisfied. :return: ''' - return bool(asyncio and json) + return bool(asyncio and _json) @staticmethod def get_self_ip(): @@ -186,7 +187,13 @@ class SSDPFactory(SSDPBase): return self.log.debug('Received "{}" from {}'.format(message, "{}:{}".format(*addr))) - self._sendto('{0}:@:{1}'.format(self.signature, json.dumps(self.answer)), addr) + self._sendto( + str('{0}:@:{1}').format( # future lint: disable=blacklisted-function + self.signature, + salt.utils.json.dumps(self.answer, _json_module=_json) + ), + addr + ) else: if self.disable_hidden: self._sendto('{0}:E:{1}'.format(self.signature, 'Invalid packet signature').encode(), addr) @@ -393,5 +400,7 @@ class SSDPDiscoveryClient(SSDPBase): else: if addr not in masters: masters[addr] = [] - masters[addr].append(json.loads(msg.split(':@:')[-1])) + masters[addr].append( + salt.utils.json.loads(msg.split(':@:')[-1], _json_module=_json) + ) return masters diff --git a/salt/utils/thin.py b/salt/utils/thin.py index 9f904e505cd..aa8176d7f5a 100644 --- a/salt/utils/thin.py +++ b/salt/utils/thin.py @@ -8,7 +8,6 @@ from __future__ import absolute_import import os import sys -import json import shutil import tarfile import zipfile @@ -72,6 +71,7 @@ except ImportError: import salt import salt.utils.files import salt.utils.hashutils +import salt.utils.json import salt.utils.path import salt.exceptions import salt.version @@ -241,14 +241,14 @@ def gen_thin(cachedir, extra_mods='', overwrite=False, so_mods='', # Get python 3 tops py_shell_cmd = ( python3_bin + ' -c \'import sys; import json; import salt.utils.thin; ' - 'print(json.dumps(salt.utils.thin.get_tops(**(json.loads(sys.argv[1]))))); exit(0);\' ' - '\'{0}\''.format(json.dumps({'extra_mods': extra_mods, 'so_mods': so_mods})) + 'print(json.dumps(salt.utils.thin.get_tops(**(json.loads(sys.argv[1]))), ensure_ascii=False)); exit(0);\' ' + '\'{0}\''.format(salt.utils.json.dumps({'extra_mods': extra_mods, 'so_mods': so_mods})) ) cmd = subprocess.Popen(py_shell_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) stdout, stderr = cmd.communicate() if cmd.returncode == 0: try: - tops = json.loads(stdout) + tops = salt.utils.json.loads(stdout) tops_py_version_mapping['3'] = tops except ValueError: pass @@ -257,14 +257,14 @@ def gen_thin(cachedir, extra_mods='', overwrite=False, so_mods='', py_shell_cmd = ( python2_bin + ' -c \'from __future__ import print_function; ' 'import sys; import json; import salt.utils.thin; ' - 'print(json.dumps(salt.utils.thin.get_tops(**(json.loads(sys.argv[1]))))); exit(0);\' ' - '\'{0}\''.format(json.dumps({'extra_mods': extra_mods, 'so_mods': so_mods})) + 'print(json.dumps(salt.utils.thin.get_tops(**(json.loads(sys.argv[1]))), ensure_ascii=False)); exit(0);\' ' + '\'{0}\''.format(salt.utils.json.dumps({'extra_mods': extra_mods, 'so_mods': so_mods})) ) cmd = subprocess.Popen(py_shell_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) stdout, stderr = cmd.communicate() if cmd.returncode == 0: try: - tops = json.loads(stdout.decode('utf-8')) + tops = salt.utils.json.loads(stdout.decode('utf-8')) tops_py_version_mapping['2'] = tops except ValueError: pass @@ -426,14 +426,14 @@ def gen_min(cachedir, extra_mods='', overwrite=False, so_mods='', # Get python 3 tops py_shell_cmd = ( python3_bin + ' -c \'import sys; import json; import salt.utils.thin; ' - 'print(json.dumps(salt.utils.thin.get_tops(**(json.loads(sys.argv[1]))))); exit(0);\' ' - '\'{0}\''.format(json.dumps({'extra_mods': extra_mods, 'so_mods': so_mods})) + 'print(json.dumps(salt.utils.thin.get_tops(**(json.loads(sys.argv[1]))), ensure_ascii=False)); exit(0);\' ' + '\'{0}\''.format(salt.utils.json.dumps({'extra_mods': extra_mods, 'so_mods': so_mods})) ) cmd = subprocess.Popen(py_shell_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) stdout, stderr = cmd.communicate() if cmd.returncode == 0: try: - tops = json.loads(stdout) + tops = salt.utils.json.loads(stdout) tops_py_version_mapping['3'] = tops except ValueError: pass @@ -442,14 +442,14 @@ def gen_min(cachedir, extra_mods='', overwrite=False, so_mods='', py_shell_cmd = ( python2_bin + ' -c \'from __future__ import print_function; ' 'import sys; import json; import salt.utils.thin; ' - 'print(json.dumps(salt.utils.thin.get_tops(**(json.loads(sys.argv[1]))))); exit(0);\' ' - '\'{0}\''.format(json.dumps({'extra_mods': extra_mods, 'so_mods': so_mods})) + 'print(json.dumps(salt.utils.thin.get_tops(**(json.loads(sys.argv[1]))), ensure_ascii=False)); exit(0);\' ' + '\'{0}\''.format(salt.utils.json.dumps({'extra_mods': extra_mods, 'so_mods': so_mods})) ) cmd = subprocess.Popen(py_shell_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) stdout, stderr = cmd.communicate() if cmd.returncode == 0: try: - tops = json.loads(stdout.decode('utf-8')) + tops = salt.utils.json.loads(stdout.decode('utf-8')) tops_py_version_mapping['2'] = tops except ValueError: pass diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index b36614ccd6b..9c9ccb5714b 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -10,7 +10,6 @@ import os import re import sys import copy -import json import time import stat import errno diff --git a/tests/integration/cloud/helpers/virtualbox.py b/tests/integration/cloud/helpers/virtualbox.py index c7eb9897ce1..6979f852f32 100644 --- a/tests/integration/cloud/helpers/virtualbox.py +++ b/tests/integration/cloud/helpers/virtualbox.py @@ -2,7 +2,6 @@ # Import Python libs from __future__ import absolute_import -import json import logging import os @@ -14,6 +13,7 @@ from tests.support.paths import FILES # Import Salt libs from salt.ext import six +import salt.utils.json import salt.utils.virtualbox # Create the cloud instance name to be used throughout the tests @@ -77,7 +77,7 @@ class VirtualboxCloudTestCase(ShellCase): if len(output) is 0: return dict() else: - return json.loads("".join(output)) + return salt.utils.json.loads(''.join(output)) def run_cloud_function(self, function, kw_function_args=None, **kwargs): """ diff --git a/tests/integration/netapi/rest_cherrypy/test_app.py b/tests/integration/netapi/rest_cherrypy/test_app.py index 8eb6529f8b8..000b7418bf0 100644 --- a/tests/integration/netapi/rest_cherrypy/test_app.py +++ b/tests/integration/netapi/rest_cherrypy/test_app.py @@ -2,9 +2,9 @@ # Import python libs from __future__ import absolute_import -import json # Import salt libs +import salt.utils.json import salt.utils.stringutils # Import test support libs @@ -183,7 +183,7 @@ class TestArgKwarg(cptc.BaseRestCherryPyTest): are supported by runners. ''' cmd = dict(self.low) - body = json.dumps(cmd) + body = salt.utils.json.dumps(cmd) request, response = self.request( '/', @@ -195,7 +195,7 @@ class TestArgKwarg(cptc.BaseRestCherryPyTest): 'Accept': 'application/json', } ) - resp = json.loads(salt.utils.stringutils.to_str(response.body[0])) + resp = salt.utils.json.loads(salt.utils.stringutils.to_str(response.body[0])) self.assertEqual(resp['return'][0]['args'], [1234]) self.assertEqual(resp['return'][0]['kwargs'], {'ext_source': 'redis'}) @@ -253,6 +253,6 @@ class TestJobs(cptc.BaseRestCherryPyTest): 'X-Auth-Token': self._token(), }) - resp = json.loads(salt.utils.stringutils.to_str(response.body[0])) + resp = salt.utils.json.loads(salt.utils.stringutils.to_str(response.body[0])) self.assertIn('test.ping', str(resp['return'])) self.assertEqual(response.status, '200 OK') diff --git a/tests/integration/netapi/rest_tornado/test_app.py b/tests/integration/netapi/rest_tornado/test_app.py index 6672508132e..605a971100c 100644 --- a/tests/integration/netapi/rest_tornado/test_app.py +++ b/tests/integration/netapi/rest_tornado/test_app.py @@ -2,11 +2,11 @@ # Import Python Libs from __future__ import absolute_import, print_function -import json import time import threading # Import Salt Libs +import salt.utils.json import salt.utils.stringutils from salt.netapi.rest_tornado import saltnado from salt.utils.versions import StrictVersion @@ -26,12 +26,6 @@ except ImportError: HAS_ZMQ_IOLOOP = False -def json_loads(data): - if six.PY3 and isinstance(data, bytes): - data = data.decode('utf-8') - return json.loads(data) - - class _SaltnadoIntegrationTestCase(SaltnadoTestCase): # pylint: disable=abstract-method @property @@ -63,7 +57,7 @@ class TestSaltAPIHandler(_SaltnadoIntegrationTestCase): request_timeout=30, ) self.assertEqual(response.code, 200) - response_obj = json_loads(response.body) + response_obj = salt.utils.json.loads(response.body) self.assertEqual(sorted(response_obj['clients']), ['local', 'local_async', 'runner', 'runner_async']) self.assertEqual(response_obj['return'], 'Welcome') @@ -79,7 +73,7 @@ class TestSaltAPIHandler(_SaltnadoIntegrationTestCase): }] response = self.fetch('/', method='POST', - body=json.dumps(low), + body=salt.utils.json.dumps(low), headers={'Content-Type': self.content_type_map['json']}, follow_redirects=False, connect_timeout=30, @@ -100,13 +94,13 @@ class TestSaltAPIHandler(_SaltnadoIntegrationTestCase): }] response = self.fetch('/', method='POST', - body=json.dumps(low), + body=salt.utils.json.dumps(low), headers={'Content-Type': self.content_type_map['json'], saltnado.AUTH_TOKEN_HEADER: self.token['token']}, connect_timeout=30, request_timeout=30, ) - response_obj = json_loads(response.body) + response_obj = salt.utils.json.loads(response.body) self.assertEqual(response_obj['return'], [{'minion': True, 'sub_minion': True}]) def test_simple_local_post_no_tgt(self): @@ -119,13 +113,13 @@ class TestSaltAPIHandler(_SaltnadoIntegrationTestCase): }] response = self.fetch('/', method='POST', - body=json.dumps(low), + body=salt.utils.json.dumps(low), headers={'Content-Type': self.content_type_map['json'], saltnado.AUTH_TOKEN_HEADER: self.token['token']}, connect_timeout=30, request_timeout=30, ) - response_obj = json_loads(response.body) + response_obj = salt.utils.json.loads(response.body) self.assertEqual(response_obj['return'], ["No minions matched the target. No command was sent, no jid was assigned."]) # local client request body test @@ -140,13 +134,13 @@ class TestSaltAPIHandler(_SaltnadoIntegrationTestCase): } response = self.fetch('/', method='POST', - body=json.dumps(low), + body=salt.utils.json.dumps(low), headers={'Content-Type': self.content_type_map['json'], saltnado.AUTH_TOKEN_HEADER: self.token['token']}, connect_timeout=30, request_timeout=30, ) - response_obj = json_loads(response.body) + response_obj = salt.utils.json.loads(response.body) self.assertEqual(response_obj['return'], [{'minion': True, 'sub_minion': True}]) def test_simple_local_post_invalid_request(self): @@ -156,7 +150,7 @@ class TestSaltAPIHandler(_SaltnadoIntegrationTestCase): low = ["invalid request"] response = self.fetch('/', method='POST', - body=json.dumps(low), + body=salt.utils.json.dumps(low), headers={'Content-Type': self.content_type_map['json'], saltnado.AUTH_TOKEN_HEADER: self.token['token']}, connect_timeout=30, @@ -172,12 +166,12 @@ class TestSaltAPIHandler(_SaltnadoIntegrationTestCase): }] response = self.fetch('/', method='POST', - body=json.dumps(low), + body=salt.utils.json.dumps(low), headers={'Content-Type': self.content_type_map['json'], saltnado.AUTH_TOKEN_HEADER: self.token['token']}, ) - response_obj = json_loads(response.body) + response_obj = salt.utils.json.loads(response.body) ret = response_obj['return'] ret[0]['minions'] = sorted(ret[0]['minions']) @@ -197,12 +191,12 @@ class TestSaltAPIHandler(_SaltnadoIntegrationTestCase): }] response = self.fetch('/', method='POST', - body=json.dumps(low), + body=salt.utils.json.dumps(low), headers={'Content-Type': self.content_type_map['json'], saltnado.AUTH_TOKEN_HEADER: self.token['token']}, ) - response_obj = json_loads(response.body) + response_obj = salt.utils.json.loads(response.body) ret = response_obj['return'] ret[0]['minions'] = sorted(ret[0]['minions']) ret[1]['minions'] = sorted(ret[1]['minions']) @@ -231,12 +225,12 @@ class TestSaltAPIHandler(_SaltnadoIntegrationTestCase): ] response = self.fetch('/', method='POST', - body=json.dumps(low), + body=salt.utils.json.dumps(low), headers={'Content-Type': self.content_type_map['json'], saltnado.AUTH_TOKEN_HEADER: self.token['token']}, ) - response_obj = json_loads(response.body) + response_obj = salt.utils.json.loads(response.body) ret = response_obj['return'] ret[0]['minions'] = sorted(ret[0]['minions']) ret[1]['minions'] = sorted(ret[1]['minions']) @@ -255,11 +249,11 @@ class TestSaltAPIHandler(_SaltnadoIntegrationTestCase): }] response = self.fetch('/', method='POST', - body=json.dumps(low), + body=salt.utils.json.dumps(low), headers={'Content-Type': self.content_type_map['json'], saltnado.AUTH_TOKEN_HEADER: self.token['token']}, ) - response_obj = json_loads(response.body) + response_obj = salt.utils.json.loads(response.body) self.assertEqual(response_obj['return'], [{}]) # runner tests @@ -269,13 +263,13 @@ class TestSaltAPIHandler(_SaltnadoIntegrationTestCase): }] response = self.fetch('/', method='POST', - body=json.dumps(low), + body=salt.utils.json.dumps(low), headers={'Content-Type': self.content_type_map['json'], saltnado.AUTH_TOKEN_HEADER: self.token['token']}, connect_timeout=30, request_timeout=30, ) - response_obj = json_loads(response.body) + response_obj = salt.utils.json.loads(response.body) self.assertEqual(len(response_obj['return']), 1) self.assertEqual(set(response_obj['return'][0]), set(['minion', 'sub_minion'])) @@ -286,13 +280,13 @@ class TestSaltAPIHandler(_SaltnadoIntegrationTestCase): }] response = self.fetch('/', method='POST', - body=json.dumps(low), + body=salt.utils.json.dumps(low), headers={'Content-Type': self.content_type_map['json'], saltnado.AUTH_TOKEN_HEADER: self.token['token']}, connect_timeout=10, request_timeout=10, ) - response_obj = json_loads(response.body) + response_obj = salt.utils.json.loads(response.body) self.assertIn('return', response_obj) self.assertEqual(1, len(response_obj['return'])) self.assertIn('jid', response_obj['return'][0]) @@ -317,7 +311,7 @@ class TestMinionSaltAPIHandler(_SaltnadoIntegrationTestCase): headers={saltnado.AUTH_TOKEN_HEADER: self.token['token']}, follow_redirects=False, ) - response_obj = json_loads(response.body) + response_obj = salt.utils.json.loads(response.body) self.assertEqual(len(response_obj['return']), 1) # one per minion self.assertEqual(len(response_obj['return'][0]), 2) @@ -332,7 +326,7 @@ class TestMinionSaltAPIHandler(_SaltnadoIntegrationTestCase): headers={saltnado.AUTH_TOKEN_HEADER: self.token['token']}, follow_redirects=False, ) - response_obj = json_loads(response.body) + response_obj = salt.utils.json.loads(response.body) self.assertEqual(len(response_obj['return']), 1) self.assertEqual(len(response_obj['return'][0]), 1) # check a single grain @@ -344,12 +338,12 @@ class TestMinionSaltAPIHandler(_SaltnadoIntegrationTestCase): }] response = self.fetch('/minions', method='POST', - body=json.dumps(low), + body=salt.utils.json.dumps(low), headers={'Content-Type': self.content_type_map['json'], saltnado.AUTH_TOKEN_HEADER: self.token['token']}, ) - response_obj = json_loads(response.body) + response_obj = salt.utils.json.loads(response.body) ret = response_obj['return'] ret[0]['minions'] = sorted(ret[0]['minions']) @@ -366,12 +360,12 @@ class TestMinionSaltAPIHandler(_SaltnadoIntegrationTestCase): }] response = self.fetch('/minions', method='POST', - body=json.dumps(low), + body=salt.utils.json.dumps(low), headers={'Content-Type': self.content_type_map['json'], saltnado.AUTH_TOKEN_HEADER: self.token['token']}, ) - response_obj = json_loads(response.body) + response_obj = salt.utils.json.loads(response.body) ret = response_obj['return'] ret[0]['minions'] = sorted(ret[0]['minions']) @@ -392,7 +386,7 @@ class TestMinionSaltAPIHandler(_SaltnadoIntegrationTestCase): }] response = self.fetch('/minions', method='POST', - body=json.dumps(low), + body=salt.utils.json.dumps(low), headers={'Content-Type': self.content_type_map['json'], saltnado.AUTH_TOKEN_HEADER: self.token['token']}, ) @@ -419,7 +413,7 @@ class TestJobsSaltAPIHandler(_SaltnadoIntegrationTestCase): follow_redirects=False, ) response = self.wait(timeout=30) - response_obj = json_loads(response.body)['return'][0] + response_obj = salt.utils.json.loads(response.body)['return'][0] try: for jid, ret in six.iteritems(response_obj): self.assertIn('Function', ret) @@ -429,7 +423,7 @@ class TestJobsSaltAPIHandler(_SaltnadoIntegrationTestCase): self.assertIn('StartTime', ret) self.assertIn('Arguments', ret) except AttributeError as attribute_error: - print(json_loads(response.body)) + print(salt.utils.json.loads(response.body)) raise # test with a specific JID passed in @@ -441,7 +435,7 @@ class TestJobsSaltAPIHandler(_SaltnadoIntegrationTestCase): follow_redirects=False, ) response = self.wait(timeout=30) - response_obj = json_loads(response.body)['return'][0] + response_obj = salt.utils.json.loads(response.body)['return'][0] self.assertIn('Function', response_obj) self.assertIn('Target', response_obj) self.assertIn('Target-type', response_obj) @@ -470,11 +464,11 @@ class TestRunSaltAPIHandler(_SaltnadoIntegrationTestCase): }] response = self.fetch('/run', method='POST', - body=json.dumps(low), + body=salt.utils.json.dumps(low), headers={'Content-Type': self.content_type_map['json'], saltnado.AUTH_TOKEN_HEADER: self.token['token']}, ) - response_obj = json_loads(response.body) + response_obj = salt.utils.json.loads(response.body) self.assertEqual(response_obj['return'], [{'minion': True, 'sub_minion': True}]) @@ -561,7 +555,7 @@ class TestWebhookSaltAPIHandler(_SaltnadoIntegrationTestCase): body='foo=bar', headers={saltnado.AUTH_TOKEN_HEADER: self.token['token']}, ) - response_obj = json_loads(response.body) + response_obj = salt.utils.json.loads(response.body) self.assertTrue(response_obj['success']) resolve_future_timeout = 60 self._future_resolved.wait(resolve_future_timeout) diff --git a/tests/integration/runners/test_state.py b/tests/integration/runners/test_state.py index d41c2ea5152..5283ff90c05 100644 --- a/tests/integration/runners/test_state.py +++ b/tests/integration/runners/test_state.py @@ -6,7 +6,6 @@ Tests for the state runner # Import Python Libs from __future__ import absolute_import, print_function, unicode_literals import errno -import json import os import shutil import signal @@ -25,6 +24,7 @@ from tests.support.paths import TMP import salt.utils.platform import salt.utils.event import salt.utils.files +import salt.utils.json import salt.utils.stringutils # Import 3rd-party libs @@ -94,7 +94,7 @@ class StateRunnerTest(ShellCase): See https://github.com/saltstack/salt/issues/43204 ''' self.run_run('saltutil.sync_modules') - ret = json.loads( + ret = salt.utils.json.loads( '\n'.join( self.run_run('state.orchestrate orch.issue43204 --out=json') ) diff --git a/tests/integration/shell/test_key.py b/tests/integration/shell/test_key.py index 79d64594c02..9600c5d1849 100644 --- a/tests/integration/shell/test_key.py +++ b/tests/integration/shell/test_key.py @@ -132,8 +132,8 @@ class KeyTest(ShellCase, ShellCaseCommonTestsMixin): data = self.run_key('-L --out json') ret = {} try: - import json - ret = json.loads('\n'.join(data)) + import salt.utils.json + ret = salt.utils.json.loads('\n'.join(data)) except ValueError: pass diff --git a/tests/integration/states/test_bower.py b/tests/integration/states/test_bower.py index afbc2c4b982..02e43eea6c7 100644 --- a/tests/integration/states/test_bower.py +++ b/tests/integration/states/test_bower.py @@ -4,7 +4,6 @@ ''' # Import Python libs from __future__ import absolute_import -import json # Import Salt Testing libs from tests.support.case import ModuleCase @@ -13,6 +12,7 @@ from tests.support.helpers import destructiveTest from tests.support.mixins import SaltReturnAssertsMixin # Import salt libs +import salt.utils.json import salt.utils.path @@ -58,7 +58,7 @@ class BowerStateTest(ModuleCase, SaltReturnAssertsMixin): ret = self.run_state('file.directory', name='/salt_test_bower_3', makedirs=True) self.assertSaltTrueReturn(ret) - bower_json = json.dumps({ + bower_json = salt.utils.json.dumps({ 'name': 'salt_test_bower_3', 'dependencies': { 'numeral': '~1.5.3', diff --git a/tests/jenkins.py b/tests/jenkins.py index 2eda9878f98..0bbb0f89b7f 100644 --- a/tests/jenkins.py +++ b/tests/jenkins.py @@ -13,7 +13,6 @@ import glob import os import re import sys -import json import time import shutil import optparse @@ -22,6 +21,7 @@ import random # Import Salt libs import salt.utils.files +import salt.utils.json import salt.utils.stringutils try: from salt.utils.nb_popen import NonBlockingPopen @@ -472,7 +472,7 @@ def run(opts): sys.exit(retcode) try: - version_info = json.loads(outstr) + version_info = salt.utils.json.loads(outstr) bootstrap_minion_version = os.environ.get( 'SALT_MINION_BOOTSTRAP_RELEASE', opts.bootstrap_salt_commit[:7] @@ -657,7 +657,7 @@ def run(opts): sys.exit(retcode) try: - remotes_info = json.loads(stdout.strip()) + remotes_info = salt.utils.json.loads(stdout.strip()) if remotes_info is None or remotes_info[vm_name] is None or opts.test_git_url not in remotes_info[vm_name]: print('The cloned repository remote is not the desired one:') print(' \'{0}\' is not in {1}'.format(opts.test_git_url, remotes_info)) @@ -704,7 +704,7 @@ def run(opts): sys.exit(retcode) try: - revision_info = json.loads(stdout.strip()) + revision_info = salt.utils.json.loads(stdout.strip()) if revision_info[vm_name][7:] != opts.test_git_commit[7:]: print('The cloned repository commit is not the desired one:') print(' \'{0}\' != \'{1}\''.format(revision_info[vm_name][:7], opts.test_git_commit[:7])) diff --git a/tests/modparser.py b/tests/modparser.py index 7e0aac21e42..cc8b2d7cac0 100644 --- a/tests/modparser.py +++ b/tests/modparser.py @@ -13,7 +13,7 @@ import os import modulefinder import pprint import yaml -import json +import salt.utils.json def parse(): @@ -99,7 +99,7 @@ if __name__ == '__main__': if opts['format'] == 'yaml': print(yaml.dump(scand)) if opts['format'] == 'json': - print(json.dumps(scand)) + print(salt.utils.json.dumps(scand)) else: pprint.pprint(scand) exit(0) diff --git a/tests/support/case.py b/tests/support/case.py index 45b2a23f6db..aee748c9a5b 100644 --- a/tests/support/case.py +++ b/tests/support/case.py @@ -17,7 +17,6 @@ from __future__ import absolute_import import os import re import sys -import json import time import stat import errno @@ -36,6 +35,7 @@ from tests.support.mixins import AdaptedConfigurationTestCaseMixin, SaltClientTe from tests.support.paths import ScriptPathMixin, INTEGRATION_TEST_DIR, CODE_DIR, PYEXEC, SCRIPT_DIR # Import 3rd-party libs +import salt.utils.json from salt.ext import six from salt.ext.six.moves import cStringIO # pylint: disable=import-error @@ -827,7 +827,7 @@ class SSHCase(ShellCase): log.debug('SSHCase run_function executed %s with arg %s', function, arg) log.debug('SSHCase JSON return: %s', ret) try: - return json.loads(ret)['localhost'] + return salt.utils.json.loads(ret)['localhost'] except Exception: return ret diff --git a/tests/support/parser/cover.py b/tests/support/parser/cover.py index bb91fea4d87..aa08c47acd7 100644 --- a/tests/support/parser/cover.py +++ b/tests/support/parser/cover.py @@ -16,10 +16,12 @@ from __future__ import absolute_import, print_function import os import re import sys -import json import shutil import warnings +# Import Salt libs +import salt.utils.json + # Import salt testing libs from tests.support.parser import SaltTestingParser @@ -42,7 +44,7 @@ try: coverage_object.save() def multiprocessing_start(obj): - coverage_options = json.loads(os.environ.get('COVERAGE_OPTIONS', '{}')) + coverage_options = salt.utils.json.loads(os.environ.get('COVERAGE_OPTIONS', '{}')) if not coverage_options: return @@ -176,7 +178,7 @@ class SaltCoverageTestingParser(SaltTestingParser): # included in the report coverage_options['data_suffix'] = True os.environ['COVERAGE_PROCESS_START'] = '1' - os.environ['COVERAGE_OPTIONS'] = json.dumps(coverage_options) + os.environ['COVERAGE_OPTIONS'] = salt.utils.json.dumps(coverage_options) # Setup coverage self.code_coverage = coverage.coverage(**coverage_options) diff --git a/tests/support/runtests.py b/tests/support/runtests.py index 59765e16a33..f2a896a7aa6 100644 --- a/tests/support/runtests.py +++ b/tests/support/runtests.py @@ -50,11 +50,12 @@ from __future__ import absolute_import, print_function import os import sys -import json import shutil import logging import multiprocessing +import salt.utils.json + # Import tests support libs import tests.support.paths as paths @@ -78,7 +79,7 @@ try: coverage_object.save() def multiprocessing_start(obj): - coverage_options = json.loads(os.environ.get('SALT_RUNTESTS_COVERAGE_OPTIONS', '{}')) + coverage_options = salt.utils.json.loads(os.environ.get('SALT_RUNTESTS_COVERAGE_OPTIONS', '{}')) if not coverage_options: return diff --git a/tests/unit/config/schemas/test_ssh.py b/tests/unit/config/schemas/test_ssh.py index bd3377ac24a..7ba495f7b3f 100644 --- a/tests/unit/config/schemas/test_ssh.py +++ b/tests/unit/config/schemas/test_ssh.py @@ -126,8 +126,8 @@ class RosterEntryConfigTest(TestCase): self.assertDictContainsSubset(expected['properties'], config.serialize()['properties']) self.assertDictContainsSubset(expected, config.serialize()) except AssertionError: - import json - print(json.dumps(config.serialize(), indent=4)) + import salt.utils.json + print(salt.utils.json.dumps(config.serialize(), indent=4)) raise @skipIf(HAS_JSONSCHEMA is False, 'The \'jsonschema\' library is missing') @@ -263,8 +263,8 @@ class RosterItemTest(TestCase): ssh_schemas.RosterItem.serialize() ) except AssertionError: - import json - print(json.dumps(ssh_schemas.RosterItem.serialize(), indent=4)) + import salt.utils.json + print(salt.utils.json.dumps(ssh_schemas.RosterItem.serialize(), indent=4)) raise @skipIf(HAS_JSONSCHEMA is False, 'The \'jsonschema\' library is missing') diff --git a/tests/unit/modules/test_ddns.py b/tests/unit/modules/test_ddns.py index 16d7c9ef4b9..b6c0cc87b3e 100644 --- a/tests/unit/modules/test_ddns.py +++ b/tests/unit/modules/test_ddns.py @@ -5,7 +5,6 @@ # Import Python libs from __future__ import absolute_import import textwrap -import json try: import dns.query import dns.tsigkeyring @@ -26,6 +25,7 @@ from tests.support.mock import ( ) # Import Salt Libs +import salt.utils.json import salt.modules.ddns as ddns @@ -109,7 +109,7 @@ class DDNSTestCase(TestCase, LoaderModuleMockMixin): ''' Test to delete a DNS record. ''' - file_data = json.dumps({'A': 'B'}) + file_data = salt.utils.json.dumps({'A': 'B'}) class MockAnswer(object): def __init__(self, *args, **kwargs): diff --git a/tests/unit/modules/test_junos.py b/tests/unit/modules/test_junos.py index e8b1c6d99a8..6bcc57b9ea8 100644 --- a/tests/unit/modules/test_junos.py +++ b/tests/unit/modules/test_junos.py @@ -1497,7 +1497,7 @@ class Test_Junos_Module(TestCase, LoaderModuleMockMixin, XMLEqualityMixin): def test_rpc_write_file_format_json(self): with patch('jnpr.junos.device.Device.execute') as mock_execute, \ - patch('salt.modules.junos.json.dumps') as mock_dumps: + patch('salt.utils.json.dumps') as mock_dumps: mock_dumps.return_value = 'json rpc reply' m = mock_open() with patch('salt.utils.files.fopen', m, create=True): diff --git a/tests/unit/modules/test_k8s.py b/tests/unit/modules/test_k8s.py index dde48aa27ca..c5068da9628 100644 --- a/tests/unit/modules/test_k8s.py +++ b/tests/unit/modules/test_k8s.py @@ -5,7 +5,6 @@ Unit Tests for the k8s execution module. # Import Python libs from __future__ import absolute_import -import json import hashlib import base64 import time @@ -17,6 +16,7 @@ from tests.support.helpers import skip_if_binaries_missing # Import Salt libs import salt.utils.files +import salt.utils.json import salt.modules.k8s as k8s # Import 3rd-party libs @@ -32,7 +32,7 @@ class TestK8SNamespace(TestCase): res = k8s.get_namespaces(apiserver_url="http://127.0.0.1:8080") a = len(res.get("items")) proc = Popen(["kubectl", "get", "namespaces", "-o", "json"], stdout=PIPE) - kubectl_out = json.loads(proc.communicate()[0]) + kubectl_out = salt.utils.json.loads(proc.communicate()[0]) b = len(kubectl_out.get("items")) self.assertEqual(a, b) @@ -40,7 +40,7 @@ class TestK8SNamespace(TestCase): res = k8s.get_namespaces("default", apiserver_url="http://127.0.0.1:8080") a = res.get("metadata", {}).get("name", "a") proc = Popen(["kubectl", "get", "namespaces", "default", "-o", "json"], stdout=PIPE) - kubectl_out = json.loads(proc.communicate()[0]) + kubectl_out = salt.utils.json.loads(proc.communicate()[0]) b = kubectl_out.get("metadata", {}).get("name", "b") self.assertEqual(a, b) @@ -50,7 +50,7 @@ class TestK8SNamespace(TestCase): nsname = hash.hexdigest()[:16] res = k8s.create_namespace(nsname, apiserver_url="http://127.0.0.1:8080") proc = Popen(["kubectl", "get", "namespaces", nsname, "-o", "json"], stdout=PIPE) - kubectl_out = json.loads(proc.communicate()[0]) + kubectl_out = salt.utils.json.loads(proc.communicate()[0]) # if creation is failed, kubernetes return non json error message self.assertTrue(isinstance(kubectl_out, dict)) @@ -79,7 +79,7 @@ class TestK8SSecrets(TestCase): res = k8s.get_secrets("default", apiserver_url="http://127.0.0.1:8080") a = len(res.get("items", [])) proc = Popen(["kubectl", "--namespace=default", "get", "secrets", "-o", "json"], stdout=PIPE) - kubectl_out = json.loads(proc.communicate()[0]) + kubectl_out = salt.utils.json.loads(proc.communicate()[0]) b = len(kubectl_out.get("items", [])) self.assertEqual(a, b) @@ -87,7 +87,7 @@ class TestK8SSecrets(TestCase): name = self.name filename = "/tmp/{0}.json".format(name) with salt.utils.files.fopen(filename, 'w') as f: - json.dump(self.request, f) + salt.utils.json.dump(self.request, f) create = Popen(["kubectl", "--namespace=default", "create", "-f", filename], stdout=PIPE) # wee need to give kubernetes time save data in etcd @@ -95,7 +95,7 @@ class TestK8SSecrets(TestCase): res = k8s.get_secrets("default", name, apiserver_url="http://127.0.0.1:8080") a = res.get("metadata", {}).get("name", "a") proc = Popen(["kubectl", "--namespace=default", "get", "secrets", name, "-o", "json"], stdout=PIPE) - kubectl_out = json.loads(proc.communicate()[0]) + kubectl_out = salt.utils.json.loads(proc.communicate()[0]) b = kubectl_out.get("metadata", {}).get("name", "b") self.assertEqual(a, b) @@ -103,7 +103,7 @@ class TestK8SSecrets(TestCase): name = self.name filename = "/tmp/{0}.json".format(name) with salt.utils.files.fopen(filename, 'w') as f: - json.dump(self.request, f) + salt.utils.json.dump(self.request, f) create = Popen(["kubectl", "--namespace=default", "create", "-f", filename], stdout=PIPE) # wee need to give etcd to populate data on all nodes @@ -123,7 +123,7 @@ class TestK8SSecrets(TestCase): f.write("{0}{1}".format(name, i)) res = k8s.create_secret("default", name, names, apiserver_url="http://127.0.0.1:8080") proc = Popen(["kubectl", "--namespace=default", "get", "secrets", name, "-o", "json"], stdout=PIPE) - kubectl_out = json.loads(proc.communicate()[0]) + kubectl_out = salt.utils.json.loads(proc.communicate()[0]) # if creation is failed, kubernetes return non json error message b = kubectl_out.get("data", {}) self.assertTrue(isinstance(kubectl_out, dict)) @@ -133,7 +133,7 @@ class TestK8SSecrets(TestCase): name = self.name filename = "/tmp/{0}.json".format(name) with salt.utils.files.fopen(filename, 'w') as f: - json.dump(self.request, f) + salt.utils.json.dump(self.request, f) create = Popen(["kubectl", "--namespace=default", "create", "-f", filename], stdout=PIPE) # wee need to give kubernetes time save data in etcd @@ -149,7 +149,7 @@ class TestK8SSecrets(TestCase): res = k8s.update_secret("default", name, names, apiserver_url="http://127.0.0.1:8080") # if creation is failed, kubernetes return non json error message proc = Popen(["kubectl", "--namespace=default", "get", "secrets", name, "-o", "json"], stdout=PIPE) - kubectl_out = json.loads(proc.communicate()[0]) + kubectl_out = salt.utils.json.loads(proc.communicate()[0]) # if creation is failed, kubernetes return non json error message b = kubectl_out.get("data", {}) self.assertTrue(isinstance(kubectl_out, dict)) @@ -159,7 +159,7 @@ class TestK8SSecrets(TestCase): name = self.name filename = "/tmp/{0}.json".format(name) with salt.utils.files.fopen(filename, 'w') as f: - json.dump(self.request, f) + salt.utils.json.dump(self.request, f) create = Popen(["kubectl", "--namespace=default", "create", "-f", filename], stdout=PIPE) # wee need to give kubernetes time save data in etcd @@ -214,7 +214,7 @@ spec: res = k8s.get_resource_quotas(namespace, apiserver_url="http://127.0.0.1:8080") a = len(res.get("items", [])) proc = Popen(["kubectl", "--namespace={0}".format(namespace), "get", "quota", "-o", "json"], stdout=PIPE) - kubectl_out = json.loads(proc.communicate()[0]) + kubectl_out = salt.utils.json.loads(proc.communicate()[0]) b = len(kubectl_out.get("items", [])) self.assertEqual(a, b) @@ -248,7 +248,7 @@ spec: res = k8s.get_resource_quotas(namespace, name, apiserver_url="http://127.0.0.1:8080") a = res.get("metadata", {}).get("name", "a") proc = Popen(["kubectl", "--namespace={0}".format(namespace), "get", "quota", name, "-o", "json"], stdout=PIPE) - kubectl_out = json.loads(proc.communicate()[0]) + kubectl_out = salt.utils.json.loads(proc.communicate()[0]) b = kubectl_out.get("metadata", {}).get("name", "b") self.assertEqual(a, b) @@ -262,7 +262,7 @@ spec: } res = k8s.create_resource_quota(namespace, quota, name=name, apiserver_url="http://127.0.0.1:8080") proc = Popen(["kubectl", "--namespace={0}".format(namespace), "get", "quota", name, "-o", "json"], stdout=PIPE) - kubectl_out = json.loads(proc.communicate()[0]) + kubectl_out = salt.utils.json.loads(proc.communicate()[0]) self.assertTrue(isinstance(kubectl_out, dict)) def test_update_resource_quota(self): @@ -298,7 +298,7 @@ spec: } res = k8s.create_resource_quota(namespace, quota, name=name, apiserver_url="http://127.0.0.1:8080", update=True) proc = Popen(["kubectl", "--namespace={0}".format(namespace), "get", "quota", name, "-o", "json"], stdout=PIPE) - kubectl_out = json.loads(proc.communicate()[0]) + kubectl_out = salt.utils.json.loads(proc.communicate()[0]) limit = kubectl_out.get("spec").get("hard").get("memory") self.assertEqual("2Gi", limit) @@ -324,7 +324,7 @@ class TestK8SLimitRange(TestCase): } res = k8s.create_limit_range("default", limits, name=name, apiserver_url="http://127.0.0.1:8080") proc = Popen(["kubectl", "--namespace=default", "get", "limits", name, "-o", "json"], stdout=PIPE) - kubectl_out = json.loads(proc.communicate()[0]) + kubectl_out = salt.utils.json.loads(proc.communicate()[0]) self.assertTrue(isinstance(kubectl_out, dict)) def test_update_limit_range(self): @@ -360,7 +360,7 @@ spec: time.sleep(0.1) res = k8s.create_limit_range("default", limits, name=name, apiserver_url="http://127.0.0.1:8080", update=True) proc = Popen(["kubectl", "--namespace=default", "get", "limits", name, "-o", "json"], stdout=PIPE) - kubectl_out = json.loads(proc.communicate()[0]) + kubectl_out = salt.utils.json.loads(proc.communicate()[0]) limit = kubectl_out.get("spec").get("limits")[0].get("defaultRequest").get("cpu") self.assertEqual("100m", limit) @@ -368,7 +368,7 @@ spec: res = k8s.get_limit_ranges("default", apiserver_url="http://127.0.0.1:8080") a = len(res.get("items", [])) proc = Popen(["kubectl", "--namespace=default", "get", "limits", "-o", "json"], stdout=PIPE) - kubectl_out = json.loads(proc.communicate()[0]) + kubectl_out = salt.utils.json.loads(proc.communicate()[0]) b = len(kubectl_out.get("items", [])) self.assertEqual(a, b) @@ -399,6 +399,6 @@ spec: res = k8s.get_limit_ranges("default", name, apiserver_url="http://127.0.0.1:8080") a = res.get("metadata", {}).get("name", "a") proc = Popen(["kubectl", "--namespace=default", "get", "limits", name, "-o", "json"], stdout=PIPE) - kubectl_out = json.loads(proc.communicate()[0]) + kubectl_out = salt.utils.json.loads(proc.communicate()[0]) b = kubectl_out.get("metadata", {}).get("name", "b") self.assertEqual(a, b) diff --git a/tests/unit/modules/test_kapacitor.py b/tests/unit/modules/test_kapacitor.py index cc8eb1fd16e..31be297fc83 100644 --- a/tests/unit/modules/test_kapacitor.py +++ b/tests/unit/modules/test_kapacitor.py @@ -2,9 +2,9 @@ # Import Python libs from __future__ import absolute_import -import json # Import Salt libs +import salt.utils.json import salt.modules.kapacitor as kapacitor # Import Salt testing libs @@ -27,7 +27,7 @@ class KapacitorTestCase(TestCase, LoaderModuleMockMixin): } def test_get_task_success(self): - http_body = json.dumps({ + http_body = salt.utils.json.dumps({ 'script': 'test', 'type': 'stream', 'dbrps': [{'db': 'db', 'rp': 'rp'}], diff --git a/tests/unit/modules/test_npm.py b/tests/unit/modules/test_npm.py index 4c38bf29386..2c23549c688 100644 --- a/tests/unit/modules/test_npm.py +++ b/tests/unit/modules/test_npm.py @@ -5,7 +5,6 @@ # Import Python Libs from __future__ import absolute_import -import json # Import Salt Testing Libs from tests.support.mixins import LoaderModuleMockMixin @@ -18,6 +17,7 @@ from tests.support.mock import ( ) # Import Salt Libs +import salt.utils.json import salt.modules.npm as npm from salt.exceptions import CommandExecutionError @@ -49,14 +49,14 @@ class NpmTestCase(TestCase, LoaderModuleMockMixin): 'stdout': '{"salt": ["SALT"]}'}) with patch.dict(npm.__salt__, {'cmd.run_all': mock}): mock_err = MagicMock(return_value='SALT') - with patch.object(json, 'loads', mock_err): + with patch.object(salt.utils.json, 'loads', mock_err): self.assertEqual(npm.install('coffee-script'), 'SALT') mock = MagicMock(return_value={'retcode': 0, 'stderr': 'error', 'stdout': '{"salt": ["SALT"]}'}) with patch.dict(npm.__salt__, {'cmd.run_all': mock}): mock_err = MagicMock(side_effect=ValueError()) - with patch.object(json, 'loads', mock_err): + with patch.object(salt.utils.json, 'loads', mock_err): self.assertEqual(npm.install('coffee-script'), '{"salt": ["SALT"]}') @@ -88,7 +88,7 @@ class NpmTestCase(TestCase, LoaderModuleMockMixin): 'stdout': '{"salt": ["SALT"]}'}) with patch.dict(npm.__salt__, {'cmd.run_all': mock}): mock_err = MagicMock(return_value={'dependencies': 'SALT'}) - with patch.object(json, 'loads', mock_err): + with patch.object(salt.utils.json, 'loads', mock_err): self.assertEqual(npm.list_('coffee-script'), 'SALT') # 'cache_clean' function tests: 1 diff --git a/tests/unit/modules/test_pagerduty.py b/tests/unit/modules/test_pagerduty.py index 2e7e59cbb92..63eaf55bef9 100644 --- a/tests/unit/modules/test_pagerduty.py +++ b/tests/unit/modules/test_pagerduty.py @@ -5,7 +5,6 @@ # Import Python Libs from __future__ import absolute_import -import json # Import Salt Testing Libs from tests.support.mixins import LoaderModuleMockMixin @@ -18,6 +17,7 @@ from tests.support.mock import ( ) # Import Salt Libs +import salt.utils.json import salt.utils.pagerduty import salt.modules.pagerduty as pagerduty @@ -82,7 +82,7 @@ class PagerdutyTestCase(TestCase, LoaderModuleMockMixin): ''' Test for Create an event in PagerDuty. Designed for use in states. ''' - with patch.object(json, 'loads', return_value=['A']): + with patch.object(salt.utils.json, 'loads', return_value=['A']): with patch.object(salt.utils.pagerduty, 'query', return_value='A'): self.assertListEqual(pagerduty.create_event(), ['A']) diff --git a/tests/unit/modules/test_saltcloudmod.py b/tests/unit/modules/test_saltcloudmod.py index fabf283a7ef..59e52185c3c 100644 --- a/tests/unit/modules/test_saltcloudmod.py +++ b/tests/unit/modules/test_saltcloudmod.py @@ -18,26 +18,7 @@ from tests.support.mock import ( # Import Salt Libs import salt.modules.saltcloudmod as saltcloudmod - - -class MockJson(object): - ''' - Mock json class - ''' - flag = None - - def __init__(self): - pass - - @staticmethod - def loads(data, object_hook): - ''' - Mock load method - ''' - if MockJson.flag: - return data, object_hook - else: - raise ValueError +import salt.utils.json @skipIf(NO_MOCK, NO_MOCK_REASON) @@ -46,23 +27,22 @@ class SaltcloudmodTestCase(TestCase, LoaderModuleMockMixin): Test cases for salt.modules.saltcloudmod ''' def setup_loader_modules(self): - return {saltcloudmod: {'json': MockJson}} + return {saltcloudmod: {}} + + def setUp(self): + self.mock_json_loads = MagicMock(side_effect=ValueError()) def test_create(self): ''' Test if create the named vm ''' - MockJson.flag = True - mock = MagicMock(return_value=True) + mock = MagicMock(return_value='''{"foo": "bar"}''') with patch.dict(saltcloudmod.__salt__, {'cmd.run_stdout': mock}): - self.assertTrue(saltcloudmod.create("webserver", - "rackspace_centos_512" - ) - ) + self.assertTrue( + saltcloudmod.create("webserver", "rackspace_centos_512")) - MockJson.flag = False - self.assertDictEqual(saltcloudmod.create("webserver", - "rackspace_centos_512" - ), - {} - ) + with patch.object(salt.utils.json, 'loads', self.mock_json_loads): + self.assertDictEqual( + saltcloudmod.create("webserver", "rackspace_centos_512"), + {} + ) diff --git a/tests/unit/modules/test_serverdensity_device.py b/tests/unit/modules/test_serverdensity_device.py index 4061f80db6e..81d73f7a3fb 100644 --- a/tests/unit/modules/test_serverdensity_device.py +++ b/tests/unit/modules/test_serverdensity_device.py @@ -17,40 +17,17 @@ from tests.support.mock import ( ) # Import Salt Libs +import salt.utils.json import salt.modules.serverdensity_device as serverdensity_device from salt.exceptions import CommandExecutionError -class MockJson(Exception): - ''' - Mock SMTP_SSL class - ''' - flag = None - - def loads(self, content): - ''' - Mock loads method. - ''' - if self.flag == 1: - raise ValueError - return content - - def dumps(self, dumps): - ''' - Mock dumps method. - ''' - if self.flag == 1: - return None - return dumps - - class MockRequests(object): ''' Mock smtplib class ''' flag = None - content = {"message": "Invalid token", - "errors": [{"type": "invalid_token", "subject": "token"}]} + content = '''{"message": "Invalid token", "errors": [{"type": "invalid_token", "subject": "token"}]}''' status_code = None def __init__(self): @@ -105,11 +82,13 @@ class ServerdensityDeviceTestCase(TestCase, LoaderModuleMockMixin): def setup_loader_modules(self): return { serverdensity_device: { - 'json': MockJson(), 'requests': MockRequests() } } + def setUp(self): + self.mock_json_loads = MagicMock(side_effect=ValueError()) + # 'get_sd_auth' function tests: 1 def test_get_sd_auth(self): @@ -140,10 +119,10 @@ class ServerdensityDeviceTestCase(TestCase, LoaderModuleMockMixin): self.assertTrue(serverdensity_device.create('rich_lama', group='lama_band')) - MockJson.flag = 1 - self.assertRaises(CommandExecutionError, - serverdensity_device.create, 'rich_lama', - group='lama_band') + with patch.object(salt.utils.json, 'loads', self.mock_json_loads): + self.assertRaises(CommandExecutionError, + serverdensity_device.create, 'rich_lama', + group='lama_band') MockRequests.flag = 1 self.assertIsNone(serverdensity_device.create('rich_lama', @@ -158,12 +137,11 @@ class ServerdensityDeviceTestCase(TestCase, LoaderModuleMockMixin): with patch.dict(serverdensity_device.__pillar__, {'serverdensity': {'api_token': 'salt'}}): MockRequests.flag = 0 - MockJson.flag = 0 self.assertTrue(serverdensity_device.delete('51f7eaf')) - MockJson.flag = 1 - self.assertRaises(CommandExecutionError, - serverdensity_device.delete, '51f7eaf') + with patch.object(salt.utils.json, 'loads', self.mock_json_loads): + self.assertRaises(CommandExecutionError, + serverdensity_device.delete, '51f7eaf') MockRequests.flag = 1 self.assertIsNone(serverdensity_device.delete('51f7eaf')) @@ -177,12 +155,11 @@ class ServerdensityDeviceTestCase(TestCase, LoaderModuleMockMixin): with patch.dict(serverdensity_device.__pillar__, {'serverdensity': {'api_token': 'salt'}}): MockRequests.flag = 0 - MockJson.flag = 0 self.assertTrue(serverdensity_device.ls(name='lama')) - MockJson.flag = 1 - self.assertRaises(CommandExecutionError, - serverdensity_device.ls, name='lama') + with patch.object(salt.utils.json, 'loads', self.mock_json_loads): + self.assertRaises(CommandExecutionError, + serverdensity_device.ls, name='lama') MockRequests.flag = 1 self.assertIsNone(serverdensity_device.ls(name='lama')) @@ -196,13 +173,12 @@ class ServerdensityDeviceTestCase(TestCase, LoaderModuleMockMixin): with patch.dict(serverdensity_device.__pillar__, {'serverdensity': {'api_token': 'salt'}}): MockRequests.flag = 0 - MockJson.flag = 0 self.assertTrue(serverdensity_device.update('51f7eaf', name='lama')) - MockJson.flag = 1 - self.assertRaises(CommandExecutionError, - serverdensity_device.update, '51f7eaf', - name='lama') + with patch.object(salt.utils.json, 'loads', self.mock_json_loads): + self.assertRaises(CommandExecutionError, + serverdensity_device.update, '51f7eaf', + name='lama') MockRequests.flag = 1 self.assertIsNone(serverdensity_device.update('51f7eaf', diff --git a/tests/unit/modules/test_state.py b/tests/unit/modules/test_state.py index 4f11f4ed032..15e7806c50d 100644 --- a/tests/unit/modules/test_state.py +++ b/tests/unit/modules/test_state.py @@ -325,27 +325,6 @@ class MockTarFile(object): return True -class MockJson(object): - ''' - Mock json class - ''' - flag = None - - def __init__(self): - pass - - def load(self, data, object_hook=None): - ''' - Mock load method - ''' - data = data - object_hook = object_hook - if self.flag: - return [True] - else: - return [{"test": ""}] - - @skipIf(NO_MOCK, NO_MOCK_REASON) class StateTestCase(TestCase, LoaderModuleMockMixin): ''' @@ -357,6 +336,7 @@ class StateTestCase(TestCase, LoaderModuleMockMixin): salt.config.DEFAULT_MINION_OPTS, whitelist=['state'] ) + utils.keys() patcher = patch('salt.modules.state.salt.state', MockState()) patcher.start() self.addCleanup(patcher.stop) @@ -1031,10 +1011,13 @@ class StateTestCase(TestCase, LoaderModuleMockMixin): Test to execute a packaged state run ''' tar_file = os.sep + os.path.join('tmp', 'state_pkg.tgz') - mock = MagicMock(side_effect=[False, True, True, True, True, True, True, True]) + mock = MagicMock(side_effect=[False, True, True, True, True, True, + True, True, True, True, True]) + mock_json_loads_true = MagicMock(return_value=[True]) + mock_json_loads_dictlist = MagicMock(return_value=[{"test": ""}]) with patch.object(os.path, 'isfile', mock), \ patch('salt.modules.state.tarfile', MockTarFile), \ - patch('salt.modules.state.json', MockJson()): + patch.object(salt.utils, 'json', mock_json_loads_dictlist): self.assertEqual(state.pkg(tar_file, "", "md5"), {}) mock = MagicMock(side_effect=[False, 0, 0, 0, 0]) @@ -1046,12 +1029,11 @@ class StateTestCase(TestCase, LoaderModuleMockMixin): self.assertDictEqual(state.pkg(tar_file, 0, "md5"), {}) MockTarFile.path = "" - MockJson.flag = True - with patch('salt.utils.files.fopen', mock_open()): - self.assertListEqual(state.pkg(tar_file, 0, "md5"), [True]) + with patch('salt.utils.files.fopen', mock_open()), \ + patch.object(salt.utils.json, 'loads', mock_json_loads_true): + self.assertEqual(state.pkg(tar_file, 0, "md5"), True) MockTarFile.path = "" - MockJson.flag = False if six.PY2: with patch('salt.utils.files.fopen', mock_open()), \ patch.dict(state.__utils__, {'state.check_result': MagicMock(return_value=True)}): diff --git a/tests/unit/modules/test_win_iis.py b/tests/unit/modules/test_win_iis.py index 81015db1308..c4ba2093638 100644 --- a/tests/unit/modules/test_win_iis.py +++ b/tests/unit/modules/test_win_iis.py @@ -8,11 +8,11 @@ # Import Python Libs from __future__ import absolute_import -import json # Import Salt Libs from salt.exceptions import SaltInvocationError import salt.modules.win_iis as win_iis +import salt.utils.json # Import Salt Testing Libs from tests.support.mixins import LoaderModuleMockMixin @@ -78,7 +78,7 @@ VDIR_LIST = { LIST_APPS_SRVMGR = { 'retcode': 0, - 'stdout': json.dumps([{ + 'stdout': salt.utils.json.dumps([{ 'applicationPool': 'MyTestPool', 'name': 'testApp', 'path': '/testApp', 'PhysicalPath': r'C:\inetpub\apps\testApp', @@ -89,7 +89,7 @@ LIST_APPS_SRVMGR = { LIST_APPPOOLS_SRVMGR = { 'retcode': 0, - 'stdout': json.dumps([{ + 'stdout': salt.utils.json.dumps([{ 'name': 'MyTestPool', 'state': 'Started', 'Applications': { 'value': ['MyTestSite'], @@ -100,7 +100,7 @@ LIST_APPPOOLS_SRVMGR = { LIST_VDIRS_SRVMGR = { 'retcode': 0, - 'stdout': json.dumps([{ + 'stdout': salt.utils.json.dumps([{ 'name': 'TestVdir', 'physicalPath': r'C:\inetpub\vdirs\TestVdir' }]) @@ -108,7 +108,7 @@ LIST_VDIRS_SRVMGR = { CONTAINER_SETTING = { 'retcode': 0, - 'stdout': json.dumps([{ + 'stdout': salt.utils.json.dumps([{ 'managedPipelineMode': 'Integrated' }]) } @@ -346,7 +346,7 @@ class WinIisTestCase(TestCase, LoaderModuleMockMixin): MagicMock(return_value={'9988776655443322111000AAABBBCCCDDDEEEFFF': None})), \ patch('salt.modules.win_iis._srvmgr', MagicMock(return_value={'retcode': 0, 'stdout': 10})), \ - patch('json.loads', MagicMock(return_value=[{'MajorVersion': 10, 'MinorVersion': 0}])), \ + patch('salt.utils.json.loads', MagicMock(return_value=[{'MajorVersion': 10, 'MinorVersion': 0}])), \ patch('salt.modules.win_iis.list_bindings', MagicMock(return_value=BINDING_LIST)), \ patch('salt.modules.win_iis.list_cert_bindings', diff --git a/tests/unit/netapi/rest_cherrypy/test_tools.py b/tests/unit/netapi/rest_cherrypy/test_tools.py index a1e911f06e4..1d1b9291ae8 100644 --- a/tests/unit/netapi/rest_cherrypy/test_tools.py +++ b/tests/unit/netapi/rest_cherrypy/test_tools.py @@ -2,14 +2,15 @@ # Import Python libs from __future__ import absolute_import -import json +import yaml + +# Import Salt libs +import salt.utils.json +from tests.support.cherrypy_testclasses import BaseToolsTest # Import 3rd-party libs -import yaml from salt.ext.six.moves.urllib.parse import urlencode # pylint: disable=no-name-in-module,import-error -from tests.support.cherrypy_testclasses import BaseToolsTest - class TestOutFormats(BaseToolsTest): def __get_cp_config__(self): @@ -58,7 +59,7 @@ class TestInFormats(BaseToolsTest): def test_json_ctype(self): data = {'valid': 'stuff'} request, response = self.request('/', method='POST', - body=json.dumps(data), headers=( + body=salt.utils.json.dumps(data), headers=( ('Content-type', 'application/json'), )) self.assertEqual(response.status, '200 OK') @@ -70,7 +71,7 @@ class TestInFormats(BaseToolsTest): ''' data = {'valid': 'stuff'} request, response = self.request('/', method='POST', - body=json.dumps(data), headers=( + body=salt.utils.json.dumps(data), headers=( ('Content-type', 'text/plain'), )) self.assertEqual(response.status, '200 OK') diff --git a/tests/unit/netapi/rest_tornado/test_handlers.py b/tests/unit/netapi/rest_tornado/test_handlers.py index c7e88ccbdb0..3a5329f7275 100644 --- a/tests/unit/netapi/rest_tornado/test_handlers.py +++ b/tests/unit/netapi/rest_tornado/test_handlers.py @@ -2,7 +2,6 @@ # Import Python libs from __future__ import absolute_import -import json import os import copy import hashlib @@ -13,6 +12,7 @@ from tests.support.unit import TestCase, skipIf # Import Salt libs import salt.auth +import salt.utils.json from salt.ext.six.moves import map # pylint: disable=import-error try: import salt.netapi.rest_tornado as rest_tornado @@ -177,12 +177,12 @@ class TestBaseSaltAPIHandler(SaltnadoTestCase): # send NO accept header, should come back with json response = self.fetch('/') self.assertEqual(response.headers['Content-Type'], self.content_type_map['json']) - self.assertEqual(type(json.loads(response.body)), dict) + self.assertEqual(type(salt.utils.json.loads(response.body)), dict) # Request application/json response = self.fetch('/', headers={'Accept': self.content_type_map['json']}) self.assertEqual(response.headers['Content-Type'], self.content_type_map['json']) - self.assertEqual(type(json.loads(response.body)), dict) + self.assertEqual(type(salt.utils.json.loads(response.body)), dict) # Request application/x-yaml response = self.fetch('/', headers={'Accept': self.content_type_map['yaml']}) @@ -197,7 +197,7 @@ class TestBaseSaltAPIHandler(SaltnadoTestCase): accept_header = self.content_type_map['real-accept-header-json'] response = self.fetch('/', headers={'Accept': accept_header}) self.assertEqual(response.headers['Content-Type'], self.content_type_map['json']) - self.assertEqual(type(json.loads(response.body)), dict) + self.assertEqual(type(salt.utils.json.loads(response.body)), dict) # Request some YAML with a browser like Accept accept_header = self.content_type_map['real-accept-header-yaml'] @@ -209,23 +209,23 @@ class TestBaseSaltAPIHandler(SaltnadoTestCase): ''' Test that the token is returned correctly ''' - token = json.loads(self.fetch('/').body)['token'] + token = salt.utils.json.loads(self.fetch('/').body)['token'] self.assertIs(token, None) # send a token as a header response = self.fetch('/', headers={saltnado.AUTH_TOKEN_HEADER: 'foo'}) - token = json.loads(response.body)['token'] + token = salt.utils.json.loads(response.body)['token'] self.assertEqual(token, 'foo') # send a token as a cookie response = self.fetch('/', headers={'Cookie': '{0}=foo'.format(saltnado.AUTH_COOKIE_NAME)}) - token = json.loads(response.body)['token'] + token = salt.utils.json.loads(response.body)['token'] self.assertEqual(token, 'foo') # send both, make sure its the header response = self.fetch('/', headers={saltnado.AUTH_TOKEN_HEADER: 'foo', 'Cookie': '{0}=bar'.format(saltnado.AUTH_COOKIE_NAME)}) - token = json.loads(response.body)['token'] + token = salt.utils.json.loads(response.body)['token'] self.assertEqual(token, 'foo') def test_deserialize(self): @@ -248,10 +248,10 @@ class TestBaseSaltAPIHandler(SaltnadoTestCase): # send as JSON response = self.fetch('/', method='POST', - body=json.dumps(valid_lowstate), + body=salt.utils.json.dumps(valid_lowstate), headers={'Content-Type': self.content_type_map['json']}) - self.assertEqual(valid_lowstate, json.loads(response.body)['lowstate']) + self.assertEqual(valid_lowstate, salt.utils.json.loads(response.body)['lowstate']) # send yaml as json (should break) response = self.fetch('/', @@ -265,21 +265,21 @@ class TestBaseSaltAPIHandler(SaltnadoTestCase): method='POST', body=yaml.dump(valid_lowstate), headers={'Content-Type': self.content_type_map['yaml']}) - self.assertEqual(valid_lowstate, json.loads(response.body)['lowstate']) + self.assertEqual(valid_lowstate, salt.utils.json.loads(response.body)['lowstate']) # send json as yaml (works since yaml is a superset of json) response = self.fetch('/', method='POST', - body=json.dumps(valid_lowstate), + body=salt.utils.json.dumps(valid_lowstate), headers={'Content-Type': self.content_type_map['yaml']}) - self.assertEqual(valid_lowstate, json.loads(response.body)['lowstate']) + self.assertEqual(valid_lowstate, salt.utils.json.loads(response.body)['lowstate']) # send json as text/plain response = self.fetch('/', method='POST', - body=json.dumps(valid_lowstate), + body=salt.utils.json.dumps(valid_lowstate), headers={'Content-Type': self.content_type_map['text']}) - self.assertEqual(valid_lowstate, json.loads(response.body)['lowstate']) + self.assertEqual(valid_lowstate, salt.utils.json.loads(response.body)['lowstate']) # send form-urlencoded form_lowstate = ( @@ -293,7 +293,7 @@ class TestBaseSaltAPIHandler(SaltnadoTestCase): method='POST', body=urlencode(form_lowstate), headers={'Content-Type': self.content_type_map['form']}) - returned_lowstate = json.loads(response.body)['lowstate'] + returned_lowstate = salt.utils.json.loads(response.body)['lowstate'] self.assertEqual(len(returned_lowstate), 1) returned_lowstate = returned_lowstate[0] @@ -305,9 +305,9 @@ class TestBaseSaltAPIHandler(SaltnadoTestCase): # Send json with utf8 charset response = self.fetch('/', method='POST', - body=json.dumps(valid_lowstate), + body=salt.utils.json.dumps(valid_lowstate), headers={'Content-Type': self.content_type_map['json-utf8']}) - self.assertEqual(valid_lowstate, json.loads(response.body)['lowstate']) + self.assertEqual(valid_lowstate, salt.utils.json.loads(response.body)['lowstate']) def test_get_lowstate(self): ''' @@ -330,10 +330,10 @@ class TestBaseSaltAPIHandler(SaltnadoTestCase): response = self.fetch('/', method='POST', - body=json.dumps(request_lowstate), + body=salt.utils.json.dumps(request_lowstate), headers={'Content-Type': self.content_type_map['json']}) - self.assertEqual(valid_lowstate, json.loads(response.body)['lowstate']) + self.assertEqual(valid_lowstate, salt.utils.json.loads(response.body)['lowstate']) # Case 2. string type of arg request_lowstate = { @@ -345,10 +345,10 @@ class TestBaseSaltAPIHandler(SaltnadoTestCase): response = self.fetch('/', method='POST', - body=json.dumps(request_lowstate), + body=salt.utils.json.dumps(request_lowstate), headers={'Content-Type': self.content_type_map['json']}) - self.assertEqual(valid_lowstate, json.loads(response.body)['lowstate']) + self.assertEqual(valid_lowstate, salt.utils.json.loads(response.body)['lowstate']) # Case 3. Combine Case 1 and Case 2. request_lowstate = { @@ -361,24 +361,24 @@ class TestBaseSaltAPIHandler(SaltnadoTestCase): # send as json response = self.fetch('/', method='POST', - body=json.dumps(request_lowstate), + body=salt.utils.json.dumps(request_lowstate), headers={'Content-Type': self.content_type_map['json']}) - self.assertEqual(valid_lowstate, json.loads(response.body)['lowstate']) + self.assertEqual(valid_lowstate, salt.utils.json.loads(response.body)['lowstate']) # send as yaml response = self.fetch('/', method='POST', body=yaml.dump(request_lowstate), headers={'Content-Type': self.content_type_map['yaml']}) - self.assertEqual(valid_lowstate, json.loads(response.body)['lowstate']) + self.assertEqual(valid_lowstate, salt.utils.json.loads(response.body)['lowstate']) # send as plain text response = self.fetch('/', method='POST', - body=json.dumps(request_lowstate), + body=salt.utils.json.dumps(request_lowstate), headers={'Content-Type': self.content_type_map['text']}) - self.assertEqual(valid_lowstate, json.loads(response.body)['lowstate']) + self.assertEqual(valid_lowstate, salt.utils.json.loads(response.body)['lowstate']) # send as form-urlencoded request_form_lowstate = ( @@ -392,7 +392,7 @@ class TestBaseSaltAPIHandler(SaltnadoTestCase): method='POST', body=urlencode(request_form_lowstate), headers={'Content-Type': self.content_type_map['form']}) - self.assertEqual(valid_lowstate, json.loads(response.body)['lowstate']) + self.assertEqual(valid_lowstate, salt.utils.json.loads(response.body)['lowstate']) def test_cors_origin_wildcard(self): ''' @@ -487,7 +487,7 @@ class TestWebhookSaltHandler(SaltnadoTestCase): event.fire_event.return_value = True get_event.return_value = event response = self.fetch('/hook/my_service/?param=1¶m=2', - body=json.dumps({}), + body=salt.utils.json.dumps({}), method='POST', headers={'Content-Type': self.content_type_map['json']}) self.assertEqual(response.code, 200, response.body) @@ -530,7 +530,7 @@ class TestSaltAuthHandler(SaltnadoTestCase): headers={'Content-Type': self.content_type_map['form']}) self.assertEqual(response.code, 200) - response_obj = json.loads(response.body)['return'][0] + response_obj = salt.utils.json.loads(response.body)['return'][0] self.assertEqual(response_obj['perms'], self.opts['external_auth']['auto'][self.auth_creds_dict['username']]) self.assertIn('token', response_obj) # TODO: verify that its valid? self.assertEqual(response_obj['user'], self.auth_creds_dict['username']) @@ -539,11 +539,11 @@ class TestSaltAuthHandler(SaltnadoTestCase): # Test in JSON response = self.fetch('/login', method='POST', - body=json.dumps(self.auth_creds_dict), + body=salt.utils.json.dumps(self.auth_creds_dict), headers={'Content-Type': self.content_type_map['json']}) self.assertEqual(response.code, 200) - response_obj = json.loads(response.body)['return'][0] + response_obj = salt.utils.json.loads(response.body)['return'][0] self.assertEqual(response_obj['perms'], self.opts['external_auth']['auto'][self.auth_creds_dict['username']]) self.assertIn('token', response_obj) # TODO: verify that its valid? self.assertEqual(response_obj['user'], self.auth_creds_dict['username']) @@ -556,7 +556,7 @@ class TestSaltAuthHandler(SaltnadoTestCase): headers={'Content-Type': self.content_type_map['yaml']}) self.assertEqual(response.code, 200) - response_obj = json.loads(response.body)['return'][0] + response_obj = salt.utils.json.loads(response.body)['return'][0] self.assertEqual(response_obj['perms'], self.opts['external_auth']['auto'][self.auth_creds_dict['username']]) self.assertIn('token', response_obj) # TODO: verify that its valid? self.assertEqual(response_obj['user'], self.auth_creds_dict['username']) @@ -600,21 +600,21 @@ class TestSaltAuthHandler(SaltnadoTestCase): ''' response = self.fetch('/login', method='POST', - body=json.dumps(self.auth_creds), + body=salt.utils.json.dumps(self.auth_creds), headers={'Content-Type': self.content_type_map['form']}) self.assertEqual(response.code, 400) response = self.fetch('/login', method='POST', - body=json.dumps(42), + body=salt.utils.json.dumps(42), headers={'Content-Type': self.content_type_map['form']}) self.assertEqual(response.code, 400) response = self.fetch('/login', method='POST', - body=json.dumps('mystring42'), + body=salt.utils.json.dumps('mystring42'), headers={'Content-Type': self.content_type_map['form']}) self.assertEqual(response.code, 400) @@ -642,10 +642,10 @@ class TestSaltRunHandler(SaltnadoTestCase): for request_lowstate in request_lowstates: response = self.fetch('/run', method='POST', - body=json.dumps(request_lowstate), + body=salt.utils.json.dumps(request_lowstate), headers={'Content-Type': self.content_type_map['json']}) - self.assertEqual(valid_response, json.loads(response.body)) + self.assertEqual(valid_response, salt.utils.json.loads(response.body)) @skipIf(HAS_TORNADO is False, 'The tornado package needs to be installed') # pylint: disable=W0223 @@ -662,7 +662,7 @@ class TestWebsocketSaltAPIHandler(SaltnadoTestCase): method='POST', body=urlencode(self.auth_creds), headers={'Content-Type': self.content_type_map['form']}) - token = json.loads(self.decode_body(response).body)['return'][0]['token'] + token = salt.utils.json.loads(self.decode_body(response).body)['return'][0]['token'] url = 'ws://127.0.0.1:{0}/all_events/{1}'.format(self.get_http_port(), token) request = HTTPRequest(url, headers={'Origin': 'http://example.com', @@ -694,7 +694,7 @@ class TestWebsocketSaltAPIHandler(SaltnadoTestCase): method='POST', body=urlencode(self.auth_creds), headers={'Content-Type': self.content_type_map['form']}) - token = json.loads(self.decode_body(response).body)['return'][0]['token'] + token = salt.utils.json.loads(self.decode_body(response).body)['return'][0]['token'] url = 'ws://127.0.0.1:{0}/all_events/{1}'.format(self.get_http_port(), token) request = HTTPRequest(url, headers={'Origin': 'http://foo.bar', @@ -711,7 +711,7 @@ class TestWebsocketSaltAPIHandler(SaltnadoTestCase): method='POST', body=urlencode(self.auth_creds), headers={'Content-Type': self.content_type_map['form']}) - token = json.loads(self.decode_body(response).body)['return'][0]['token'] + token = salt.utils.json.loads(self.decode_body(response).body)['return'][0]['token'] url = 'ws://127.0.0.1:{0}/all_events/{1}'.format(self.get_http_port(), token) # Example.com should works @@ -737,7 +737,7 @@ class TestWebsocketSaltAPIHandler(SaltnadoTestCase): method='POST', body=urlencode(self.auth_creds), headers={'Content-Type': self.content_type_map['form']}) - token = json.loads(self.decode_body(response).body)['return'][0]['token'] + token = salt.utils.json.loads(self.decode_body(response).body)['return'][0]['token'] url = 'ws://127.0.0.1:{0}/all_events/{1}'.format(self.get_http_port(), token) # Example.com should works diff --git a/tests/unit/output/test_json_out.py b/tests/unit/output/test_json_out.py index 0e0be5f4fae..9b218c9240c 100644 --- a/tests/unit/output/test_json_out.py +++ b/tests/unit/output/test_json_out.py @@ -4,8 +4,7 @@ unittests for json outputter ''' # Import Python Libs -from __future__ import absolute_import -import json +from __future__ import absolute_import, print_function, unicode_literals # Import Salt Testing Libs from tests.support.mixins import LoaderModuleMockMixin @@ -14,8 +13,7 @@ from tests.support.mock import patch # Import Salt Libs import salt.output.json_out as json_out - -# Import 3rd-party libs +import salt.utils.stringutils from salt.ext import six @@ -61,11 +59,13 @@ class JsonTestCase(TestCase, LoaderModuleMockMixin): def test_unicode_output(self): with patch.dict(json_out.__opts__, {'output_indent': 'pretty'}): - data = {'test': '\xe1', 'example': 'one'} - expect = ('{"message": "\'utf8\' codec can\'t decode byte 0xe1 in position 0: unexpected end of data", ' - '"error": "Unable to serialize output to json"}') - ret = json_out.output(data) + decoded = {'test': 'Д', 'example': 'one'} + encoded = {'test': salt.utils.stringutils.to_str('Д'), 'example': 'one'} + # json.dumps on Python 2 adds a space before a newline while in the + # process of dumping a dictionary. if six.PY2: - self.assertEqual(expect, ret) + expected = salt.utils.stringutils.to_str('{\n "example": "one", \n "test": "Д"\n}') else: - self.assertEqual(json.loads(ret), data) + expected = '{\n "example": "one",\n "test": "Д"\n}' + self.assertEqual(json_out.output(decoded), expected) + self.assertEqual(json_out.output(encoded), expected) diff --git a/tests/unit/states/test_boto_lambda.py b/tests/unit/states/test_boto_lambda.py index eeac382640a..37f49c4fabf 100644 --- a/tests/unit/states/test_boto_lambda.py +++ b/tests/unit/states/test_boto_lambda.py @@ -2,7 +2,6 @@ # Import Python libs from __future__ import absolute_import -import json import logging import random import string @@ -15,6 +14,7 @@ from tests.support.mock import MagicMock, NO_MOCK, NO_MOCK_REASON, patch # Import Salt libs import salt.config import salt.loader +import salt.utils.json from salt.utils.versions import LooseVersion import salt.states.boto_lambda as boto_lambda @@ -255,7 +255,7 @@ class BotoLambdaFunctionTestCase(BotoLambdaStateTestCaseBase, BotoLambdaTestCase self.conn.list_functions.return_value = {'Functions': [function_ret]} self.conn.update_function_code.return_value = function_ret self.conn.get_policy.return_value = { - "Policy": json.dumps( + "Policy": salt.utils.json.dumps( {"Version": "2012-10-17", "Statement": [ {"Condition": diff --git a/tests/unit/states/test_file.py b/tests/unit/states/test_file.py index e4a433beb7f..bc16898c007 100644 --- a/tests/unit/states/test_file.py +++ b/tests/unit/states/test_file.py @@ -4,7 +4,6 @@ from __future__ import absolute_import, print_function, unicode_literals from datetime import datetime import os -import json import pprint import shutil @@ -32,6 +31,7 @@ import yaml # Import salt libs import salt.utils.files +import salt.utils.json import salt.utils.platform import salt.states.file as filestate import salt.serializers.yaml as yamlserializer @@ -81,7 +81,7 @@ class TestFileState(TestCase, LoaderModuleMockMixin): self.assertEqual(yaml.load(returner.returned), dataset) filestate.serialize('/tmp', dataset, formatter="json") - self.assertEqual(json.loads(returner.returned), dataset) + self.assertEqual(salt.utils.json.loads(returner.returned), dataset) filestate.serialize('/tmp', dataset, formatter="python") self.assertEqual(returner.returned, pprint.pformat(dataset) + '\n') diff --git a/tests/unit/states/test_grafana.py b/tests/unit/states/test_grafana.py index 4718fddd001..a139c7714bd 100644 --- a/tests/unit/states/test_grafana.py +++ b/tests/unit/states/test_grafana.py @@ -4,7 +4,6 @@ ''' # Import Python libs from __future__ import absolute_import -import json # Import Salt Testing Libs from tests.support.mixins import LoaderModuleMockMixin @@ -17,6 +16,7 @@ from tests.support.mock import ( ) # Import Salt Libs +import salt.utils.json import salt.states.grafana as grafana from salt.exceptions import SaltInvocationError @@ -83,13 +83,13 @@ class GrafanaTestCase(TestCase, LoaderModuleMockMixin): mock = MagicMock(return_value={'rows': [{'panels': 'b', 'title': 'systemhealth'}]}) - with patch.object(json, 'loads', mock): + with patch.object(salt.utils.json, 'loads', mock): ret.update({'comment': comt1, 'result': None}) self.assertDictEqual(grafana.dashboard_present(name, True, rows=row), ret) - with patch.object(json, 'loads', + with patch.object(salt.utils.json, 'loads', MagicMock(return_value={'rows': {}})): self.assertRaises(SaltInvocationError, grafana.dashboard_present, name, @@ -102,7 +102,7 @@ class GrafanaTestCase(TestCase, LoaderModuleMockMixin): mock = MagicMock(return_value={'rows': [{'panels': 'b', 'title': 'systemhealth'}]}) with patch.dict(grafana.__opts__, {'test': False}): - with patch.object(json, 'loads', mock): + with patch.object(salt.utils.json, 'loads', mock): comt = ('Failed to update dashboard myservice.') ret.update({'comment': comt, 'result': False}) self.assertDictEqual(grafana.dashboard_present(name, True, diff --git a/tests/unit/templates/test_jinja.py b/tests/unit/templates/test_jinja.py index d8049b96ee7..1b6914802a7 100644 --- a/tests/unit/templates/test_jinja.py +++ b/tests/unit/templates/test_jinja.py @@ -6,7 +6,6 @@ from jinja2 import Environment, DictLoader, exceptions import ast import copy import datetime -import json import os import pprint import re @@ -27,6 +26,7 @@ from salt.exceptions import SaltRenderError from salt.ext import six from salt.ext.six.moves import builtins +import salt.utils.json from salt.utils.decorators.jinja import JinjaFilter from salt.utils.jinja import ( SaltCacheLoader, @@ -625,7 +625,7 @@ class TestCustomExtensions(TestCase): } env = Environment(extensions=[SerializerExtension]) rendered = env.from_string('{{ dataset|json }}').render(dataset=dataset) - self.assertEqual(dataset, json.loads(rendered)) + self.assertEqual(dataset, salt.utils.json.loads(rendered)) def test_serialize_yaml(self): dataset = { diff --git a/tests/unit/test_context.py b/tests/unit/test_context.py index 843c4ecb8ca..780744118bc 100644 --- a/tests/unit/test_context.py +++ b/tests/unit/test_context.py @@ -5,7 +5,6 @@ ''' # Import python libs from __future__ import absolute_import -import json import tornado.stack_context import tornado.gen from tornado.testing import AsyncTestCase, gen_test @@ -17,6 +16,7 @@ from tests.support.unit import TestCase from salt.ext.six.moves import range # Import Salt libs +import salt.utils.json from salt.utils.context import ContextDict, NamespacedDictWrapper @@ -188,9 +188,9 @@ class NamespacedDictWrapperTests(TestCase): def test_json_dumps_single_key(self): self._dict['prefix'] = {'foo': {'bar': 'baz'}} w = NamespacedDictWrapper(self._dict, 'prefix') - self.assertEqual(json.dumps(w), '{"foo": {"bar": "baz"}}') + self.assertEqual(salt.utils.json.dumps(w), '{"foo": {"bar": "baz"}}') def test_json_dumps_multiple_key(self): self._dict['prefix'] = {'foo': {'bar': 'baz'}} w = NamespacedDictWrapper(self._dict, ('prefix', 'foo')) - self.assertEqual(json.dumps(w), '{"bar": "baz"}') + self.assertEqual(salt.utils.json.dumps(w), '{"bar": "baz"}') diff --git a/tests/unit/utils/test_schema.py b/tests/unit/utils/test_schema.py index edf8841b6b6..dd5500f5fd3 100644 --- a/tests/unit/utils/test_schema.py +++ b/tests/unit/utils/test_schema.py @@ -6,13 +6,13 @@ # Import python libs from __future__ import absolute_import import copy -import json import yaml # Import Salt Testing Libs from tests.support.unit import TestCase, skipIf # Import Salt Libs +import salt.utils.json import salt.utils.schema as schema from salt.utils.versions import LooseVersion as _LooseVersion @@ -2112,7 +2112,7 @@ class ComplexSchemaTestCase(TestCase): self.assertDictEqual(complex_obj.get_definition(), expected_def) def test_complex_definition_schema(self): - serialized = yaml.safe_load(json.dumps(self.schema.serialize())) + serialized = yaml.safe_load(salt.utils.json.dumps(self.schema.serialize())) expected = { '$schema': 'http://json-schema.org/draft-04/schema#', 'title': 'Test Complex Definition Schema', @@ -2134,7 +2134,7 @@ class ComplexSchemaTestCase(TestCase): self.assertDictEqual(serialized, expected) def test_one_of_complex_definition_schema(self): - serialized = yaml.safe_load(json.dumps(self.one_of_schema.serialize())) + serialized = yaml.safe_load(salt.utils.json.dumps(self.one_of_schema.serialize())) expected = { '$schema': 'http://json-schema.org/draft-04/schema#', 'title': 'Test OneOf Complex Definitions Schema', @@ -2157,7 +2157,7 @@ class ComplexSchemaTestCase(TestCase): self.assertDictEqual(serialized, expected) def test_array_complex_definition_schema(self): - serialized = yaml.safe_load(json.dumps(self.array_schema.serialize())) + serialized = yaml.safe_load(salt.utils.json.dumps(self.array_schema.serialize())) expected = { '$schema': 'http://json-schema.org/draft-04/schema#', 'title': 'Test Array Complex Definitions Schema', @@ -2181,7 +2181,7 @@ class ComplexSchemaTestCase(TestCase): self.assertDictEqual(serialized, expected) def test_dict_complex_definition_schema(self): - serialized = yaml.safe_load(json.dumps(self.dict_schema.serialize())) + serialized = yaml.safe_load(salt.utils.json.dumps(self.dict_schema.serialize())) expected = { '$schema': 'http://json-schema.org/draft-04/schema#', 'title': 'Test Dict Complex Definitions Schema', @@ -2210,7 +2210,7 @@ class ComplexSchemaTestCase(TestCase): self.assertDictEqual(serialized, expected) def test_complex_complex_definition_schema(self): - serialized = yaml.safe_load(json.dumps( + serialized = yaml.safe_load(salt.utils.json.dumps( self.complex_schema.serialize())) expected = { '$schema': 'http://json-schema.org/draft-04/schema#',