mirror of
https://github.com/saltstack/salt.git
synced 2025-04-17 10:10:20 +00:00
salt.utils.hashutils: Fix UnicodeEncodeError in several funcs
Many funcs in this module only convert the input to bytes on Python 3 because the hashing and base64-encoding funcs used in this module require bytes as input. However, on Python 2, to get bytes the input will be coerced to a str type if it is a unicode type. This will result in a UnicodeEncodeError when the unicode string contains non-ascii characters. This commit ensures that all input which is going to be encoded/hashed is converted to a bytes type first, and also ensures that all base64-encoded results are returned as a unicode string on Python 2 to conform with changes that have been made in 2018.3 to use unicode types where possible.
This commit is contained in:
Erik Johnson
parent
3658604c43
commit
7266c9984d
2 changed files with 181 additions and 49 deletions
|
|
@ -26,11 +26,9 @@ def base64_b64encode(instr):
|
|||
Among other possible differences, the "modern" encoder does not include
|
||||
newline ('\\n') characters in the encoded output.
|
||||
'''
|
||||
if six.PY3:
|
||||
b = salt.utils.stringutils.to_bytes(instr)
|
||||
b64 = base64.b64encode(b)
|
||||
return salt.utils.stringutils.to_str(b64)
|
||||
return base64.b64encode(instr)
|
||||
return salt.utils.stringutils.to_unicode(
|
||||
base64.b64encode(salt.utils.stringutils.to_bytes(instr))
|
||||
)
|
||||
|
||||
|
||||
@jinja_filter('base64_decode')
|
||||
|
|
@ -38,14 +36,11 @@ def base64_b64decode(instr):
|
|||
'''
|
||||
Decode a base64-encoded string using the "modern" Python interface.
|
||||
'''
|
||||
if six.PY3:
|
||||
b = salt.utils.stringutils.to_bytes(instr)
|
||||
data = base64.b64decode(b)
|
||||
try:
|
||||
return salt.utils.stringutils.to_str(data)
|
||||
except UnicodeDecodeError:
|
||||
return data
|
||||
return base64.b64decode(instr)
|
||||
decoded = base64.b64decode(salt.utils.stringutils.to_bytes(instr))
|
||||
try:
|
||||
return salt.utils.stringutils.to_unicode(decoded)
|
||||
except UnicodeDecodeError:
|
||||
return decoded
|
||||
|
||||
|
||||
def base64_encodestring(instr):
|
||||
|
|
@ -56,26 +51,26 @@ def base64_encodestring(instr):
|
|||
a newline ('\\n') character after every 76 characters and always
|
||||
at the end of the encoded string.
|
||||
'''
|
||||
if six.PY3:
|
||||
b = salt.utils.stringutils.to_bytes(instr)
|
||||
b64 = base64.encodebytes(b)
|
||||
return salt.utils.stringutils.to_str(b64)
|
||||
return base64.encodestring(instr)
|
||||
return salt.utils.stringutils.to_unicode(
|
||||
base64.encodestring(salt.utils.stringutils.to_bytes(instr))
|
||||
)
|
||||
|
||||
|
||||
def base64_decodestring(instr):
|
||||
'''
|
||||
Decode a base64-encoded string using the "legacy" Python interface.
|
||||
|
||||
'''
|
||||
if six.PY3:
|
||||
b = salt.utils.stringutils.to_bytes(instr)
|
||||
data = base64.decodebytes(b)
|
||||
try:
|
||||
return salt.utils.stringutils.to_str(data)
|
||||
except UnicodeDecodeError:
|
||||
return data
|
||||
return base64.decodestring(instr)
|
||||
b = salt.utils.stringutils.to_bytes(instr)
|
||||
try:
|
||||
# PY3
|
||||
decoded = base64.decodebytes(b)
|
||||
except AttributeError:
|
||||
# PY2
|
||||
decoded = base64.decodestring(b)
|
||||
try:
|
||||
return salt.utils.stringutils.to_unicode(decoded)
|
||||
except UnicodeDecodeError:
|
||||
return decoded
|
||||
|
||||
|
||||
@jinja_filter('md5')
|
||||
|
|
@ -83,32 +78,29 @@ def md5_digest(instr):
|
|||
'''
|
||||
Generate an md5 hash of a given string.
|
||||
'''
|
||||
if six.PY3:
|
||||
b = salt.utils.stringutils.to_bytes(instr)
|
||||
return hashlib.md5(b).hexdigest()
|
||||
return hashlib.md5(instr).hexdigest()
|
||||
return salt.utils.stringutils.to_unicode(
|
||||
hashlib.md5(salt.utils.stringutils.to_bytes(instr)).hexdigest()
|
||||
)
|
||||
|
||||
|
||||
@jinja_filter('sha256')
|
||||
def sha256_digest(instr):
|
||||
'''
|
||||
Generate an sha256 hash of a given string.
|
||||
Generate a sha256 hash of a given string.
|
||||
'''
|
||||
if six.PY3:
|
||||
b = salt.utils.stringutils.to_bytes(instr)
|
||||
return hashlib.sha256(b).hexdigest()
|
||||
return hashlib.sha256(instr).hexdigest()
|
||||
return salt.utils.stringutils.to_unicode(
|
||||
hashlib.sha256(salt.utils.stringutils.to_bytes(instr)).hexdigest()
|
||||
)
|
||||
|
||||
|
||||
@jinja_filter('sha512')
|
||||
def sha512_digest(instr):
|
||||
'''
|
||||
Generate an sha512 hash of a given string
|
||||
Generate a sha512 hash of a given string
|
||||
'''
|
||||
if six.PY3:
|
||||
b = salt.utils.stringutils.to_bytes(instr)
|
||||
return hashlib.sha512(b).hexdigest()
|
||||
return hashlib.sha512(instr).hexdigest()
|
||||
return salt.utils.stringutils.to_unicode(
|
||||
hashlib.sha512(salt.utils.stringutils.to_bytes(instr)).hexdigest()
|
||||
)
|
||||
|
||||
|
||||
@jinja_filter('hmac')
|
||||
|
|
@ -117,14 +109,9 @@ def hmac_signature(string, shared_secret, challenge_hmac):
|
|||
Verify a challenging hmac signature against a string / shared-secret
|
||||
Returns a boolean if the verification succeeded or failed.
|
||||
'''
|
||||
if six.text_type:
|
||||
msg = salt.utils.stringutils.to_bytes(string)
|
||||
key = salt.utils.stringutils.to_bytes(shared_secret)
|
||||
challenge = salt.utils.stringutils.to_bytes(challenge_hmac)
|
||||
else:
|
||||
msg = string
|
||||
key = shared_secret
|
||||
challenge = challenge_hmac
|
||||
msg = salt.utils.stringutils.to_bytes(string)
|
||||
key = salt.utils.stringutils.to_bytes(shared_secret)
|
||||
challenge = salt.utils.stringutils.to_bytes(challenge_hmac)
|
||||
hmac_hash = hmac.new(key, msg, hashlib.sha256)
|
||||
valid_hmac = base64.b64encode(hmac_hash.digest())
|
||||
return valid_hmac == challenge
|
||||
|
|
|
|||
|
|
@ -12,6 +12,151 @@ import salt.utils.hashutils
|
|||
|
||||
class HashutilsTestCase(TestCase):
|
||||
|
||||
hmac_secret = 's00p3r s3kr1t'
|
||||
|
||||
# Use a non-ascii unicode type to confirm that no UnicodeEncodeError is
|
||||
# raised on Python 2.
|
||||
str = 'спам'
|
||||
str_b64encode_result = '0YHQv9Cw0Lw='
|
||||
str_encodestring_result = '0YHQv9Cw0Lw=\n'
|
||||
str_md5 = 'a035ac08ab2f03556f9b3ee640052e5c'
|
||||
str_sha256 = '095291ffa3d361436d4617879e22c1da06c6ab61a3fb081321ec854a27a091ac'
|
||||
str_sha512 = '12efd90e507289f1f21e5dcfe2e92cf0bb4904abccb55c3ce9177670c711981501054b32b807c37058675590d1c484bd2b72a4215a2fa397aa4f2b12f298b1f0'
|
||||
str_hmac_challenge = b'qz2k0t1aevKEme3JGsNQJX/xpmf+/w3q6qmWDk1ZqbY='
|
||||
|
||||
# 16 bytes of random data
|
||||
bytes = b'b\x19\xf6\x86\x0e\x1a\x1cs\x0c\xda&zv\xfc\xa2\xdd'
|
||||
bytes_b64encode_result = 'Yhn2hg4aHHMM2iZ6dvyi3Q=='
|
||||
bytes_encodestring_result = 'Yhn2hg4aHHMM2iZ6dvyi3Q==\n'
|
||||
bytes_md5 = '4d064241724791641dc15930c65f75c8'
|
||||
bytes_sha256 = '25711a31c2673a48f3d1f29b25add574697872968e546d266f441de63b17954a'
|
||||
bytes_sha512 = '69f1524e602c1599fc374e1e3e2941e6f6949f4f7fe7321304e4e67bb850f3204dd5cbf9c13e231814540c2f5cd370c24ea257771d9fbf311d8f6085bad12b24'
|
||||
bytes_hmac_challenge = b'lQibiD9r1Hpo+5JYknaudIKfTx1L5J3U58M9yQOd04c='
|
||||
|
||||
def test_base64_b64encode(self):
|
||||
'''
|
||||
Ensure that this function converts the value passed to bytes before
|
||||
attempting to encode, avoiding a UnicodeEncodeError on Python 2 and a
|
||||
TypeError on Python 3.
|
||||
'''
|
||||
self.assertEqual(
|
||||
salt.utils.hashutils.base64_b64encode(self.str),
|
||||
self.str_b64encode_result
|
||||
)
|
||||
self.assertEqual(
|
||||
salt.utils.hashutils.base64_b64encode(self.bytes),
|
||||
self.bytes_b64encode_result
|
||||
)
|
||||
|
||||
def test_base64_b64decode(self):
|
||||
'''
|
||||
Ensure that this function converts the value passed to a unicode type
|
||||
(if possible) on Python 2, and a str type (if possible) on Python 3.
|
||||
'''
|
||||
self.assertEqual(
|
||||
salt.utils.hashutils.base64_b64decode(self.str_b64encode_result),
|
||||
self.str
|
||||
)
|
||||
self.assertEqual(
|
||||
salt.utils.hashutils.base64_b64decode(self.bytes_b64encode_result),
|
||||
self.bytes
|
||||
)
|
||||
|
||||
def test_base64_encodestring(self):
|
||||
'''
|
||||
Ensure that this function converts the value passed to bytes before
|
||||
attempting to encode, avoiding a UnicodeEncodeError on Python 2 and a
|
||||
TypeError on Python 3.
|
||||
'''
|
||||
self.assertEqual(
|
||||
salt.utils.hashutils.base64_encodestring(self.str),
|
||||
self.str_encodestring_result
|
||||
)
|
||||
self.assertEqual(
|
||||
salt.utils.hashutils.base64_encodestring(self.bytes),
|
||||
self.bytes_encodestring_result
|
||||
)
|
||||
|
||||
def test_base64_decodestring(self):
|
||||
'''
|
||||
Ensure that this function converts the value passed to a unicode type
|
||||
(if possible) on Python 2, and a str type (if possible) on Python 3.
|
||||
'''
|
||||
self.assertEqual(
|
||||
salt.utils.hashutils.base64_decodestring(self.str_encodestring_result),
|
||||
self.str
|
||||
)
|
||||
self.assertEqual(
|
||||
salt.utils.hashutils.base64_decodestring(self.bytes_encodestring_result),
|
||||
self.bytes
|
||||
)
|
||||
|
||||
def test_md5_digest(self):
|
||||
'''
|
||||
Ensure that this function converts the value passed to bytes before
|
||||
attempting to encode, avoiding a UnicodeEncodeError on Python 2 and a
|
||||
TypeError on Python 3.
|
||||
'''
|
||||
self.assertEqual(
|
||||
salt.utils.hashutils.md5_digest(self.str),
|
||||
self.str_md5
|
||||
)
|
||||
self.assertEqual(
|
||||
salt.utils.hashutils.md5_digest(self.bytes),
|
||||
self.bytes_md5
|
||||
)
|
||||
|
||||
def test_sha256_digest(self):
|
||||
'''
|
||||
Ensure that this function converts the value passed to bytes before
|
||||
attempting to encode, avoiding a UnicodeEncodeError on Python 2 and a
|
||||
TypeError on Python 3.
|
||||
'''
|
||||
self.assertEqual(
|
||||
salt.utils.hashutils.sha256_digest(self.str),
|
||||
self.str_sha256
|
||||
)
|
||||
self.assertEqual(
|
||||
salt.utils.hashutils.sha256_digest(self.bytes),
|
||||
self.bytes_sha256
|
||||
)
|
||||
|
||||
def test_sha512_digest(self):
|
||||
'''
|
||||
Ensure that this function converts the value passed to bytes before
|
||||
attempting to encode, avoiding a UnicodeEncodeError on Python 2 and a
|
||||
TypeError on Python 3.
|
||||
'''
|
||||
self.assertEqual(
|
||||
salt.utils.hashutils.sha512_digest(self.str),
|
||||
self.str_sha512
|
||||
)
|
||||
self.assertEqual(
|
||||
salt.utils.hashutils.sha512_digest(self.bytes),
|
||||
self.bytes_sha512
|
||||
)
|
||||
|
||||
def test_hmac_signature(self):
|
||||
'''
|
||||
Ensure that this function converts the value passed to bytes before
|
||||
attempting to validate the hmac challenge, avoiding a
|
||||
UnicodeEncodeError on Python 2 and a TypeError on Python 3.
|
||||
'''
|
||||
self.assertTrue(
|
||||
salt.utils.hashutils.hmac_signature(
|
||||
self.str,
|
||||
self.hmac_secret,
|
||||
self.str_hmac_challenge
|
||||
)
|
||||
)
|
||||
self.assertTrue(
|
||||
salt.utils.hashutils.hmac_signature(
|
||||
self.bytes,
|
||||
self.hmac_secret,
|
||||
self.bytes_hmac_challenge
|
||||
)
|
||||
)
|
||||
|
||||
def test_get_hash_exception(self):
|
||||
self.assertRaises(
|
||||
ValueError,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue