saltstack/salt
1
0
Fork 0
mirror of https://github.com/saltstack/salt.git synced 2025-04-17 10:10:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Merge pull request #46873 from terminalmage/utf8-first

Browse source 
Attempt UTF-8 first when decoding/encoding
This commit is contained in:
Mike Place 2018-04-05 09:16:27 -05:00 • committed by GitHub
commit 4e5e291c99
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 174 additions and 728 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
salt/modules/cmdmod.py
View file

@ -610,7 +610,7 @@ def _run(cmd,
ret['retcode'] = 1
return ret
if output_encoding is not None:
if output_loglevel != 'quiet' and output_encoding is not None:
log.debug('Decoding output from command %s using %s encoding',
cmd, output_encoding)
@ -626,10 +626,11 @@ def _run(cmd,
proc.stdout,
encoding=output_encoding,
errors='replace')
log.error(
'Failed to decode stdout from command %s, non-decodable '
'characters have been replaced', cmd
)
if output_loglevel != 'quiet':
log.error(
'Failed to decode stdout from command %s, non-decodable '
'characters have been replaced', cmd
)
try:
err = salt.utils.stringutils.to_unicode(
@ -643,10 +644,11 @@ def _run(cmd,
proc.stderr,
encoding=output_encoding,
errors='replace')
log.error(
'Failed to decode stderr from command %s, non-decodable '
'characters have been replaced', cmd
)
if output_loglevel != 'quiet':
log.error(
'Failed to decode stderr from command %s, non-decodable '
'characters have been replaced', cmd
)
if rstrip:
if out is not None:

741
salt/modules/git.py
View file

@ -221,9 +221,6 @@ def _git_run(command, cwd=None, user=None, password=None, identity=None,
'cmd.run_all', and used as an alternative to 'cmd.run_all'. Some
commands don't return proper retcodes, so this can't replace 'cmd.run_all'.
'''
if salt.utils.platform.is_windows() and output_encoding is None:
output_encoding = 'utf-8'
env = {}
if identity:
@ -516,21 +513,9 @@ def add(cwd,
cases.
.. note::
On Windows, this option works slightly differently in the git state
and execution module than it does in the :mod:`"cmd" execution
module <salt.modules.cmdmod>`. The filenames in most git
repositories are created using a UTF-8 locale, and the system
encoding on Windows (CP1252) will successfully (but incorrectly)
decode many UTF-8 characters. This makes interacting with
repositories containing UTF-8 filenames on Windows unreliable.
Therefore, Windows will default to decoding the output from git
commands using UTF-8 unless this option is explicitly used to
specify the encoding.
On non-Windows platforms, the default output decoding behavior will
be observed (i.e. the encoding specified by the locale will be
tried first, and if that fails, UTF-8 will be used as a fallback).
This should only be needed if the files in the repository were
created with filenames using an encoding other than UTF-8 to handle
Unicode characters.
.. versionadded:: 2018.3.1
@ -664,21 +649,9 @@ def archive(cwd,
cases.
.. note::
On Windows, this option works slightly differently in the git state
and execution module than it does in the :mod:`"cmd" execution
module <salt.modules.cmdmod>`. The filenames in most git
repositories are created using a UTF-8 locale, and the system
encoding on Windows (CP1252) will successfully (but incorrectly)
decode many UTF-8 characters. This makes interacting with
repositories containing UTF-8 filenames on Windows unreliable.
Therefore, Windows will default to decoding the output from git
commands using UTF-8 unless this option is explicitly used to
specify the encoding.
On non-Windows platforms, the default output decoding behavior will
be observed (i.e. the encoding specified by the locale will be
tried first, and if that fails, UTF-8 will be used as a fallback).
This should only be needed if the files in the repository were
created with filenames using an encoding other than UTF-8 to handle
Unicode characters.
.. versionadded:: 2018.3.1
@ -790,21 +763,9 @@ def branch(cwd,
cases.
.. note::
On Windows, this option works slightly differently in the git state
and execution module than it does in the :mod:`"cmd" execution
module <salt.modules.cmdmod>`. The filenames in most git
repositories are created using a UTF-8 locale, and the system
encoding on Windows (CP1252) will successfully (but incorrectly)
decode many UTF-8 characters. This makes interacting with
repositories containing UTF-8 filenames on Windows unreliable.
Therefore, Windows will default to decoding the output from git
commands using UTF-8 unless this option is explicitly used to
specify the encoding.
On non-Windows platforms, the default output decoding behavior will
be observed (i.e. the encoding specified by the locale will be
tried first, and if that fails, UTF-8 will be used as a fallback).
This should only be needed if the files in the repository were
created with filenames using an encoding other than UTF-8 to handle
Unicode characters.
.. versionadded:: 2018.3.1
@ -903,21 +864,9 @@ def checkout(cwd,
cases.
.. note::
On Windows, this option works slightly differently in the git state
and execution module than it does in the :mod:`"cmd" execution
module <salt.modules.cmdmod>`. The filenames in most git
repositories are created using a UTF-8 locale, and the system
encoding on Windows (CP1252) will successfully (but incorrectly)
decode many UTF-8 characters. This makes interacting with
repositories containing UTF-8 filenames on Windows unreliable.
Therefore, Windows will default to decoding the output from git
commands using UTF-8 unless this option is explicitly used to
specify the encoding.
On non-Windows platforms, the default output decoding behavior will
be observed (i.e. the encoding specified by the locale will be
tried first, and if that fails, UTF-8 will be used as a fallback).
This should only be needed if the files in the repository were
created with filenames using an encoding other than UTF-8 to handle
Unicode characters.
.. versionadded:: 2018.3.1
@ -1069,21 +1018,9 @@ def clone(cwd,
cases.
.. note::
On Windows, this option works slightly differently in the git state
and execution module than it does in the :mod:`"cmd" execution
module <salt.modules.cmdmod>`. The filenames in most git
repositories are created using a UTF-8 locale, and the system
encoding on Windows (CP1252) will successfully (but incorrectly)
decode many UTF-8 characters. This makes interacting with
repositories containing UTF-8 filenames on Windows unreliable.
Therefore, Windows will default to decoding the output from git
commands using UTF-8 unless this option is explicitly used to
specify the encoding.
On non-Windows platforms, the default output decoding behavior will
be observed (i.e. the encoding specified by the locale will be
tried first, and if that fails, UTF-8 will be used as a fallback).
This should only be needed if the files in the repository were
created with filenames using an encoding other than UTF-8 to handle
Unicode characters.
.. versionadded:: 2018.3.1
@ -1210,21 +1147,9 @@ def commit(cwd,
cases.
.. note::
On Windows, this option works slightly differently in the git state
and execution module than it does in the :mod:`"cmd" execution
module <salt.modules.cmdmod>`. The filenames in most git
repositories are created using a UTF-8 locale, and the system
encoding on Windows (CP1252) will successfully (but incorrectly)
decode many UTF-8 characters. This makes interacting with
repositories containing UTF-8 filenames on Windows unreliable.
Therefore, Windows will default to decoding the output from git
commands using UTF-8 unless this option is explicitly used to
specify the encoding.
On non-Windows platforms, the default output decoding behavior will
be observed (i.e. the encoding specified by the locale will be
tried first, and if that fails, UTF-8 will be used as a fallback).
This should only be needed if the files in the repository were
created with filenames using an encoding other than UTF-8 to handle
Unicode characters.
.. versionadded:: 2018.3.1
@ -1309,21 +1234,9 @@ def config_get(key,
cases.
.. note::
On Windows, this option works slightly differently in the git state
and execution module than it does in the :mod:`"cmd" execution
module <salt.modules.cmdmod>`. The filenames in most git
repositories are created using a UTF-8 locale, and the system
encoding on Windows (CP1252) will successfully (but incorrectly)
decode many UTF-8 characters. This makes interacting with
repositories containing UTF-8 filenames on Windows unreliable.
Therefore, Windows will default to decoding the output from git
commands using UTF-8 unless this option is explicitly used to
specify the encoding.
On non-Windows platforms, the default output decoding behavior will
be observed (i.e. the encoding specified by the locale will be
tried first, and if that fails, UTF-8 will be used as a fallback).
This should only be needed if the files in the repository were
created with filenames using an encoding other than UTF-8 to handle
Unicode characters.
.. versionadded:: 2018.3.1
@ -1420,21 +1333,9 @@ def config_get_regexp(key,
cases.
.. note::
On Windows, this option works slightly differently in the git state
and execution module than it does in the :mod:`"cmd" execution
module <salt.modules.cmdmod>`. The filenames in most git
repositories are created using a UTF-8 locale, and the system
encoding on Windows (CP1252) will successfully (but incorrectly)
decode many UTF-8 characters. This makes interacting with
repositories containing UTF-8 filenames on Windows unreliable.
Therefore, Windows will default to decoding the output from git
commands using UTF-8 unless this option is explicitly used to
specify the encoding.
On non-Windows platforms, the default output decoding behavior will
be observed (i.e. the encoding specified by the locale will be
tried first, and if that fails, UTF-8 will be used as a fallback).
This should only be needed if the files in the repository were
created with filenames using an encoding other than UTF-8 to handle
Unicode characters.
.. versionadded:: 2018.3.1
@ -1545,21 +1446,9 @@ def config_set(key,
cases.
.. note::
On Windows, this option works slightly differently in the git state
and execution module than it does in the :mod:`"cmd" execution
module <salt.modules.cmdmod>`. The filenames in most git
repositories are created using a UTF-8 locale, and the system
encoding on Windows (CP1252) will successfully (but incorrectly)
decode many UTF-8 characters. This makes interacting with
repositories containing UTF-8 filenames on Windows unreliable.
Therefore, Windows will default to decoding the output from git
commands using UTF-8 unless this option is explicitly used to
specify the encoding.
On non-Windows platforms, the default output decoding behavior will
be observed (i.e. the encoding specified by the locale will be
tried first, and if that fails, UTF-8 will be used as a fallback).
This should only be needed if the files in the repository were
created with filenames using an encoding other than UTF-8 to handle
Unicode characters.
.. versionadded:: 2018.3.1
@ -1696,21 +1585,9 @@ def config_unset(key,
cases.
.. note::
On Windows, this option works slightly differently in the git state
and execution module than it does in the :mod:`"cmd" execution
module <salt.modules.cmdmod>`. The filenames in most git
repositories are created using a UTF-8 locale, and the system
encoding on Windows (CP1252) will successfully (but incorrectly)
decode many UTF-8 characters. This makes interacting with
repositories containing UTF-8 filenames on Windows unreliable.
Therefore, Windows will default to decoding the output from git
commands using UTF-8 unless this option is explicitly used to
specify the encoding.
On non-Windows platforms, the default output decoding behavior will
be observed (i.e. the encoding specified by the locale will be
tried first, and if that fails, UTF-8 will be used as a fallback).
This should only be needed if the files in the repository were
created with filenames using an encoding other than UTF-8 to handle
Unicode characters.
.. versionadded:: 2018.3.1
@ -1819,21 +1696,9 @@ def current_branch(cwd,
cases.
.. note::
On Windows, this option works slightly differently in the git state
and execution module than it does in the :mod:`"cmd" execution
module <salt.modules.cmdmod>`. The filenames in most git
repositories are created using a UTF-8 locale, and the system
encoding on Windows (CP1252) will successfully (but incorrectly)
decode many UTF-8 characters. This makes interacting with
repositories containing UTF-8 filenames on Windows unreliable.
Therefore, Windows will default to decoding the output from git
commands using UTF-8 unless this option is explicitly used to
specify the encoding.
On non-Windows platforms, the default output decoding behavior will
be observed (i.e. the encoding specified by the locale will be
tried first, and if that fails, UTF-8 will be used as a fallback).
This should only be needed if the files in the repository were
created with filenames using an encoding other than UTF-8 to handle
Unicode characters.
.. versionadded:: 2018.3.1
@ -1891,21 +1756,9 @@ def describe(cwd,
cases.
.. note::
On Windows, this option works slightly differently in the git state
and execution module than it does in the :mod:`"cmd" execution
module <salt.modules.cmdmod>`. The filenames in most git
repositories are created using a UTF-8 locale, and the system
encoding on Windows (CP1252) will successfully (but incorrectly)
decode many UTF-8 characters. This makes interacting with
repositories containing UTF-8 filenames on Windows unreliable.
Therefore, Windows will default to decoding the output from git
commands using UTF-8 unless this option is explicitly used to
specify the encoding.
On non-Windows platforms, the default output decoding behavior will
be observed (i.e. the encoding specified by the locale will be
tried first, and if that fails, UTF-8 will be used as a fallback).
This should only be needed if the files in the repository were
created with filenames using an encoding other than UTF-8 to handle
Unicode characters.
.. versionadded:: 2018.3.1
@ -2015,21 +1868,9 @@ def diff(cwd,
cases.
.. note::
On Windows, this option works slightly differently in the git state
and execution module than it does in the :mod:`"cmd" execution
module <salt.modules.cmdmod>`. The filenames in most git
repositories are created using a UTF-8 locale, and the system
encoding on Windows (CP1252) will successfully (but incorrectly)
decode many UTF-8 characters. This makes interacting with
repositories containing UTF-8 filenames on Windows unreliable.
Therefore, Windows will default to decoding the output from git
commands using UTF-8 unless this option is explicitly used to
specify the encoding.
On non-Windows platforms, the default output decoding behavior will
be observed (i.e. the encoding specified by the locale will be
tried first, and if that fails, UTF-8 will be used as a fallback).
This should only be needed if the files in the repository were
created with filenames using an encoding other than UTF-8 to handle
Unicode characters.
.. versionadded:: 2018.3.1
@ -2222,21 +2063,9 @@ def fetch(cwd,
cases.
.. note::
On Windows, this option works slightly differently in the git state
and execution module than it does in the :mod:`"cmd" execution
module <salt.modules.cmdmod>`. The filenames in most git
repositories are created using a UTF-8 locale, and the system
encoding on Windows (CP1252) will successfully (but incorrectly)
decode many UTF-8 characters. This makes interacting with
repositories containing UTF-8 filenames on Windows unreliable.
Therefore, Windows will default to decoding the output from git
commands using UTF-8 unless this option is explicitly used to
specify the encoding.
On non-Windows platforms, the default output decoding behavior will
be observed (i.e. the encoding specified by the locale will be
tried first, and if that fails, UTF-8 will be used as a fallback).
This should only be needed if the files in the repository were
created with filenames using an encoding other than UTF-8 to handle
Unicode characters.
.. versionadded:: 2018.3.1
@ -2382,21 +2211,9 @@ def init(cwd,
cases.
.. note::
On Windows, this option works slightly differently in the git state
and execution module than it does in the :mod:`"cmd" execution
module <salt.modules.cmdmod>`. The filenames in most git
repositories are created using a UTF-8 locale, and the system
encoding on Windows (CP1252) will successfully (but incorrectly)
decode many UTF-8 characters. This makes interacting with
repositories containing UTF-8 filenames on Windows unreliable.
Therefore, Windows will default to decoding the output from git
commands using UTF-8 unless this option is explicitly used to
specify the encoding.
On non-Windows platforms, the default output decoding behavior will
be observed (i.e. the encoding specified by the locale will be
tried first, and if that fails, UTF-8 will be used as a fallback).
This should only be needed if the files in the repository were
created with filenames using an encoding other than UTF-8 to handle
Unicode characters.
.. versionadded:: 2018.3.1
@ -2471,21 +2288,9 @@ def is_worktree(cwd,
cases.
.. note::
On Windows, this option works slightly differently in the git state
and execution module than it does in the :mod:`"cmd" execution
module <salt.modules.cmdmod>`. The filenames in most git
repositories are created using a UTF-8 locale, and the system
encoding on Windows (CP1252) will successfully (but incorrectly)
decode many UTF-8 characters. This makes interacting with
repositories containing UTF-8 filenames on Windows unreliable.
Therefore, Windows will default to decoding the output from git
commands using UTF-8 unless this option is explicitly used to
specify the encoding.
On non-Windows platforms, the default output decoding behavior will
be observed (i.e. the encoding specified by the locale will be
tried first, and if that fails, UTF-8 will be used as a fallback).
This should only be needed if the files in the repository were
created with filenames using an encoding other than UTF-8 to handle
Unicode characters.
.. versionadded:: 2018.3.1
@ -2571,21 +2376,9 @@ def list_branches(cwd,
cases.
.. note::
On Windows, this option works slightly differently in the git state
and execution module than it does in the :mod:`"cmd" execution
module <salt.modules.cmdmod>`. The filenames in most git
repositories are created using a UTF-8 locale, and the system
encoding on Windows (CP1252) will successfully (but incorrectly)
decode many UTF-8 characters. This makes interacting with
repositories containing UTF-8 filenames on Windows unreliable.
Therefore, Windows will default to decoding the output from git
commands using UTF-8 unless this option is explicitly used to
specify the encoding.
On non-Windows platforms, the default output decoding behavior will
be observed (i.e. the encoding specified by the locale will be
tried first, and if that fails, UTF-8 will be used as a fallback).
This should only be needed if the files in the repository were
created with filenames using an encoding other than UTF-8 to handle
Unicode characters.
.. versionadded:: 2018.3.1
@ -2642,21 +2435,9 @@ def list_tags(cwd,
cases.
.. note::
On Windows, this option works slightly differently in the git state
and execution module than it does in the :mod:`"cmd" execution
module <salt.modules.cmdmod>`. The filenames in most git
repositories are created using a UTF-8 locale, and the system
encoding on Windows (CP1252) will successfully (but incorrectly)
decode many UTF-8 characters. This makes interacting with
repositories containing UTF-8 filenames on Windows unreliable.
Therefore, Windows will default to decoding the output from git
commands using UTF-8 unless this option is explicitly used to
specify the encoding.
On non-Windows platforms, the default output decoding behavior will
be observed (i.e. the encoding specified by the locale will be
tried first, and if that fails, UTF-8 will be used as a fallback).
This should only be needed if the files in the repository were
created with filenames using an encoding other than UTF-8 to handle
Unicode characters.
.. versionadded:: 2018.3.1
@ -2730,21 +2511,9 @@ def list_worktrees(cwd,
cases.
.. note::
On Windows, this option works slightly differently in the git state
and execution module than it does in the :mod:`"cmd" execution
module <salt.modules.cmdmod>`. The filenames in most git
repositories are created using a UTF-8 locale, and the system
encoding on Windows (CP1252) will successfully (but incorrectly)
decode many UTF-8 characters. This makes interacting with
repositories containing UTF-8 filenames on Windows unreliable.
Therefore, Windows will default to decoding the output from git
commands using UTF-8 unless this option is explicitly used to
specify the encoding.
On non-Windows platforms, the default output decoding behavior will
be observed (i.e. the encoding specified by the locale will be
tried first, and if that fails, UTF-8 will be used as a fallback).
This should only be needed if the files in the repository were
created with filenames using an encoding other than UTF-8 to handle
Unicode characters.
.. versionadded:: 2018.3.1
@ -3116,21 +2885,9 @@ def ls_remote(cwd=None,
cases.
.. note::
On Windows, this option works slightly differently in the git state
and execution module than it does in the :mod:`"cmd" execution
module <salt.modules.cmdmod>`. The filenames in most git
repositories are created using a UTF-8 locale, and the system
encoding on Windows (CP1252) will successfully (but incorrectly)
decode many UTF-8 characters. This makes interacting with
repositories containing UTF-8 filenames on Windows unreliable.
Therefore, Windows will default to decoding the output from git
commands using UTF-8 unless this option is explicitly used to
specify the encoding.
On non-Windows platforms, the default output decoding behavior will
be observed (i.e. the encoding specified by the locale will be
tried first, and if that fails, UTF-8 will be used as a fallback).
This should only be needed if the files in the repository were
created with filenames using an encoding other than UTF-8 to handle
Unicode characters.
.. versionadded:: 2018.3.1
@ -3237,21 +2994,9 @@ def merge(cwd,
cases.
.. note::
On Windows, this option works slightly differently in the git state
and execution module than it does in the :mod:`"cmd" execution
module <salt.modules.cmdmod>`. The filenames in most git
repositories are created using a UTF-8 locale, and the system
encoding on Windows (CP1252) will successfully (but incorrectly)
decode many UTF-8 characters. This makes interacting with
repositories containing UTF-8 filenames on Windows unreliable.
Therefore, Windows will default to decoding the output from git
commands using UTF-8 unless this option is explicitly used to
specify the encoding.
On non-Windows platforms, the default output decoding behavior will
be observed (i.e. the encoding specified by the locale will be
tried first, and if that fails, UTF-8 will be used as a fallback).
This should only be needed if the files in the repository were
created with filenames using an encoding other than UTF-8 to handle
Unicode characters.
.. versionadded:: 2018.3.1
@ -3390,21 +3135,9 @@ def merge_base(cwd,
cases.
.. note::
On Windows, this option works slightly differently in the git state
and execution module than it does in the :mod:`"cmd" execution
module <salt.modules.cmdmod>`. The filenames in most git
repositories are created using a UTF-8 locale, and the system
encoding on Windows (CP1252) will successfully (but incorrectly)
decode many UTF-8 characters. This makes interacting with
repositories containing UTF-8 filenames on Windows unreliable.
Therefore, Windows will default to decoding the output from git
commands using UTF-8 unless this option is explicitly used to
specify the encoding.
On non-Windows platforms, the default output decoding behavior will
be observed (i.e. the encoding specified by the locale will be
tried first, and if that fails, UTF-8 will be used as a fallback).
This should only be needed if the files in the repository were
created with filenames using an encoding other than UTF-8 to handle
Unicode characters.
.. versionadded:: 2018.3.1
@ -3559,21 +3292,9 @@ def merge_tree(cwd,
cases.
.. note::
On Windows, this option works slightly differently in the git state
and execution module than it does in the :mod:`"cmd" execution
module <salt.modules.cmdmod>`. The filenames in most git
repositories are created using a UTF-8 locale, and the system
encoding on Windows (CP1252) will successfully (but incorrectly)
decode many UTF-8 characters. This makes interacting with
repositories containing UTF-8 filenames on Windows unreliable.
Therefore, Windows will default to decoding the output from git
commands using UTF-8 unless this option is explicitly used to
specify the encoding.
On non-Windows platforms, the default output decoding behavior will
be observed (i.e. the encoding specified by the locale will be
tried first, and if that fails, UTF-8 will be used as a fallback).
This should only be needed if the files in the repository were
created with filenames using an encoding other than UTF-8 to handle
Unicode characters.
.. versionadded:: 2018.3.1
@ -3689,21 +3410,9 @@ def pull(cwd,
cases.
.. note::
On Windows, this option works slightly differently in the git state
and execution module than it does in the :mod:`"cmd" execution
module <salt.modules.cmdmod>`. The filenames in most git
repositories are created using a UTF-8 locale, and the system
encoding on Windows (CP1252) will successfully (but incorrectly)
decode many UTF-8 characters. This makes interacting with
repositories containing UTF-8 filenames on Windows unreliable.
Therefore, Windows will default to decoding the output from git
commands using UTF-8 unless this option is explicitly used to
specify the encoding.
On non-Windows platforms, the default output decoding behavior will
be observed (i.e. the encoding specified by the locale will be
tried first, and if that fails, UTF-8 will be used as a fallback).
This should only be needed if the files in the repository were
created with filenames using an encoding other than UTF-8 to handle
Unicode characters.
.. versionadded:: 2018.3.1
@ -3827,21 +3536,9 @@ def push(cwd,
cases.
.. note::
On Windows, this option works slightly differently in the git state
and execution module than it does in the :mod:`"cmd" execution
module <salt.modules.cmdmod>`. The filenames in most git
repositories are created using a UTF-8 locale, and the system
encoding on Windows (CP1252) will successfully (but incorrectly)
decode many UTF-8 characters. This makes interacting with
repositories containing UTF-8 filenames on Windows unreliable.
Therefore, Windows will default to decoding the output from git
commands using UTF-8 unless this option is explicitly used to
specify the encoding.
On non-Windows platforms, the default output decoding behavior will
be observed (i.e. the encoding specified by the locale will be
tried first, and if that fails, UTF-8 will be used as a fallback).
This should only be needed if the files in the repository were
created with filenames using an encoding other than UTF-8 to handle
Unicode characters.
.. versionadded:: 2018.3.1
@ -3935,21 +3632,9 @@ def rebase(cwd,
cases.
.. note::
On Windows, this option works slightly differently in the git state
and execution module than it does in the :mod:`"cmd" execution
module <salt.modules.cmdmod>`. The filenames in most git
repositories are created using a UTF-8 locale, and the system
encoding on Windows (CP1252) will successfully (but incorrectly)
decode many UTF-8 characters. This makes interacting with
repositories containing UTF-8 filenames on Windows unreliable.
Therefore, Windows will default to decoding the output from git
commands using UTF-8 unless this option is explicitly used to
specify the encoding.
On non-Windows platforms, the default output decoding behavior will
be observed (i.e. the encoding specified by the locale will be
tried first, and if that fails, UTF-8 will be used as a fallback).
This should only be needed if the files in the repository were
created with filenames using an encoding other than UTF-8 to handle
Unicode characters.
.. versionadded:: 2018.3.1
@ -4032,21 +3717,9 @@ def remote_get(cwd,
cases.
.. note::
On Windows, this option works slightly differently in the git state
and execution module than it does in the :mod:`"cmd" execution
module <salt.modules.cmdmod>`. The filenames in most git
repositories are created using a UTF-8 locale, and the system
encoding on Windows (CP1252) will successfully (but incorrectly)
decode many UTF-8 characters. This makes interacting with
repositories containing UTF-8 filenames on Windows unreliable.
Therefore, Windows will default to decoding the output from git
commands using UTF-8 unless this option is explicitly used to
specify the encoding.
On non-Windows platforms, the default output decoding behavior will
be observed (i.e. the encoding specified by the locale will be
tried first, and if that fails, UTF-8 will be used as a fallback).
This should only be needed if the files in the repository were
created with filenames using an encoding other than UTF-8 to handle
Unicode characters.
.. versionadded:: 2018.3.1
@ -4151,21 +3824,9 @@ def remote_refs(url,
cases.
.. note::
On Windows, this option works slightly differently in the git state
and execution module than it does in the :mod:`"cmd" execution
module <salt.modules.cmdmod>`. The filenames in most git
repositories are created using a UTF-8 locale, and the system
encoding on Windows (CP1252) will successfully (but incorrectly)
decode many UTF-8 characters. This makes interacting with
repositories containing UTF-8 filenames on Windows unreliable.
Therefore, Windows will default to decoding the output from git
commands using UTF-8 unless this option is explicitly used to
specify the encoding.
On non-Windows platforms, the default output decoding behavior will
be observed (i.e. the encoding specified by the locale will be
tried first, and if that fails, UTF-8 will be used as a fallback).
This should only be needed if the files in the repository were
created with filenames using an encoding other than UTF-8 to handle
Unicode characters.
.. versionadded:: 2018.3.1
@ -4275,21 +3936,9 @@ def remote_set(cwd,
cases.
.. note::
On Windows, this option works slightly differently in the git state
and execution module than it does in the :mod:`"cmd" execution
module <salt.modules.cmdmod>`. The filenames in most git
repositories are created using a UTF-8 locale, and the system
encoding on Windows (CP1252) will successfully (but incorrectly)
decode many UTF-8 characters. This makes interacting with
repositories containing UTF-8 filenames on Windows unreliable.
Therefore, Windows will default to decoding the output from git
commands using UTF-8 unless this option is explicitly used to
specify the encoding.
On non-Windows platforms, the default output decoding behavior will
be observed (i.e. the encoding specified by the locale will be
tried first, and if that fails, UTF-8 will be used as a fallback).
This should only be needed if the files in the repository were
created with filenames using an encoding other than UTF-8 to handle
Unicode characters.
.. versionadded:: 2018.3.1
@ -4402,21 +4051,9 @@ def remotes(cwd,
cases.
.. note::
On Windows, this option works slightly differently in the git state
and execution module than it does in the :mod:`"cmd" execution
module <salt.modules.cmdmod>`. The filenames in most git
repositories are created using a UTF-8 locale, and the system
encoding on Windows (CP1252) will successfully (but incorrectly)
decode many UTF-8 characters. This makes interacting with
repositories containing UTF-8 filenames on Windows unreliable.
Therefore, Windows will default to decoding the output from git
commands using UTF-8 unless this option is explicitly used to
specify the encoding.
On non-Windows platforms, the default output decoding behavior will
be observed (i.e. the encoding specified by the locale will be
tried first, and if that fails, UTF-8 will be used as a fallback).
This should only be needed if the files in the repository were
created with filenames using an encoding other than UTF-8 to handle
Unicode characters.
.. versionadded:: 2018.3.1
@ -4511,21 +4148,9 @@ def reset(cwd,
cases.
.. note::
On Windows, this option works slightly differently in the git state
and execution module than it does in the :mod:`"cmd" execution
module <salt.modules.cmdmod>`. The filenames in most git
repositories are created using a UTF-8 locale, and the system
encoding on Windows (CP1252) will successfully (but incorrectly)
decode many UTF-8 characters. This makes interacting with
repositories containing UTF-8 filenames on Windows unreliable.
Therefore, Windows will default to decoding the output from git
commands using UTF-8 unless this option is explicitly used to
specify the encoding.
On non-Windows platforms, the default output decoding behavior will
be observed (i.e. the encoding specified by the locale will be
tried first, and if that fails, UTF-8 will be used as a fallback).
This should only be needed if the files in the repository were
created with filenames using an encoding other than UTF-8 to handle
Unicode characters.
.. versionadded:: 2018.3.1
@ -4609,21 +4234,9 @@ def rev_parse(cwd,
cases.
.. note::
On Windows, this option works slightly differently in the git state
and execution module than it does in the :mod:`"cmd" execution
module <salt.modules.cmdmod>`. The filenames in most git
repositories are created using a UTF-8 locale, and the system
encoding on Windows (CP1252) will successfully (but incorrectly)
decode many UTF-8 characters. This makes interacting with
repositories containing UTF-8 filenames on Windows unreliable.
Therefore, Windows will default to decoding the output from git
commands using UTF-8 unless this option is explicitly used to
specify the encoding.
On non-Windows platforms, the default output decoding behavior will
be observed (i.e. the encoding specified by the locale will be
tried first, and if that fails, UTF-8 will be used as a fallback).
This should only be needed if the files in the repository were
created with filenames using an encoding other than UTF-8 to handle
Unicode characters.
.. versionadded:: 2018.3.1
@ -4701,21 +4314,9 @@ def revision(cwd,
cases.
.. note::
On Windows, this option works slightly differently in the git state
and execution module than it does in the :mod:`"cmd" execution
module <salt.modules.cmdmod>`. The filenames in most git
repositories are created using a UTF-8 locale, and the system
encoding on Windows (CP1252) will successfully (but incorrectly)
decode many UTF-8 characters. This makes interacting with
repositories containing UTF-8 filenames on Windows unreliable.
Therefore, Windows will default to decoding the output from git
commands using UTF-8 unless this option is explicitly used to
specify the encoding.
On non-Windows platforms, the default output decoding behavior will
be observed (i.e. the encoding specified by the locale will be
tried first, and if that fails, UTF-8 will be used as a fallback).
This should only be needed if the files in the repository were
created with filenames using an encoding other than UTF-8 to handle
Unicode characters.
.. versionadded:: 2018.3.1
@ -4799,21 +4400,9 @@ def rm_(cwd,
cases.
.. note::
On Windows, this option works slightly differently in the git state
and execution module than it does in the :mod:`"cmd" execution
module <salt.modules.cmdmod>`. The filenames in most git
repositories are created using a UTF-8 locale, and the system
encoding on Windows (CP1252) will successfully (but incorrectly)
decode many UTF-8 characters. This makes interacting with
repositories containing UTF-8 filenames on Windows unreliable.
Therefore, Windows will default to decoding the output from git
commands using UTF-8 unless this option is explicitly used to
specify the encoding.
On non-Windows platforms, the default output decoding behavior will
be observed (i.e. the encoding specified by the locale will be
tried first, and if that fails, UTF-8 will be used as a fallback).
This should only be needed if the files in the repository were
created with filenames using an encoding other than UTF-8 to handle
Unicode characters.
.. versionadded:: 2018.3.1
@ -4893,21 +4482,9 @@ def stash(cwd,
cases.
.. note::
On Windows, this option works slightly differently in the git state
and execution module than it does in the :mod:`"cmd" execution
module <salt.modules.cmdmod>`. The filenames in most git
repositories are created using a UTF-8 locale, and the system
encoding on Windows (CP1252) will successfully (but incorrectly)
decode many UTF-8 characters. This makes interacting with
repositories containing UTF-8 filenames on Windows unreliable.
Therefore, Windows will default to decoding the output from git
commands using UTF-8 unless this option is explicitly used to
specify the encoding.
On non-Windows platforms, the default output decoding behavior will
be observed (i.e. the encoding specified by the locale will be
tried first, and if that fails, UTF-8 will be used as a fallback).
This should only be needed if the files in the repository were
created with filenames using an encoding other than UTF-8 to handle
Unicode characters.
.. versionadded:: 2018.3.1
@ -4970,21 +4547,9 @@ def status(cwd,
cases.
.. note::
On Windows, this option works slightly differently in the git state
and execution module than it does in the :mod:`"cmd" execution
module <salt.modules.cmdmod>`. The filenames in most git
repositories are created using a UTF-8 locale, and the system
encoding on Windows (CP1252) will successfully (but incorrectly)
decode many UTF-8 characters. This makes interacting with
repositories containing UTF-8 filenames on Windows unreliable.
Therefore, Windows will default to decoding the output from git
commands using UTF-8 unless this option is explicitly used to
specify the encoding.
On non-Windows platforms, the default output decoding behavior will
be observed (i.e. the encoding specified by the locale will be
tried first, and if that fails, UTF-8 will be used as a fallback).
This should only be needed if the files in the repository were
created with filenames using an encoding other than UTF-8 to handle
Unicode characters.
.. versionadded:: 2018.3.1
@ -5125,21 +4690,9 @@ def submodule(cwd,
cases.
.. note::
On Windows, this option works slightly differently in the git state
and execution module than it does in the :mod:`"cmd" execution
module <salt.modules.cmdmod>`. The filenames in most git
repositories are created using a UTF-8 locale, and the system
encoding on Windows (CP1252) will successfully (but incorrectly)
decode many UTF-8 characters. This makes interacting with
repositories containing UTF-8 filenames on Windows unreliable.
Therefore, Windows will default to decoding the output from git
commands using UTF-8 unless this option is explicitly used to
specify the encoding.
On non-Windows platforms, the default output decoding behavior will
be observed (i.e. the encoding specified by the locale will be
tried first, and if that fails, UTF-8 will be used as a fallback).
This should only be needed if the files in the repository were
created with filenames using an encoding other than UTF-8 to handle
Unicode characters.
.. versionadded:: 2018.3.1
@ -5252,21 +4805,9 @@ def symbolic_ref(cwd,
cases.
.. note::
On Windows, this option works slightly differently in the git state
and execution module than it does in the :mod:`"cmd" execution
module <salt.modules.cmdmod>`. The filenames in most git
repositories are created using a UTF-8 locale, and the system
encoding on Windows (CP1252) will successfully (but incorrectly)
decode many UTF-8 characters. This makes interacting with
repositories containing UTF-8 filenames on Windows unreliable.
Therefore, Windows will default to decoding the output from git
commands using UTF-8 unless this option is explicitly used to
specify the encoding.
On non-Windows platforms, the default output decoding behavior will
be observed (i.e. the encoding specified by the locale will be
tried first, and if that fails, UTF-8 will be used as a fallback).
This should only be needed if the files in the repository were
created with filenames using an encoding other than UTF-8 to handle
Unicode characters.
.. versionadded:: 2018.3.1
@ -5447,21 +4988,9 @@ def worktree_add(cwd,
cases.
.. note::
On Windows, this option works slightly differently in the git state
and execution module than it does in the :mod:`"cmd" execution
module <salt.modules.cmdmod>`. The filenames in most git
repositories are created using a UTF-8 locale, and the system
encoding on Windows (CP1252) will successfully (but incorrectly)
decode many UTF-8 characters. This makes interacting with
repositories containing UTF-8 filenames on Windows unreliable.
Therefore, Windows will default to decoding the output from git
commands using UTF-8 unless this option is explicitly used to
specify the encoding.
On non-Windows platforms, the default output decoding behavior will
be observed (i.e. the encoding specified by the locale will be
tried first, and if that fails, UTF-8 will be used as a fallback).
This should only be needed if the files in the repository were
created with filenames using an encoding other than UTF-8 to handle
Unicode characters.
.. versionadded:: 2018.3.1
@ -5593,21 +5122,9 @@ def worktree_prune(cwd,
cases.
.. note::
On Windows, this option works slightly differently in the git state
and execution module than it does in the :mod:`"cmd" execution
module <salt.modules.cmdmod>`. The filenames in most git
repositories are created using a UTF-8 locale, and the system
encoding on Windows (CP1252) will successfully (but incorrectly)
decode many UTF-8 characters. This makes interacting with
repositories containing UTF-8 filenames on Windows unreliable.
Therefore, Windows will default to decoding the output from git
commands using UTF-8 unless this option is explicitly used to
specify the encoding.
On non-Windows platforms, the default output decoding behavior will
be observed (i.e. the encoding specified by the locale will be
tried first, and if that fails, UTF-8 will be used as a fallback).
This should only be needed if the files in the repository were
created with filenames using an encoding other than UTF-8 to handle
Unicode characters.
.. versionadded:: 2018.3.1
@ -5672,21 +5189,9 @@ def worktree_rm(cwd, user=None, output_encoding=None):
cases.
.. note::
On Windows, this option works slightly differently in the git state
and execution module than it does in the :mod:`"cmd" execution
module <salt.modules.cmdmod>`. The filenames in most git
repositories are created using a UTF-8 locale, and the system
encoding on Windows (CP1252) will successfully (but incorrectly)
decode many UTF-8 characters. This makes interacting with
repositories containing UTF-8 filenames on Windows unreliable.
Therefore, Windows will default to decoding the output from git
commands using UTF-8 unless this option is explicitly used to
specify the encoding.
On non-Windows platforms, the default output decoding behavior will
be observed (i.e. the encoding specified by the locale will be
tried first, and if that fails, UTF-8 will be used as a fallback).
This should only be needed if the files in the repository were
created with filenames using an encoding other than UTF-8 to handle
Unicode characters.
.. versionadded:: 2018.3.1

2
salt/modules/win_pkg.py
View file

@ -1882,7 +1882,7 @@ def get_repo_data(saltenv='base'):
serial = salt.payload.Serial(__opts__)
with salt.utils.files.fopen(repo_details.winrepo_file, 'rb') as repofile:
try:
repodata = salt.utils.data.decode(serial.loads(repofile.read(), encoding='utf-8') or {})
repodata = salt.utils.data.decode(serial.loads(repofile.read()) or {})
__context__['winrepo.data'] = repodata
return repodata
except Exception as exc:

90
salt/states/git.py
View file

@ -532,21 +532,9 @@ def latest(name,
cases.
.. note::
On Windows, this option works slightly differently in the git state
and execution module than it does in the :mod:`"cmd" execution
module <salt.modules.cmdmod>`. The filenames in most git
repositories are created using a UTF-8 locale, and the system
encoding on Windows (CP1252) will successfully (but incorrectly)
decode many UTF-8 characters. This makes interacting with
repositories containing UTF-8 filenames on Windows unreliable.
Therefore, Windows will default to decoding the output from git
commands using UTF-8 unless this option is explicitly used to
specify the encoding.
On non-Windows platforms, the default output decoding behavior will
be observed (i.e. the encoding specified by the locale will be
tried first, and if that fails, UTF-8 will be used as a fallback).
This should only be needed if the files in the repository were
created with filenames using an encoding other than UTF-8 to handle
Unicode characters.
.. versionadded:: 2018.3.1
@ -2032,21 +2020,9 @@ def present(name,
cases.
.. note::
On Windows, this option works slightly differently in the git state
and execution module than it does in the :mod:`"cmd" execution
module <salt.modules.cmdmod>`. The filenames in most git
repositories are created using a UTF-8 locale, and the system
encoding on Windows (CP1252) will successfully (but incorrectly)
decode many UTF-8 characters. This makes interacting with
repositories containing UTF-8 filenames on Windows unreliable.
Therefore, Windows will default to decoding the output from git
commands using UTF-8 unless this option is explicitly used to
specify the encoding.
On non-Windows platforms, the default output decoding behavior will
be observed (i.e. the encoding specified by the locale will be
tried first, and if that fails, UTF-8 will be used as a fallback).
This should only be needed if the files in the repository were
created with filenames using an encoding other than UTF-8 to handle
Unicode characters.
.. versionadded:: 2018.3.1
@ -2250,21 +2226,9 @@ def detached(name,
cases.
.. note::
On Windows, this option works slightly differently in the git state
and execution module than it does in the :mod:`"cmd" execution
module <salt.modules.cmdmod>`. The filenames in most git
repositories are created using a UTF-8 locale, and the system
encoding on Windows (CP1252) will successfully (but incorrectly)
decode many UTF-8 characters. This makes interacting with
repositories containing UTF-8 filenames on Windows unreliable.
Therefore, Windows will default to decoding the output from git
commands using UTF-8 unless this option is explicitly used to
specify the encoding.
On non-Windows platforms, the default output decoding behavior will
be observed (i.e. the encoding specified by the locale will be
tried first, and if that fails, UTF-8 will be used as a fallback).
This should only be needed if the files in the repository were
created with filenames using an encoding other than UTF-8 to handle
Unicode characters.
.. versionadded:: 2018.3.1
'''
@ -2750,21 +2714,9 @@ def config_unset(name,
cases.
.. note::
On Windows, this option works slightly differently in the git state
and execution module than it does in the :mod:`"cmd" execution
module <salt.modules.cmdmod>`. The filenames in most git
repositories are created using a UTF-8 locale, and the system
encoding on Windows (CP1252) will successfully (but incorrectly)
decode many UTF-8 characters. This makes interacting with
repositories containing UTF-8 filenames on Windows unreliable.
Therefore, Windows will default to decoding the output from git
commands using UTF-8 unless this option is explicitly used to
specify the encoding.
On non-Windows platforms, the default output decoding behavior will
be observed (i.e. the encoding specified by the locale will be
tried first, and if that fails, UTF-8 will be used as a fallback).
This should only be needed if the files in the repository were
created with filenames using an encoding other than UTF-8 to handle
Unicode characters.
.. versionadded:: 2018.3.1
@ -3023,21 +2975,9 @@ def config_set(name,
cases.
.. note::
On Windows, this option works slightly differently in the git state
and execution module than it does in the :mod:`"cmd" execution
module <salt.modules.cmdmod>`. The filenames in most git
repositories are created using a UTF-8 locale, and the system
encoding on Windows (CP1252) will successfully (but incorrectly)
decode many UTF-8 characters. This makes interacting with
repositories containing UTF-8 filenames on Windows unreliable.
Therefore, Windows will default to decoding the output from git
commands using UTF-8 unless this option is explicitly used to
specify the encoding.
On non-Windows platforms, the default output decoding behavior will
be observed (i.e. the encoding specified by the locale will be
tried first, and if that fails, UTF-8 will be used as a fallback).
This should only be needed if the files in the repository were
created with filenames using an encoding other than UTF-8 to handle
Unicode characters.
.. versionadded:: 2018.3.1

28
salt/utils/stringutils.py
View file

@ -41,10 +41,11 @@ def to_bytes(s, encoding=None, errors='strict'):
return s.encode(encoding, errors)
else:
try:
return s.encode(__salt_system_encoding__, errors)
except UnicodeEncodeError:
# Fall back to UTF-8
# Try UTF-8 first
return s.encode('utf-8', errors)
except UnicodeEncodeError:
# Fall back to detected encoding
return s.encode(__salt_system_encoding__, errors)
raise TypeError('expected bytes, bytearray, or str')
else:
return to_str(s, encoding, errors)
@ -64,10 +65,11 @@ def to_str(s, encoding=None, errors='strict'):
return s.decode(encoding, errors)
else:
try:
return s.decode(__salt_system_encoding__, errors)
except UnicodeDecodeError:
# Fall back to UTF-8
# Try UTF-8 first
return s.decode('utf-8', errors)
except UnicodeDecodeError:
# Fall back to detected encoding
return s.decode(__salt_system_encoding__, errors)
raise TypeError('expected str, bytes, or bytearray not {}'.format(type(s)))
else:
if isinstance(s, bytearray):
@ -77,10 +79,11 @@ def to_str(s, encoding=None, errors='strict'):
return s.encode(encoding, errors)
else:
try:
return s.encode(__salt_system_encoding__, errors)
except UnicodeEncodeError:
# Fall back to UTF-8
# Try UTF-8 first
return s.encode('utf-8', errors)
except UnicodeEncodeError:
# Fall back to detected encoding
return s.encode(__salt_system_encoding__, errors)
raise TypeError('expected str, bytearray, or unicode')
@ -108,10 +111,11 @@ def to_unicode(s, encoding=None, errors='strict', normalize=False):
return _normalize(s.decode(encoding, errors))
else:
try:
return _normalize(s.decode(__salt_system_encoding__, errors))
except UnicodeDecodeError:
# Fall back to UTF-8
# Try UTF-8 first
return _normalize(s.decode('utf-8', errors))
except UnicodeDecodeError:
# Fall back to detected encoding
return _normalize(s.decode(__salt_system_encoding__, errors))
raise TypeError('expected str or bytearray')

9
tests/integration/modules/test_git.py
View file

@ -147,8 +147,7 @@ class GitModuleTest(ModuleCase):
TODO: maybe move this behavior to ModuleCase itself?
'''
return salt.utils.data.decode(
super(GitModuleTest, self).run_function(*args, **kwargs),
encoding='utf-8'
super(GitModuleTest, self).run_function(*args, **kwargs)
)
def tearDown(self):
@ -207,8 +206,7 @@ class GitModuleTest(ModuleCase):
self.run_function('cmd.run', ['cp ' + tar_archive + ' /root/'])
with closing(tarfile.open(tar_archive, 'r')) as tar_obj:
self.assertEqual(
sorted(salt.utils.data.decode(tar_obj.getnames(),
encoding='utf-8')),
sorted(salt.utils.data.decode(tar_obj.getnames())),
sorted([
'foo', 'foo/bar', 'foo/baz', 'foo/foo', 'foo/питон',
'foo/qux', 'foo/qux/bar', 'foo/qux/baz', 'foo/qux/foo',
@ -238,8 +236,7 @@ class GitModuleTest(ModuleCase):
self.assertTrue(tarfile.is_tarfile(tar_archive))
with closing(tarfile.open(tar_archive, 'r')) as tar_obj:
self.assertEqual(
sorted(salt.utils.data.decode(tar_obj.getnames(),
encoding='utf-8')),
sorted(salt.utils.data.decode(tar_obj.getnames())),
sorted(['foo', 'foo/bar', 'foo/baz', 'foo/foo', 'foo/питон'])
)
finally:

3
tests/unit/utils/test_data.py
View file

@ -240,7 +240,6 @@ class DataTestCase(TestCase):
ret = salt.utils.data.decode(
self.test_data,
encoding='utf-8',
keep=True,
normalize=True,
preserve_dict_class=True,
@ -253,7 +252,6 @@ class DataTestCase(TestCase):
UnicodeDecodeError,
salt.utils.data.decode,
self.test_data,
encoding='utf-8',
keep=False,
normalize=True,
preserve_dict_class=True,
@ -267,7 +265,6 @@ class DataTestCase(TestCase):
ret = salt.utils.data.decode(
self.test_data,
encoding='utf-8',
keep=True,
normalize=True,
preserve_dict_class=False,

9
tests/unit/utils/test_stringutils.py
View file

@ -97,7 +97,6 @@ class StringutilsTestCase(TestCase):
self.assertEqual(
salt.utils.stringutils.to_unicode(
EGGS,
encoding='utf=8',
normalize=True
),
'яйца'
@ -105,7 +104,6 @@ class StringutilsTestCase(TestCase):
self.assertNotEqual(
salt.utils.stringutils.to_unicode(
EGGS,
encoding='utf=8',
normalize=False
),
'яйца'
@ -120,9 +118,12 @@ class StringutilsTestCase(TestCase):
self.assertEqual(salt.utils.stringutils.to_unicode(str('xyzzy'), 'utf-8'), 'xyzzy') # future lint: disable=blacklisted-function
self.assertEqual(salt.utils.stringutils.to_unicode(BYTES, 'utf-8'), UNICODE)
# Test utf-8 fallback with ascii default encoding
# Test that unicode chars are decoded properly even when using
# locales which are not UTF-8 compatible
with patch.object(builtins, '__salt_system_encoding__', 'ascii'):
self.assertEqual(salt.utils.stringutils.to_unicode(u'Ψ'.encode('utf-8')), u'Ψ')
self.assertEqual(salt.utils.stringutils.to_unicode('Ψ'.encode('utf-8')), 'Ψ')
with patch.object(builtins, '__salt_system_encoding__', 'CP1252'):
self.assertEqual(salt.utils.stringutils.to_unicode('Ψ'.encode('utf-8')), 'Ψ')
def test_build_whitespace_split_regex(self):
expected_regex = '(?m)^(?:[\\s]+)?Lorem(?:[\\s]+)?ipsum(?:[\\s]+)?dolor(?:[\\s]+)?sit(?:[\\s]+)?amet\\,' \