Merge branch '2019.2.1' into fix_msgpack

salt/log/setup.py

@@ -828,13 +828,17 @@ def setup_extended_logging(opts):
 
 def get_multiprocessing_logging_queue():
     global __MP_LOGGING_QUEUE
+    from salt.utils.platform import is_darwin
 
     if __MP_IN_MAINPROCESS is False:
         # We're not in the MainProcess, return! No Queue shall be instantiated
         return __MP_LOGGING_QUEUE
 
     if __MP_LOGGING_QUEUE is None:
-        __MP_LOGGING_QUEUE = multiprocessing.Queue()
+        if is_darwin():
+            __MP_LOGGING_QUEUE = multiprocessing.Queue(32767)
+        else:
+            __MP_LOGGING_QUEUE = multiprocessing.Queue(100000)
     return __MP_LOGGING_QUEUE
 
 

salt/serializers/yaml.py

@@ -72,6 +72,7 @@ def serialize(obj, **options):
     '''
 
     options.setdefault('Dumper', Dumper)
+    options.setdefault('default_flow_style', None)
     try:
         response = yaml.dump(obj, **options)
         if response.endswith('\n...\n'):

salt/serializers/yamlex.py

@@ -182,6 +182,7 @@ def serialize(obj, **options):
     '''
 
     options.setdefault('Dumper', Dumper)
+    options.setdefault('default_flow_style', None)
     try:
         response = yaml.dump(obj, **options)
         if response.endswith('\n...\n'):

salt/utils/thin.py

@@ -430,7 +430,7 @@ def gen_thin(cachedir, extra_mods='', overwrite=False, so_mods='',
                 log.error(tops_failure_msg, 'parsing', python3_bin)
                 log.exception(err)
         else:
-            log.error(tops_failure_msg, 'collecting', python3_bin)
+            log.debug(tops_failure_msg, 'collecting', python3_bin)
             log.debug(stderr)
 
     # Collect tops, alternative to 3.x version
@@ -448,7 +448,7 @@ def gen_thin(cachedir, extra_mods='', overwrite=False, so_mods='',
                 log.error(tops_failure_msg, 'parsing', python2_bin)
                 log.exception(err)
         else:
-            log.error(tops_failure_msg, 'collecting', python2_bin)
+            log.debug(tops_failure_msg, 'collecting', python2_bin)
             log.debug(stderr)
 
     with salt.utils.files.fopen(pymap_cfg, 'wb') as fp_:

salt/utils/yamldumper.py

@@ -115,6 +115,7 @@ def dump(data, stream=None, **kwargs):
     '''
     if 'allow_unicode' not in kwargs:
         kwargs['allow_unicode'] = True
+    kwargs.setdefault('default_flow_style', None)
     return yaml.dump(data, stream, **kwargs)
 
 
@@ -126,4 +127,5 @@ def safe_dump(data, stream=None, **kwargs):
     '''
     if 'allow_unicode' not in kwargs:
         kwargs['allow_unicode'] = True
+    kwargs.setdefault('default_flow_style', None)
     return yaml.dump(data, stream, Dumper=SafeOrderedDumper, **kwargs)

tests/integration/files/log_handlers/runtests_log_handler.py

@@ -24,6 +24,7 @@ import msgpack
 
 # Import Salt libs
 from salt.ext import six
+from salt.utils.platform import is_darwin
 import salt.log.setup
 
 log = logging.getLogger(__name__)
@@ -60,7 +61,10 @@ def setup_handlers():
     # Above that value, if `process_queue` can't process fast enough,
     # start dropping. This will contain a memory leak in case `process_queue`
     # can't process fast enough of in case it can't deliver the log records at all.
-    queue_size = 10000000
+    if is_darwin():
+        queue_size = 32767
+    else:
+        queue_size = 10000000
     queue = Queue(queue_size)
     handler = salt.log.setup.QueueHandler(queue)
     level = salt.log.setup.LOG_LEVELS[(__opts__.get('runtests_log_level') or 'error').lower()]

tests/integration/states/test_pkg.py

@@ -38,7 +38,7 @@ _PKG_TARGETS = {
     'Debian': ['python-plist', 'apg'],
     'RedHat': ['units', 'zsh-html'],
     'FreeBSD': ['aalib', 'pth'],
-    'Suse': ['aalib', 'rpm-python'],
+    'Suse': ['aalib', 'htop'],
     'MacOS': ['libpng', 'jpeg'],
     'Windows': ['putty', '7zip'],
 }

tests/unit/modules/test_telegram.py

@@ -8,11 +8,13 @@ Tests for the Telegram execution module.
 # Import Python Libs
 from __future__ import absolute_import
 
+import logging
+
 # Import Salt Testing Libs
 from tests.support.mixins import LoaderModuleMockMixin
 from tests.support.unit import TestCase, skipIf
 from tests.support.mock import (
-    patch,
+    Mock,
     MagicMock,
     NO_MOCK,
     NO_MOCK_REASON
@@ -22,6 +24,47 @@ from tests.support.mock import (
 import salt.modules.telegram as telegram
 
 
+log = logging.getLogger(__name__)
+
+
+class RequestMock(Mock):
+    '''
+    Request Mock
+    '''
+
+    def get(self, *args, **kwargs):
+        return RequestResponseMock()
+
+    def put(self, *args, **kwargs):
+        self.args = args
+        self.kwargs = kwargs
+        return RequestPutResponseMock()
+
+    def delete(self, *args, **kwargs):
+        self.args = args
+        self.kwargs = kwargs
+        return RequestResponseMock()
+
+
+class RequestResponseMock(Mock):
+    '''
+    Request Response Mock
+    '''
+    def json(self):
+        return [{'url': 'http://example.org',
+                 '_id': 1234}, ]
+
+
+class RequestPutResponseMock(Mock):
+    '''
+    Request Put Response Mock
+    '''
+    ok = True
+
+    def json(self):
+        return {'_id': 4321}
+
+
 @skipIf(NO_MOCK, NO_MOCK_REASON)
 class TelegramModuleTest(TestCase, LoaderModuleMockMixin):
     '''
@@ -29,35 +72,22 @@ class TelegramModuleTest(TestCase, LoaderModuleMockMixin):
     '''
     def setup_loader_modules(self):
         module_globals = {
-            telegram: {
-                '__salt__': {
-                    'config.get': MagicMock(return_value={
-                        'telegram': {
-                            'chat_id': '123456789',
-                            'token': '000000000:xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'
-                        }
-                    })
-                }
-            }
+            '__salt__': {
+                'config.get': MagicMock(return_value={
+                    'telegram': {
+                        'chat_id': '123456789',
+                        'token': '000000000:xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'
+                    }
+                }),
+                'requests.put': Mock(),
+            },
+            'requests': RequestMock()
         }
-        if telegram.HAS_REQUESTS is False:
-            module_globals['sys.modules'] = {'requests': MagicMock()}
-        return module_globals
+        return {telegram: module_globals}
 
     def test_post_message(self):
         '''
         Test the post_message function.
         '''
         message = 'Hello World!'
-
-        class MockRequests(object):
-            """
-            Mock of requests response.
-            """
-            def json(self):
-                return {'ok': True}
-
-        with patch('salt.modules.telegram.requests.post',
-                   MagicMock(return_value=MockRequests())):
-
-            self.assertTrue(telegram.post_message(message))
+        self.assertTrue(telegram.post_message(message))

tests/unit/serializers/test_serializers.py

@@ -68,9 +68,24 @@ class TestSerializers(TestCase):
         serialized = yamlex.serialize(data)
         assert serialized == '{foo: bar}', serialized
 
+        serialized = yamlex.serialize(data, default_flow_style=False)
+        assert serialized == 'foo: bar', serialized
+
         deserialized = yamlex.deserialize(serialized)
         assert deserialized == data, deserialized
 
+        serialized = yaml.serialize(data)
+        assert serialized == '{foo: bar}', serialized
+
+        deserialized = yaml.deserialize(serialized)
+        assert deserialized == data, deserialized
+
+        serialized = yaml.serialize(data, default_flow_style=False)
+        assert serialized == 'foo: bar', serialized
+
+        deserialized = yaml.deserialize(serialized)
+        assert deserialized == data, deserialized
+
     @skipIf(not yamlex.available, SKIP_MESSAGE % 'sls')
     def test_serialize_complex_sls(self):
         data = OrderedDict([
@@ -84,6 +99,12 @@ class TestSerializers(TestCase):
         deserialized = yamlex.deserialize(serialized)
         assert deserialized == data, deserialized
 
+        serialized = yaml.serialize(data)
+        assert serialized == '{bar: 2, baz: true, foo: 1}', serialized
+
+        deserialized = yaml.deserialize(serialized)
+        assert deserialized == data, deserialized
+
     @skipIf(not yaml.available, SKIP_MESSAGE % 'yaml')
     @skipIf(not yamlex.available, SKIP_MESSAGE % 'sls')
     def test_compare_sls_vs_yaml(self):

tests/unit/utils/test_yamldumper.py

@@ -0,0 +1,45 @@
+# -*- coding: utf-8 -*-
+'''
+    Unit tests for salt.utils.yamldumper
+'''
+
+# Import python libs
+from __future__ import absolute_import, print_function, unicode_literals
+
+# Import Salt Libs
+import salt.ext.six
+import salt.utils.yamldumper
+
+# Import Salt Testing Libs
+from tests.support.unit import TestCase, skipIf
+from tests.support.mock import NO_MOCK, NO_MOCK_REASON
+
+
+@skipIf(NO_MOCK, NO_MOCK_REASON)
+class YamlDumperTestCase(TestCase):
+    '''
+    TestCase for salt.utils.yamldumper module
+    '''
+    def test_yaml_dump(self):
+        '''
+        Test yaml.dump a dict
+        '''
+        data = {'foo': 'bar'}
+
+        if salt.ext.six.PY2:
+            exp_yaml = '{!!python/unicode \'foo\': !!python/unicode \'bar\'}\n'
+        else:
+            exp_yaml = '{foo: bar}\n'
+
+        assert salt.utils.yamldumper.dump(data) == exp_yaml
+
+        assert salt.utils.yamldumper.dump(data, default_flow_style=False) == exp_yaml.replace('{', '').replace('}', '')
+
+    def test_yaml_safe_dump(self):
+        '''
+        Test yaml.safe_dump a dict
+        '''
+        data = {'foo': 'bar'}
+        assert salt.utils.yamldumper.safe_dump(data) == '{foo: bar}\n'
+
+        assert salt.utils.yamldumper.safe_dump(data, default_flow_style=False) == 'foo: bar\n'