From 69fe65b5505900ee16f605ed7b6dc6340e7683f2 Mon Sep 17 00:00:00 2001 From: Sean Brennan Date: Wed, 1 May 2019 14:34:09 -0400 Subject: [PATCH 1/4] Use rsplit Fixes exception thrown on ipv6 addresses ValueError: invalid literal for int() with base 10: ':ffff:1.2.3.4:5678' --- salt/utils/network.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/salt/utils/network.py b/salt/utils/network.py index 83269cdcf64..61c4a918b1c 100644 --- a/salt/utils/network.py +++ b/salt/utils/network.py @@ -1486,8 +1486,8 @@ def _netlink_tool_remote_on(port, which_end): elif 'ESTAB' not in line: continue chunks = line.split() - local_host, local_port = chunks[3].split(':', 1) - remote_host, remote_port = chunks[4].split(':', 1) + local_host, local_port = chunks[3].rsplit(':', 1) + remote_host, remote_port = chunks[4].rsplit(':', 1) if which_end == 'remote_port' and int(remote_port) != port: continue From c1eae841ca066ff8f9373ebeb4fb8c37f833154b Mon Sep 17 00:00:00 2001 From: Sean Brennan Date: Thu, 2 May 2019 14:22:57 -0400 Subject: [PATCH 2/4] Add test --- tests/unit/utils/test_network.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/unit/utils/test_network.py b/tests/unit/utils/test_network.py index 3d20c880bd6..5790dd7ac21 100644 --- a/tests/unit/utils/test_network.py +++ b/tests/unit/utils/test_network.py @@ -120,6 +120,12 @@ USER COMMAND PID FD PROTO LOCAL ADDRESS FOREIGN ADDRESS salt-master python2.781106 35 tcp4 127.0.0.1:61115 127.0.0.1:4506 ''' +NETLINK_SS = ''' +State Recv-Q Send-Q Local Address:Port Peer Address:Port +ESTAB 0 0 127.0.0.1:56726 127.0.0.1:4505 +ESTAB 0 0 ::ffff:1.2.3.4:5678 ::ffff:1.2.3.4:4505 +''' + IPV4_SUBNETS = {True: ('10.10.0.0/24',), False: ('10.10.0.0', '10.10.0.0/33', 'FOO', 9, '0.9.800.1000/24')} IPV6_SUBNETS = {True: ('::1/128',), @@ -637,3 +643,8 @@ class NetworkTestCase(TestCase): # An exception is raised if unicode is passed to socket.getfqdn minion_id = network.generate_minion_id() assert minion_id != '', minion_id + + def test_netlink_tool_remote_on(self): + with patch('subprocess.check_output', return_value=NETLINK_SS): + remotes = network._netlink_tool_remote_on('4505', 'remote') + self.assertEqual(remotes, set(['127.0.0.1', '::ffff:1.2.3.4'])) From fc70884ca170b0df5d55382276ce5f127fd7c961 Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Mon, 6 May 2019 17:57:11 -0400 Subject: [PATCH 3/4] Reload matcher loader when ext_pillar_first set --- salt/pillar/__init__.py | 6 ++-- tests/unit/test_pillar.py | 67 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 2 deletions(-) diff --git a/salt/pillar/__init__.py b/salt/pillar/__init__.py index 4592ac0c7bb..fc9a2a88c45 100644 --- a/salt/pillar/__init__.py +++ b/salt/pillar/__init__.py @@ -655,7 +655,7 @@ class Pillar(object): errors.append('Error encountered while rendering pillar top file.') return merged_tops, errors - def top_matches(self, top): + def top_matches(self, top, reload=False): ''' Search through the top high data for matches and return the states that this minion needs to execute. @@ -664,6 +664,8 @@ class Pillar(object): {'saltenv': ['state1', 'state2', ...]} ''' matches = {} + if reload: + self.matchers = salt.loader.matchers(self.opts) for saltenv, body in six.iteritems(top): if self.opts['pillarenv']: if saltenv != self.opts['pillarenv']: @@ -1001,7 +1003,7 @@ class Pillar(object): if self.opts.get('ext_pillar_first', False): self.opts['pillar'], errors = self.ext_pillar(self.pillar_override) self.rend = salt.loader.render(self.opts, self.functions) - matches = self.top_matches(top) + matches = self.top_matches(top, reload=True) pillar, errors = self.render_pillar(matches, errors=errors) pillar = merge( self.opts['pillar'], diff --git a/tests/unit/test_pillar.py b/tests/unit/test_pillar.py index 64c77b186ad..48710181910 100644 --- a/tests/unit/test_pillar.py +++ b/tests/unit/test_pillar.py @@ -326,6 +326,51 @@ class PillarTestCase(TestCase): 'mocked-minion', 'fake_pillar', 'bar', extra_minion_data={'fake_key': 'foo'}) + def test_ext_pillar_first(self): + ''' + test when using ext_pillar and ext_pillar_first + ''' + opts = { + 'optimization_order': [0, 1, 2], + 'renderer': 'yaml', + 'renderer_blacklist': [], + 'renderer_whitelist': [], + 'state_top': '', + 'pillar_roots': [], + 'extension_modules': '', + 'saltenv': 'base', + 'file_roots': [], + 'ext_pillar_first': True, + } + grains = { + 'os': 'Ubuntu', + 'os_family': 'Debian', + 'oscodename': 'raring', + 'osfullname': 'Ubuntu', + 'osrelease': '13.04', + 'kernel': 'Linux' + } + + tempdir = tempfile.mkdtemp(dir=TMP) + try: + sls_files = self._setup_test_topfile_sls_pillar_match( + tempdir,) + fc_mock = MockFileclient( + cache_file=sls_files['top']['dest'], + list_states=['top', 'ssh', 'ssh.minion', + 'generic', 'generic.minion'], + get_state=sls_files) + with patch.object(salt.fileclient, 'get_file_client', + MagicMock(return_value=fc_mock)), \ + patch('salt.pillar.Pillar.ext_pillar', + MagicMock(return_value=({'id': 'minion', + 'phase': 'alpha', 'role': + 'database'}, []))): + pillar = salt.pillar.Pillar(opts, grains, 'mocked-minion', 'base') + self.assertEqual(pillar.compile_pillar()['generic']['key1'], 'value1') + finally: + shutil.rmtree(tempdir, ignore_errors=True) + def test_dynamic_pillarenv(self): opts = { 'optimization_order': [0, 1, 2], @@ -581,6 +626,28 @@ class PillarTestCase(TestCase): # precedence over glob match. _run_test(nodegroup_order=2, glob_order=1, expected='foo') + def _setup_test_topfile_sls_pillar_match(self, tempdir): + # Write a simple topfile and two pillar state files + top_file = tempfile.NamedTemporaryFile(dir=tempdir, delete=False) + s = ''' +base: + 'phase:alpha': + - match: pillar + - generic +''' + top_file.write(salt.utils.stringutils.to_bytes(s)) + top_file.flush() + generic_file = tempfile.NamedTemporaryFile(dir=tempdir, delete=False) + generic_file.write(b''' +generic: + key1: value1 +''') + generic_file.flush() + return { + 'top': {'path': '', 'dest': top_file.name}, + 'generic': {'path': '', 'dest': generic_file.name}, + } + def _setup_test_topfile_sls(self, tempdir, nodegroup_order, glob_order): # Write a simple topfile and two pillar state files top_file = tempfile.NamedTemporaryFile(dir=tempdir, delete=False) From bdcae5de8207a1354ec57a5a398d9dc53da02a98 Mon Sep 17 00:00:00 2001 From: Ch3LL Date: Mon, 6 May 2019 18:01:05 -0400 Subject: [PATCH 4/4] Add docs for new reload kwarg --- salt/pillar/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/salt/pillar/__init__.py b/salt/pillar/__init__.py index fc9a2a88c45..13fed4593ae 100644 --- a/salt/pillar/__init__.py +++ b/salt/pillar/__init__.py @@ -662,6 +662,9 @@ class Pillar(object): Returns: {'saltenv': ['state1', 'state2', ...]} + + reload + Reload the matcher loader ''' matches = {} if reload: