Merge pull request #50492 from rallytime/bp-50228-and-50443

Back-port #50228 and #50443 to 2018.3
This commit is contained in:
Nicole Thomas 2018-11-13 11:59:14 -05:00 committed by GitHub
commit 6b272c1d65
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 248 additions and 235 deletions

View file

@ -11,6 +11,7 @@ import subprocess
import os
import plistlib
import time
import xml.parsers.expat
# Import Salt Libs
import salt.modules.cmdmod
@ -40,6 +41,11 @@ __salt__ = {
'cmd.run': salt.modules.cmdmod._run_quiet,
}
if six.PY2:
class InvalidFileException(Exception):
pass
plistlib.InvalidFileException = InvalidFileException
def __virtual__():
'''
@ -301,6 +307,12 @@ def launchctl(sub_cmd, *args, **kwargs):
def _available_services(refresh=False):
'''
This is a helper function for getting the available macOS services.
The strategy is to look through the known system locations for
launchd plist files, parse them, and use their information for
populating the list of services. Services can run without a plist
file present, but normally services which have an automated startup
will have a plist file, so this is a minor compromise.
'''
try:
if __context__['available_services'] and not refresh:
@ -316,6 +328,15 @@ def _available_services(refresh=False):
'/System/Library/LaunchAgents',
'/System/Library/LaunchDaemons',
]
try:
for user in os.listdir('/Users/'):
agent_path = '/Users/{}/Library/LaunchAgents'.format(user)
if os.path.isdir(agent_path):
launchd_paths.append(agent_path)
except OSError:
pass
_available_services = dict()
for launch_dir in launchd_paths:
for root, dirs, files in salt.utils.path.os_walk(launch_dir):
@ -328,39 +349,59 @@ def _available_services(refresh=False):
# Follow symbolic links of files in _launchd_paths
file_path = os.path.join(root, file_name)
true_path = os.path.realpath(file_path)
log.trace('Gathering service info for %s', true_path)
# ignore broken symlinks
if not os.path.exists(true_path):
continue
try:
# This assumes most of the plist files
# will be already in XML format
plist = plistlib.readPlist(true_path)
if six.PY2:
# py2 plistlib can't read binary plists, and
# uses a different API than py3.
plist = plistlib.readPlist(true_path)
else:
with salt.utils.files.fopen(true_path, 'rb') as handle:
plist = plistlib.load(handle)
except Exception:
# If plistlib is unable to read the file we'll need to use
# the system provided plutil program to do the conversion
except plistlib.InvalidFileException:
# Raised in python3 if the file is not XML.
# There's nothing we can do; move on to the next one.
msg = 'Unable to parse "%s" as it is invalid XML: InvalidFileException.'
logging.warning(msg, true_path)
continue
except xml.parsers.expat.ExpatError:
# Raised by py2 for all errors.
# Raised by py3 if the file is XML, but with errors.
if six.PY3:
# There's an error in the XML, so move on.
msg = 'Unable to parse "%s" as it is invalid XML: xml.parsers.expat.ExpatError.'
logging.warning(msg, true_path)
continue
# Use the system provided plutil program to attempt
# conversion from binary.
cmd = '/usr/bin/plutil -convert xml1 -o - -- "{0}"'.format(
true_path)
plist_xml = __salt__['cmd.run'](cmd)
if six.PY2:
try:
plist_xml = __salt__['cmd.run'](cmd)
plist = plistlib.readPlistFromString(plist_xml)
else:
plist = plistlib.loads(
salt.utils.stringutils.to_bytes(plist_xml))
except xml.parsers.expat.ExpatError:
# There's still an error in the XML, so move on.
msg = 'Unable to parse "%s" as it is invalid XML: xml.parsers.expat.ExpatError.'
logging.warning(msg, true_path)
continue
try:
_available_services[plist.Label.lower()] = {
'file_name': file_name,
'file_path': true_path,
'plist': plist}
except AttributeError:
# Handle malformed plist files
_available_services[os.path.basename(file_name).lower()] = {
# not all launchd plists contain a Label key
_available_services[plist['Label'].lower()] = {
'file_name': file_name,
'file_path': true_path,
'plist': plist}
except KeyError:
log.debug('Service %s does not contain a'
' Label key. Skipping.', true_path)
continue
# put this in __context__ as this is a time consuming function.
# a fix for this issue. https://github.com/saltstack/salt/issues/48414

View file

@ -5,11 +5,19 @@ mac_utils tests
# Import python libs
from __future__ import absolute_import, unicode_literals
import os
import plistlib
import xml.parsers.expat
# Import Salt Testing Libs
from tests.support.unit import TestCase, skipIf
from tests.support.mock import MagicMock, patch, NO_MOCK, NO_MOCK_REASON, call
from tests.support.mock import (
call,
MagicMock,
mock_open,
NO_MOCK,
NO_MOCK_REASON,
patch
)
from tests.support.mixins import LoaderModuleMockMixin
# Import Salt libs
@ -215,259 +223,223 @@ class MacUtilsTestCase(TestCase, LoaderModuleMockMixin):
@patch('salt.utils.path.os_walk')
@patch('os.path.exists')
@patch('plistlib.readPlist')
def test_available_services(self, mock_read_plist, mock_exists, mock_os_walk):
def test_available_services_result(self, mock_exists, mock_os_walk):
'''
test available_services
test available_services results are properly formed dicts.
'''
mock_os_walk.side_effect = [
[('/Library/LaunchAgents', [], ['com.apple.lla1.plist', 'com.apple.lla2.plist'])],
[('/Library/LaunchDaemons', [], ['com.apple.lld1.plist', 'com.apple.lld2.plist'])],
[('/System/Library/LaunchAgents', [], ['com.apple.slla1.plist', 'com.apple.slla2.plist'])],
[('/System/Library/LaunchDaemons', [], ['com.apple.slld1.plist', 'com.apple.slld2.plist'])],
]
mock_read_plist.side_effect = [
MagicMock(Label='com.apple.lla1'),
MagicMock(Label='com.apple.lla2'),
MagicMock(Label='com.apple.lld1'),
MagicMock(Label='com.apple.lld2'),
MagicMock(Label='com.apple.slla1'),
MagicMock(Label='com.apple.slla2'),
MagicMock(Label='com.apple.slld1'),
MagicMock(Label='com.apple.slld2'),
]
results = {'/Library/LaunchAgents': ['com.apple.lla1.plist']}
mock_os_walk.side_effect = _get_walk_side_effects(results)
mock_exists.return_value = True
ret = mac_utils._available_services()
# Make sure it's a dict with 8 items
self.assertTrue(isinstance(ret, dict))
self.assertEqual(len(ret), 8)
plists = [{'Label': 'com.apple.lla1'}]
ret = _run_available_services(plists)
self.assertEqual(
ret['com.apple.lla1']['file_name'],
'com.apple.lla1.plist')
self.assertEqual(
ret['com.apple.lla1']['file_path'],
os.path.realpath(
os.path.join('/Library/LaunchAgents', 'com.apple.lla1.plist')))
self.assertEqual(
ret['com.apple.slld2']['file_name'],
'com.apple.slld2.plist')
self.assertEqual(
ret['com.apple.slld2']['file_path'],
os.path.realpath(
os.path.join('/System/Library/LaunchDaemons', 'com.apple.slld2.plist')))
expected = {
'com.apple.lla1': {
'file_name': 'com.apple.lla1.plist',
'file_path': '/Library/LaunchAgents/com.apple.lla1.plist',
'plist': plists[0]}}
self.assertEqual(ret, expected)
@patch('salt.utils.path.os_walk')
@patch('os.path.exists')
@patch('plistlib.readPlist')
@patch('os.listdir')
@patch('os.path.isdir')
def test_available_services_dirs(self,
mock_isdir,
mock_listdir,
mock_exists,
mock_os_walk):
'''
test available_services checks all of the expected dirs.
'''
results = {
'/Library/LaunchAgents': ['com.apple.lla1.plist'],
'/Library/LaunchDaemons': ['com.apple.lld1.plist'],
'/System/Library/LaunchAgents': ['com.apple.slla1.plist'],
'/System/Library/LaunchDaemons': ['com.apple.slld1.plist'],
'/Users/saltymcsaltface/Library/LaunchAgents': [
'com.apple.uslla1.plist']}
mock_os_walk.side_effect = _get_walk_side_effects(results)
mock_listdir.return_value = ['saltymcsaltface']
mock_isdir.return_value = True
mock_exists.return_value = True
plists = [
{'Label': 'com.apple.lla1'},
{'Label': 'com.apple.lld1'},
{'Label': 'com.apple.slla1'},
{'Label': 'com.apple.slld1'},
{'Label': 'com.apple.uslla1'}]
ret = _run_available_services(plists)
self.assertEqual(len(ret), 5)
@patch('salt.utils.path.os_walk')
@patch('os.path.exists')
@patch('plistlib.readPlist' if six.PY2 else 'plistlib.load')
def test_available_services_broken_symlink(self, mock_read_plist, mock_exists, mock_os_walk):
'''
test available_services
test available_services when it encounters a broken symlink.
'''
mock_os_walk.side_effect = [
[('/Library/LaunchAgents', [], ['com.apple.lla1.plist', 'com.apple.lla2.plist'])],
[('/Library/LaunchDaemons', [], ['com.apple.lld1.plist', 'com.apple.lld2.plist'])],
[('/System/Library/LaunchAgents', [], ['com.apple.slla1.plist', 'com.apple.slla2.plist'])],
[('/System/Library/LaunchDaemons', [], ['com.apple.slld1.plist', 'com.apple.slld2.plist'])],
]
results = {'/Library/LaunchAgents': ['com.apple.lla1.plist', 'com.apple.lla2.plist']}
mock_os_walk.side_effect = _get_walk_side_effects(results)
mock_exists.side_effect = [True, False]
mock_read_plist.side_effect = [
MagicMock(Label='com.apple.lla1'),
MagicMock(Label='com.apple.lla2'),
MagicMock(Label='com.apple.lld1'),
MagicMock(Label='com.apple.lld2'),
MagicMock(Label='com.apple.slld1'),
MagicMock(Label='com.apple.slld2'),
]
plists = [{'Label': 'com.apple.lla1'}]
ret = _run_available_services(plists)
mock_exists.side_effect = [True, True, True, True, False, False, True, True]
ret = mac_utils._available_services()
# Make sure it's a dict with 6 items
self.assertTrue(isinstance(ret, dict))
self.assertEqual(len(ret), 6)
self.assertEqual(
ret['com.apple.lla1']['file_name'],
'com.apple.lla1.plist')
self.assertEqual(
ret['com.apple.lla1']['file_path'],
os.path.realpath(
os.path.join('/Library/LaunchAgents', 'com.apple.lla1.plist')))
self.assertEqual(
ret['com.apple.slld2']['file_name'],
'com.apple.slld2.plist')
self.assertEqual(
ret['com.apple.slld2']['file_path'],
os.path.realpath(
os.path.join('/System/Library/LaunchDaemons', 'com.apple.slld2.plist')))
expected = {
'com.apple.lla1': {
'file_name': 'com.apple.lla1.plist',
'file_path': '/Library/LaunchAgents/com.apple.lla1.plist',
'plist': plists[0]}}
self.assertEqual(ret, expected)
@patch('salt.utils.path.os_walk')
@patch('os.path.exists')
@patch('plistlib.readPlist')
@patch('salt.utils.mac_utils.__salt__')
@patch('plistlib.readPlistFromString' if six.PY2 else 'plistlib.loads')
def test_available_services_non_xml(self,
mock_read_plist_from_string,
mock_run,
mock_read_plist,
mock_exists,
mock_os_walk):
@patch('plistlib.readPlistFromString', create=True)
def test_available_services_binary_plist(self,
mock_read_plist_from_string,
mock_run,
mock_read_plist,
mock_exists,
mock_os_walk):
'''
test available_services
test available_services handles binary plist files.
'''
mock_os_walk.side_effect = [
[('/Library/LaunchAgents', [], ['com.apple.lla1.plist', 'com.apple.lla2.plist'])],
[('/Library/LaunchDaemons', [], ['com.apple.lld1.plist', 'com.apple.lld2.plist'])],
[('/System/Library/LaunchAgents', [], ['com.apple.slla1.plist', 'com.apple.slla2.plist'])],
[('/System/Library/LaunchDaemons', [], ['com.apple.slld1.plist', 'com.apple.slld2.plist'])],
]
attrs = {'cmd.run': MagicMock(return_value='<some xml>')}
def getitem(name):
return attrs[name]
mock_run.__getitem__.side_effect = getitem
mock_run.configure_mock(**attrs)
results = {'/Library/LaunchAgents': ['com.apple.lla1.plist']}
mock_os_walk.side_effect = _get_walk_side_effects(results)
mock_exists.return_value = True
mock_read_plist.side_effect = Exception()
mock_read_plist_from_string.side_effect = [
MagicMock(Label='com.apple.lla1'),
MagicMock(Label='com.apple.lla2'),
MagicMock(Label='com.apple.lld1'),
MagicMock(Label='com.apple.lld2'),
MagicMock(Label='com.apple.slla1'),
MagicMock(Label='com.apple.slla2'),
MagicMock(Label='com.apple.slld1'),
MagicMock(Label='com.apple.slld2'),
]
ret = mac_utils._available_services()
plists = [{'Label': 'com.apple.lla1'}]
cmd = '/usr/bin/plutil -convert xml1 -o - -- "{0}"'
calls = [
call.cmd.run(cmd.format(os.path.realpath(os.path.join(
'/Library/LaunchAgents', 'com.apple.lla1.plist'))),),
call.cmd.run(cmd.format(os.path.realpath(os.path.join(
'/Library/LaunchAgents', 'com.apple.lla2.plist'))),),
call.cmd.run(cmd.format(os.path.realpath(os.path.join(
'/Library/LaunchDaemons', 'com.apple.lld1.plist'))),),
call.cmd.run(cmd.format(os.path.realpath(os.path.join(
'/Library/LaunchDaemons', 'com.apple.lld2.plist'))),),
call.cmd.run(cmd.format(os.path.realpath(os.path.join(
'/System/Library/LaunchAgents', 'com.apple.slla1.plist'))),),
call.cmd.run(cmd.format(os.path.realpath(os.path.join(
'/System/Library/LaunchAgents', 'com.apple.slla2.plist'))),),
call.cmd.run(cmd.format(os.path.realpath(os.path.join(
'/System/Library/LaunchDaemons', 'com.apple.slld1.plist'))),),
call.cmd.run(cmd.format(os.path.realpath(os.path.join(
'/System/Library/LaunchDaemons', 'com.apple.slld2.plist'))),),
]
mock_run.assert_has_calls(calls, any_order=True)
if six.PY2:
attrs = {'cmd.run': MagicMock()}
# Make sure it's a dict with 8 items
self.assertTrue(isinstance(ret, dict))
self.assertEqual(len(ret), 8)
def getitem(name):
return attrs[name]
self.assertEqual(
ret['com.apple.lla1']['file_name'],
'com.apple.lla1.plist')
mock_run.__getitem__.side_effect = getitem
mock_run.configure_mock(**attrs)
cmd = '/usr/bin/plutil -convert xml1 -o - -- "{}"'.format(
'/Library/LaunchAgents/com.apple.lla1.plist')
calls = [call.cmd.run(cmd)]
self.assertEqual(
ret['com.apple.lla1']['file_path'],
os.path.realpath(
os.path.join('/Library/LaunchAgents', 'com.apple.lla1.plist')))
mock_read_plist.side_effect = xml.parsers.expat.ExpatError
mock_read_plist_from_string.side_effect = plists
ret = mac_utils._available_services()
else:
# Py3 plistlib knows how to handle binary plists without
# any extra work, so this test doesn't really do anything
# new.
ret = _run_available_services(plists)
self.assertEqual(
ret['com.apple.slld2']['file_name'],
'com.apple.slld2.plist')
expected = {
'com.apple.lla1': {
'file_name': 'com.apple.lla1.plist',
'file_path': '/Library/LaunchAgents/com.apple.lla1.plist',
'plist': plists[0]}}
self.assertEqual(ret, expected)
self.assertEqual(
ret['com.apple.slld2']['file_path'],
os.path.realpath(
os.path.join('/System/Library/LaunchDaemons', 'com.apple.slld2.plist')))
if six.PY2:
mock_run.assert_has_calls(calls, any_order=True)
@patch('salt.utils.path.os_walk')
@patch('os.path.exists')
@patch('plistlib.readPlist')
def test_available_services_invalid_file(self, mock_exists, mock_os_walk):
'''
test available_services excludes invalid files.
The py3 plistlib raises an InvalidFileException when a plist
file cannot be parsed. This test only asserts things for py3.
'''
if six.PY3:
results = {'/Library/LaunchAgents': ['com.apple.lla1.plist']}
mock_os_walk.side_effect = _get_walk_side_effects(results)
mock_exists.return_value = True
plists = [{'Label': 'com.apple.lla1'}]
mock_load = MagicMock()
mock_load.side_effect = plistlib.InvalidFileException
with patch('salt.utils.files.fopen', mock_open()):
with patch('plistlib.load', mock_load):
ret = mac_utils._available_services()
self.assertEqual(len(ret), 0)
@patch('salt.utils.mac_utils.__salt__')
@patch('plistlib.readPlistFromString' if six.PY2 else 'plistlib.loads')
def test_available_services_non_xml_malformed_plist(self,
mock_read_plist_from_string,
mock_run,
mock_read_plist,
mock_exists,
mock_os_walk):
@patch('plistlib.readPlist')
@patch('salt.utils.path.os_walk')
@patch('os.path.exists')
def test_available_services_expat_error(self,
mock_exists,
mock_os_walk,
mock_read_plist,
mock_run):
'''
test available_services
test available_services excludes files with expat errors.
Poorly formed XML will raise an ExpatError on py2. It will
also be raised by some almost-correct XML on py3.
'''
mock_os_walk.side_effect = [
[('/Library/LaunchAgents', [], ['com.apple.lla1.plist', 'com.apple.lla2.plist'])],
[('/Library/LaunchDaemons', [], ['com.apple.lld1.plist', 'com.apple.lld2.plist'])],
[('/System/Library/LaunchAgents', [], ['com.apple.slla1.plist', 'com.apple.slla2.plist'])],
[('/System/Library/LaunchDaemons', [], ['com.apple.slld1.plist', 'com.apple.slld2.plist'])],
]
attrs = {'cmd.run': MagicMock(return_value='<some xml>')}
def getitem(name):
return attrs[name]
mock_run.__getitem__.side_effect = getitem
mock_run.configure_mock(**attrs)
results = {'/Library/LaunchAgents': ['com.apple.lla1.plist']}
mock_os_walk.side_effect = _get_walk_side_effects(results)
mock_exists.return_value = True
mock_read_plist.side_effect = Exception()
mock_read_plist_from_string.return_value = 'malformedness'
ret = mac_utils._available_services()
if six.PY3:
mock_load = MagicMock()
mock_load.side_effect = xml.parsers.expat.ExpatError
with patch('salt.utils.files.fopen', mock_open()):
with patch('plistlib.load', mock_load):
ret = mac_utils._available_services()
else:
attrs = {'cmd.run': MagicMock()}
cmd = '/usr/bin/plutil -convert xml1 -o - -- "{0}"'
calls = [
call.cmd.run(cmd.format(os.path.realpath(os.path.join(
'/Library/LaunchAgents', 'com.apple.lla1.plist'))),),
call.cmd.run(cmd.format(os.path.realpath(os.path.join(
'/Library/LaunchAgents', 'com.apple.lla2.plist'))),),
call.cmd.run(cmd.format(os.path.realpath(os.path.join(
'/Library/LaunchDaemons', 'com.apple.lld1.plist'))),),
call.cmd.run(cmd.format(os.path.realpath(os.path.join(
'/Library/LaunchDaemons', 'com.apple.lld2.plist'))),),
call.cmd.run(cmd.format(os.path.realpath(os.path.join(
'/System/Library/LaunchAgents', 'com.apple.slla1.plist'))),),
call.cmd.run(cmd.format(os.path.realpath(os.path.join(
'/System/Library/LaunchAgents', 'com.apple.slla2.plist'))),),
call.cmd.run(cmd.format(os.path.realpath(os.path.join(
'/System/Library/LaunchDaemons', 'com.apple.slld1.plist'))),),
call.cmd.run(cmd.format(os.path.realpath(os.path.join(
'/System/Library/LaunchDaemons', 'com.apple.slld2.plist'))),),
]
mock_run.assert_has_calls(calls, any_order=True)
def getitem(name):
return attrs[name]
# Make sure it's a dict with 8 items
self.assertTrue(isinstance(ret, dict))
self.assertEqual(len(ret), 8)
mock_run.__getitem__.side_effect = getitem
mock_run.configure_mock(**attrs)
cmd = '/usr/bin/plutil -convert xml1 -o - -- "{}"'.format(
'/Library/LaunchAgents/com.apple.lla1.plist')
calls = [call.cmd.run(cmd)]
self.assertEqual(
ret['com.apple.lla1.plist']['file_name'],
'com.apple.lla1.plist')
mock_raise_expat_error = MagicMock(
side_effect=xml.parsers.expat.ExpatError)
self.assertEqual(
ret['com.apple.lla1.plist']['file_path'],
os.path.realpath(
os.path.join('/Library/LaunchAgents', 'com.apple.lla1.plist')))
with patch('plistlib.readPlist', mock_raise_expat_error):
with patch('plistlib.readPlistFromString', mock_raise_expat_error):
ret = mac_utils._available_services()
self.assertEqual(
ret['com.apple.slld2.plist']['file_name'],
'com.apple.slld2.plist')
mock_run.assert_has_calls(calls, any_order=True)
self.assertEqual(
ret['com.apple.slld2.plist']['file_path'],
os.path.realpath(
os.path.join('/System/Library/LaunchDaemons', 'com.apple.slld2.plist')))
self.assertEqual(len(ret), 0)
def _get_walk_side_effects(results):
'''
Data generation helper function for service tests.
'''
def walk_side_effect(*args, **kwargs):
return [(args[0], [], results.get(args[0], []))]
return walk_side_effect
def _run_available_services(plists):
if six.PY2:
mock_read_plist = MagicMock()
mock_read_plist.side_effect = plists
with patch('plistlib.readPlist', mock_read_plist):
ret = mac_utils._available_services()
else:
mock_load = MagicMock()
mock_load.side_effect = plists
with patch('salt.utils.files.fopen', mock_open()):
with patch('plistlib.load', mock_load):
ret = mac_utils._available_services()
return ret