Merge https://github.com/saltstack/salt into develop

doc/contents.rst

@@ -35,7 +35,7 @@ Full Table of Contents
     ref/runners
     ref/peer
     ref/syndic
-    ref/Python-api
+    ref/python-api
     ref/file_server/index
     ref/file_server/file_roots
     ref/file_server/dynamic-modules

doc/ref/file_server/dynamic-modules.rst

@@ -28,11 +28,11 @@ which defaults to True. This option makes the state system refresh all
 dynamic modules when states are run. To disable this behavior set
 ``autoload_dynamic_modules`` to False in the minion config.
 
-When dynamic modules are autoloaded via states, modules only pertinant to
-the environmetns matched in the master's top file are downloaded.
+When dynamic modules are autoloaded via states, modules only pertinent to
+the environments matched in the master's top file are downloaded.
 
 This is important to remember, because modules can be manually loaded from
-any specific environment that environment sopecific modules will be loaded
+any specific environment that environment specific modules will be loaded
 when a state run is executed.
 
 Sync Via the saltutil Module

doc/ref/python-api.rst

@@ -61,6 +61,12 @@ same interface used by the ``salt`` command line tool.
     * 'glob' - Bash glob completion - Default
     * 'pcre' - Perl style regular expression
     * 'list' - Python list of hosts
+    * 'grain' - Match based on a grain comparison
+    * 'grain_pcre' - Grain comparison with a regex
+    * 'pillar' - Pillar data comparison
+    * 'nodegroup' - Match on nodegroup
+    * 'range' - Use a Range server for matching
+    * 'compound' - Pass a compound match string
 
 .. cmdoption:: ret
 
@@ -102,6 +108,12 @@ same interface used by the ``salt`` command line tool.
     * 'glob' - Bash glob completion - Default
     * 'pcre' - Perl style regular expression
     * 'list' - Python list of hosts
+    * 'grain' - Match based on a grain comparison
+    * 'grain_pcre' - Grain comparison with a regex
+    * 'pillar' - Pillar data comparison
+    * 'nodegroup' - Match on nodegroup
+    * 'range' - Use a Range server for matching
+    * 'compound' - Pass a compound match string
 
 .. cmdoption:: ret
 
@@ -146,6 +158,12 @@ same interface used by the ``salt`` command line tool.
     * 'glob' - Bash glob completion - Default
     * 'pcre' - Perl style regular expression
     * 'list' - Python list of hosts
+    * 'grain' - Match based on a grain comparison
+    * 'grain_pcre' - Grain comparison with a regex
+    * 'pillar' - Pillar data comparison
+    * 'nodegroup' - Match on nodegroup
+    * 'range' - Use a Range server for matching
+    * 'compound' - Pass a compound match string
 
 .. cmdoption:: ret
 
@@ -187,6 +205,12 @@ same interface used by the ``salt`` command line tool.
     * 'glob' - Bash glob completion - Default
     * 'pcre' - Perl style regular expression
     * 'list' - Python list of hosts
+    * 'grain' - Match based on a grain comparison
+    * 'grain_pcre' - Grain comparison with a regex
+    * 'pillar' - Pillar data comparison
+    * 'nodegroup' - Match on nodegroup
+    * 'range' - Use a Range server for matching
+    * 'compound' - Pass a compound match string
 
 .. cmdoption:: ret
 

doc/topics/pillar/index.rst

@@ -127,3 +127,13 @@ locally. This is done with the ``saltutil.refresh_pillar`` function.
 .. code-block:: yaml
 
     salt '*' saltutil.refresh_pillar
+
+Targeting with Pillar
+=====================
+
+Pillar data can be used when targeting minions. This allows for ultimate
+control and flexibility when targeting minions.
+
+.. code-block:: bash
+
+    salt -I 'somekey:specialvalue' test.ping

doc/topics/targeting/compound.rst

@@ -20,6 +20,7 @@ G      Grains glob match    ``G@os:Ubuntu``
 E      PCRE Minion id match ``E@web\d+\.(dev|qa|prod)\.loc``
 P      Grains PCRE match    ``P@os:(RedHat|Fedora|CentOS)``
 L      List of minions      ``L@minion1.example.com,minion3.domain.com and bl*.domain.com``
+I      Pillar glob match    ``I@pdata:foobar``
 ====== ==================== ===============================================================
 
 Matchers can be joined using boolean ``and``, ``or``, and ``not`` operators.
@@ -39,4 +40,4 @@ That same example expressed in a :term:`top file` looks like the following::
         - webserver
 
 .. _`glob`: http://docs.python.org/library/fnmatch.html
-.. _`regular expressions`: http://docs.python.org/library/re.html#module-re
+.. _`regular expressions`: http://docs.python.org/library/re.html#module-re

salt/minion.py

@@ -655,6 +655,8 @@ class Matcher(object):
         '''
         Determines if this host is on the list
         '''
+        if isinstance(tgt, basestring):
+            tgt = tgt.split(',')
         return bool(self.opts['id'] in tgt)
 
     def grain_match(self, tgt):

salt/modules/sqlite3.py

@@ -0,0 +1,67 @@
+'''
+Support for SQLite3
+'''
+
+try:
+    import sqlite3
+    has_sqlite3 = True
+except ImportError:
+    has_sqlite3 = False
+
+def __virtual__():
+    if not has_sqlite3:
+        return False
+    return 'sqlite3'
+
+def version():
+    '''
+    Return version of pysqlite
+
+    CLI Example::
+
+        salt '*' sqlite3.version
+    '''
+    return sqlite3.version
+
+def sqlite_version():
+    '''
+    Return version of sqlite
+
+    CLI Example::
+
+        salt '*' sqlite3.sqlite_version
+    '''
+    return sqlite3.sqlite_version
+
+def query(db=None, sql=None):
+    '''
+    Issue an SQL query to sqlite3 (with no return data)
+
+    CLI Example::
+
+        salt '*' sqlite3.query /root/test.db 'CREATE TABLE test(id INT, testdata TEXT);'
+    '''
+    if db == None:
+        return False
+
+    con = sqlite3.connect(db)
+    cur = con.cursor()
+    cur.execute(sql)
+    return True
+
+def select(db=None, sql=None):
+    '''
+    SELECT data from an sqlite3 db (returns all rows, be careful!)
+
+    CLI Example::
+
+        salt '*' sqlite3.select /root/test.db 'SELECT * FROM test;'
+    '''
+    if db == None:
+        return False
+
+    con = sqlite3.connect(db)
+    cur = con.cursor()
+    cur.execute(sql)
+    rows = cur.fetchall()
+    return rows

salt/modules/ssh.py

@@ -547,7 +547,7 @@ def set_known_host(user, hostname,
     uinfo = __salt__['user.info'](user)
     full = os.path.join(uinfo['home'], config)
     line = '{hostname} {enc} {key}\n'.format(**remote_host)
-    with open(full, 'w') as fd:
+    with open(full, 'a') as fd:
         fd.write(line)
     return {'status': 'updated', 'old': stored_host, 'new': remote_host}
 

tests/integration/__init__.py

@@ -39,6 +39,8 @@ class TestDaemon(object):
             os.path.join(INTEGRATION_TEST_DIR, 'files', 'conf', 'master'))
         self.minion_opts = salt.config.minion_config(
             os.path.join(INTEGRATION_TEST_DIR, 'files', 'conf', 'minion'))
+        self.sub_minion_opts = salt.config.minion_config(
+            os.path.join(INTEGRATION_TEST_DIR, 'files', 'conf', 'sub_minion'))
         self.smaster_opts = salt.config.master_config(
             os.path.join(INTEGRATION_TEST_DIR, 'files', 'conf', 'syndic_master'))
         self.syndic_opts = salt.config.minion_config(
@@ -77,6 +79,8 @@ class TestDaemon(object):
                     os.path.join(self.smaster_opts['cachedir'], 'jobs'),
                     os.path.dirname(self.master_opts['log_file']),
                     self.minion_opts['extension_modules'],
+                    self.sub_minion_opts['extension_modules'],
+                    self.sub_minion_opts['pki_dir'],
                     self.master_opts['sock_dir'],
                     self.smaster_opts['sock_dir'],
                     ])
@@ -89,6 +93,11 @@ class TestDaemon(object):
         self.minion_process = multiprocessing.Process(target=minion.tune_in)
         self.minion_process.start()
 
+        sub_minion = salt.minion.Minion(self.sub_minion_opts)
+        self.sub_minion_process = multiprocessing.Process(
+                target=sub_minion.tune_in)
+        self.sub_minion_process.start()
+
         smaster = salt.master.Master(self.smaster_opts)
         self.smaster_process = multiprocessing.Process(target=smaster.start)
         self.smaster_process.start()
@@ -103,6 +112,7 @@ class TestDaemon(object):
         '''
         Kill the minion and master processes
         '''
+        self.sub_minion_process.terminate()
         self.minion_process.terminate()
         self.master_process.terminate()
         self.syndic_process.terminate()
@@ -230,7 +240,7 @@ class ShellCase(TestCase):
 
     def run_salt(self, arg_str):
         '''
-        Execute salt-key
+        Execute salt
         '''
         mconf = os.path.join(INTEGRATION_TEST_DIR, 'files', 'conf', 'master')
         arg_str = '-c {0} {1}'.format(mconf, arg_str)
@@ -238,7 +248,7 @@ class ShellCase(TestCase):
 
     def run_run(self, arg_str):
         '''
-        Execute salt-key
+        Execute salt-run
         '''
         mconf = os.path.join(INTEGRATION_TEST_DIR, 'files', 'conf', 'master')
         arg_str = '-c {0} {1}'.format(mconf, arg_str)

tests/integration/files/conf/minion

@@ -5,7 +5,7 @@ root_dir: /tmp/salttest
 pki_dir: pki
 id: minion
 cachedir: cachedir
-acceptance_wait_time: = 1
+#acceptance_wait_time: = 1
 open_mode: True
 log_file: minion
 

tests/integration/files/conf/sub_minion

@@ -0,0 +1,21 @@
+# basic config
+master: localhost
+master_port: 64506
+root_dir: /tmp/subsalttest
+pki_dir: pki
+id: sub_minion
+cachedir: cachedir
+#acceptance_wait_time: 1
+open_mode: True
+log_file: minion
+
+# module extension
+test.foo: baz
+hosts.file: integration/tmp/hosts
+aliases.file: integration/tmp/aliases
+integration.test: True
+
+# Grains addons
+grains:
+  test_grain: spam
+  role: sub

tests/integration/files/pillar/base/sub.sls

@@ -0,0 +1 @@
+sub: sub_minion

tests/integration/files/pillar/base/top.sls

@@ -1,3 +1,6 @@
 base:
-  '*':
+  'minion':
     - generic
+  'sub_minion':
+    - generic
+    - sub

tests/integration/shell/key.py

@@ -20,6 +20,7 @@ class KeyTest(integration.ShellCase):
                 '\x1b[1;31mUnaccepted Keys:\x1b[0m',
                 '\x1b[1;32mAccepted Keys:\x1b[0m',
                 '\x1b[0;32mminion\x1b[0m',
+                '\x1b[0;32msub_minion\x1b[0m',
                 '\x1b[1;34mRejected:\x1b[0m', '']
         self.assertEqual(data, expect)
 
@@ -30,7 +31,11 @@ class KeyTest(integration.ShellCase):
         data = self.run_key('-l acc')
         self.assertEqual(
                 data,
-                ['\x1b[0;32mminion\x1b[0m', '']
+                [
+                    '\x1b[0;32mminion\x1b[0m',
+                    '\x1b[0;32msub_minion\x1b[0m',
+                    ''
+                    ]
                 )
 
     def test_list_un(self):

tests/integration/shell/matcher.py

@@ -0,0 +1,114 @@
+# Import python libs
+import sys
+
+# Import salt libs
+from saltunittest import TestLoader, TextTestRunner
+import integration
+from integration import TestDaemon
+
+
+class MatchTest(integration.ShellCase):
+    '''
+    Test salt matchers
+    '''
+    def test_list(self):
+        '''
+        test salt -L matcher
+        '''
+        data = self.run_salt('-L minion test.ping')
+        data = '\n'.join(data)
+        self.assertIn('minion', data)
+        self.assertNotIn('sub_minion', data)
+        data = self.run_salt('-L minion,sub_minion test.ping')
+        data = '\n'.join(data)
+        self.assertIn('minion', data)
+        self.assertIn('sub_minion', data)
+
+    def test_glob(self):
+        '''
+        test salt glob matcher
+        '''
+        data = self.run_salt('minion test.ping')
+        data = '\n'.join(data)
+        self.assertIn('minion', data)
+        self.assertNotIn('sub_minion', data)
+        data = self.run_salt('"*" test.ping')
+        data = '\n'.join(data)
+        self.assertIn('minion', data)
+        self.assertIn('sub_minion', data)
+
+    def test_regex(self):
+        '''
+        test salt regex matcher
+        '''
+        data = self.run_salt('-E "^minion$" test.ping')
+        data = '\n'.join(data)
+        self.assertIn('minion', data)
+        self.assertNotIn('sub_minion', data)
+        data = self.run_salt('-E ".*" test.ping')
+        data = '\n'.join(data)
+        self.assertIn('minion', data)
+        self.assertIn('sub_minion', data)
+
+    def test_grain(self):
+        '''
+        test salt grain matcher
+        '''
+        data = self.run_salt('-t 1 -G "test_grain:cheese" test.ping')
+        data = '\n'.join(data)
+        self.assertIn('minion', data)
+        self.assertNotIn('sub_minion', data)
+        data = self.run_salt('-G "test_grain:spam" test.ping')
+        data = '\n'.join(data)
+        self.assertIn('sub_minion', data)
+        self.assertNotIn('minion', data.replace('sub_minion', 'stub'))
+            
+    def test_regrain(self):
+        '''
+        test salt grain matcher
+        '''
+        data = self.run_salt('-t 1 --grain-pcre "test_grain:^cheese$" test.ping')
+        data = '\n'.join(data)
+        self.assertIn('minion', data)
+        self.assertNotIn('sub_minion', data)
+        data = self.run_salt('--grain-pcre "test_grain:.*am$" test.ping')
+        data = '\n'.join(data)
+        self.assertIn('sub_minion', data)
+        self.assertNotIn('minion', data.replace('sub_minion', 'stub'))
+
+    def test_pillar(self):
+        '''
+        test pillar matcher
+        '''
+        data = self.run_salt('-I "monty:python" test.ping')
+        data = '\n'.join(data)
+        self.assertIn('minion', data)
+        self.assertIn('sub_minion', data)
+        data = self.run_salt('-I "sub:sub_minion" test.ping')
+        data = '\n'.join(data)
+        self.assertIn('sub_minion', data)
+        self.assertNotIn('minion', data.replace('sub_minion', 'stub'))
+
+    def test_compound(self):
+        '''
+        test compound matcher
+        '''
+        match = 'P@test_grain:^cheese$ and * and G@test_grain:cheese'
+        data = self.run_salt('-t 1 -C \'{0}\' test.ping'.format(match))
+        data = '\n'.join(data)
+        self.assertIn('minion', data)
+        self.assertNotIn('sub_minion', data)
+        match = 'L@sub_minion and E@.*'
+        data = self.run_salt('-t 1 -C "{0}" test.ping'.format(match))
+        data = '\n'.join(data)
+        self.assertIn('sub_minion', data)
+        self.assertNotIn('minion', data.replace('sub_minion', 'stub'))
+
+
+if __name__ == "__main__":
+    loader = TestLoader()
+    tests = loader.loadTestsFromTestCase(KeyTest)
+    print('Setting up Salt daemons to execute tests')
+    with TestDaemon():
+        runner = TextTestRunner(verbosity=1).run(tests)
+        sys.exit(runner.wasSuccessful())

tests/integration/states/ssh.py

@@ -8,6 +8,7 @@ import integration
 
 KNOWN_HOSTS = os.path.join(integration.TMP, 'known_hosts')
 GITHUB_FINGERPRINT = '16:27:ac:a5:76:28:2d:36:63:1b:56:4d:eb:df:a6:48'
+GITHUB_IP = '207.97.227.239'
 
 
 class SSHKnownHostsStateTest(integration.ModuleCase):
@@ -44,6 +45,18 @@ class SSHKnownHostsStateTest(integration.ModuleCase):
         _ret = self.run_state('ssh_known_hosts.present', test=True, **kwargs)
         ret = _ret.values()[0]
         self.assertEqual(ret['result'], None, ret)
+        # then add a record for IP address
+        _ret = self.run_state('ssh_known_hosts.present',
+                              **dict(kwargs, name=GITHUB_IP ))
+        ret = _ret.values()[0]
+        self.assertTrue(ret['result'], ret)
+        # record for every host must be available
+        ret = self.run_function('ssh.get_known_host', ['root', 'github.com'],
+                                config=KNOWN_HOSTS)
+        self.assertFalse(ret is None)
+        ret = self.run_function('ssh.get_known_host', ['root', GITHUB_IP],
+                                config=KNOWN_HOSTS)
+        self.assertFalse(ret is None)
 
     def test_present_fail(self):
         # save something wrong
@@ -81,3 +94,17 @@ class SSHKnownHostsStateTest(integration.ModuleCase):
         _ret = self.run_state('ssh_known_hosts.absent', test=True, **kwargs)
         ret = _ret.values()[0]
         self.assertEqual(ret['result'], None, ret)
+
+
+if __name__ == "__main__":
+    import sys
+    from saltunittest import TestLoader, TextTestRunner
+    from integration import TestDaemon
+
+
+    loader = TestLoader()
+    tests = loader.loadTestsFromTestCase(SSHKnownHostsStateTest)
+    print('Setting up Salt daemons to execute tests')
+    with TestDaemon():
+        runner = TextTestRunner(verbosity=1).run(tests)
+        sys.exit(runner.wasSuccessful())