Add scripted parsing of all mysql options

README.rst

@@ -6,8 +6,8 @@ Install the MySQL client and/or server.
 
 .. note::
 
-    See the full `Salt Formulas installation and usage instructions
-    <http://docs.saltstack.com/en/latest/topics/development/conventions/formulas.html>`_.
+   See the full `Salt Formulas installation and usage instructions
+   <http://docs.saltstack.com/en/latest/topics/development/conventions/formulas.html>`_.
 
 Available states
 ================
@@ -53,14 +53,41 @@ Create and manage MySQL databases.
 Install mysql python bindings.
 
 ``mysql.user``
-----------------
+--------------
 
 Create and manage MySQL database users with definable GRANT privileges.
 
-The state accepts MySQL hashed passwords or clear text. Hashed password have priority.
+The state accepts MySQL hashed passwords or clear text. Hashed password have
+priority.
 
 .. note::
-    See the `salt.states.mysql_user <http://docs.saltstack.com/en/latest/ref/states/all/salt.states.mysql_user.html#module-salt.states.mysql_user>`_ docs for additional information on configuring hashed passwords.
-    
+    See the `salt.states.mysql_user
+    <http://docs.saltstack.com/en/latest/ref/states/all/salt.states.mysql_user.html#module-salt.states.mysql_user>`_
+    docs for additional information on configuring hashed passwords.
+
     Make sure to **quote the passwords** in the pillar so YAML doesn't throw an exception.
 
+
+Updating the supported parameters
+=================================
+
+The ``supported_params.yaml`` file contains the full listing of options that
+are acceptable in the MySQL options file.  On occassion, especially on new
+releases of MySQL, this file may need to be updated.  To update, run the
+supplied script (requires Python 3.x)::
+
+    ./scripts/parse_supported_params.py -o ./mysql/supported_params.yaml
+
+This script will scrape the options from the official MySQL documentation
+online, and thus requires web access.  Scraping is inherently brittle, though
+this script has been defensively coded, where possible.
+
+Once the ``supported_params.yaml`` file has been updated, commit the result to
+the repository.
+
+Support for new applications
+----------------------------
+
+To add support for configuration of other MySQL applications, add the URL and
+section identifier into the relevant section of the script.  Consult the
+comments in the code to determine where your section should be added.

scripts/html_table_parser.py

@@ -0,0 +1,61 @@
+#-------------------------------------------------------------------------------
+# Name:        html_table_parser
+# Purpose:     Simple class for parsing an (x)html string to extract tables.
+#              Written in python3
+#
+# Author:      Josua Schmid
+#
+# Created:     05.03.2014
+# Copyright:   (c) Josua Schmid 2014
+# Licence:     GPLv3
+#-------------------------------------------------------------------------------
+from html.parser import HTMLParser
+
+class HTMLTableParser(HTMLParser):
+    """ This class serves as a html table parser. It is able to parse multiple
+    tables which you feed in. You can access the result per .tables field.
+    """
+    def __init__(self):
+        HTMLParser.__init__(self)
+        self.__in_td = False
+        self.__in_th = False
+        self.__current_table = []
+        self.__current_row = []
+        self.__current_cell = []
+        self.tables = []
+
+    def handle_starttag(self, tag, attrs):
+        """ We need to remember the opening point for the content of interest.
+        The other tags (<table>, <tr>) are only handled at the closing point.
+        """
+        if tag == 'td':
+            self.__in_td = True
+        if tag == 'th':
+            self.__in_th = True
+
+    def handle_data(self, data):
+        """ This is where we save content to a cell """
+        if self.__in_td ^ self.__in_th:
+            self.__current_cell.append(data.strip())
+
+    def handle_endtag(self, tag):
+        """ Here we exit the tags. If the closing tag is </tr>, we know that we
+        can save our currently parsed cells to the current table as a row and
+        prepare for a new row. If the closing tag is </table>, we save the
+        current table and prepare for a new one.
+        """
+        if tag == 'td':
+            self.__in_td = False
+        if tag == 'th':
+            self.__in_th = False
+
+        if (tag == 'td') ^ (tag == 'th'):
+            final_cell = " ".join(self.__current_cell).strip()
+            self.__current_row.append(final_cell)
+            self.__current_cell = []
+        if tag == 'tr':
+            self.__current_table.append(self.__current_row)
+            self.__current_row = []
+        if tag == 'table':
+            self.tables.append(self.__current_table)
+            self.__current_table = []

scripts/parse_supported_params.py

@@ -0,0 +1,167 @@
+#!/usr/bin/python3
+# coding: utf-8
+import argparse
+import re
+import sys
+import urllib.request
+from html_table_parser import HTMLTableParser
+
+# Regex for parsing options on MySQL documentation pages
+# Options are (normally) specified as command-line options
+# as anchor tags on the page.  Certain documentation pages only
+# show options in table listings, however.
+OPTION_REGEX = '<a name="option_%s_(.*?)"></a>'
+OPTION_TABLE_REGEX = '^(--)?([A-Za-z_-]+).*$'
+
+# File heading, as per the original supported_params file
+FILE_HEADER = """# vim
+{#- Do not edit this YAML file by hand. See README.rst for how to update -#}
+{% load_yaml as supported_params %}
+"""
+FILE_FOOTER = """{% endload %}"""
+
+# Standard YAML template for options for a section
+YAML_TEMPLATE = """# From %(url)s
+%(section)s:
+  - %(options)s
+"""
+
+# For rendering Jinja that handles multiple sections
+# Several MySQL utilities use exactly the same options
+# Note this variable is string formatted twice, hence the double-double % signs
+YAML_TEMPLATE_MULTI = """# From %%(url)s
+{%%%% for section in %(sections)r %%%%}
+{{ section  }}:
+  - %%(options)s
+{%%%% endfor %%%%}
+"""
+
+# Options specified in HTML documentation as command-line options
+# like so <a name="option_mysql_help"></a>.
+# Structure is (section_id, documentation_url, yaml_template_str)
+SECTIONS = (
+    ('mysql',
+     'https://dev.mysql.com/doc/refman/5.7/en/mysql-command-options.html',
+     YAML_TEMPLATE_MULTI % {'sections': ['client', 'mysql']}),
+    ('mysqldump',
+     'https://dev.mysql.com/doc/refman/5.7/en/mysqldump.html',
+     YAML_TEMPLATE),
+    ('mysqld_safe',
+     'https://dev.mysql.com/doc/refman/5.7/en/mysqld-safe.html',
+     YAML_TEMPLATE),
+    # Removed in MySQL 5.7
+    ('mysqlhotcopy',
+     'http://dev.mysql.com/doc/refman/5.6/en/mysqlhotcopy.html',
+     YAML_TEMPLATE),
+    ('mysqladmin',
+     'http://dev.mysql.com/doc/refman/5.7/en/mysqladmin.html',
+     YAML_TEMPLATE),
+    ('mysqlcheck',
+     'http://dev.mysql.com/doc/refman/5.7/en/mysqlcheck.html',
+     YAML_TEMPLATE),
+    ('mysqlimport',
+     'http://dev.mysql.com/doc/refman/5.7/en/mysqlimport.html',
+     YAML_TEMPLATE),
+    ('mysqlshow',
+     'http://dev.mysql.com/doc/refman/5.7/en/mysqlshow.html',
+     YAML_TEMPLATE),
+    ('myisampack',
+     'http://dev.mysql.com/doc/refman/5.7/en/myisampack.html',
+     YAML_TEMPLATE),
+)
+# Options specified in documentation as command-line and
+# option file values in a table only.
+SECTIONS_VIA_TABLE = (
+    ('myisamchk',
+     'https://dev.mysql.com/doc/refman/5.7/en/myisamchk.html',
+     YAML_TEMPLATE_MULTI % {'sections': ['myisamchk', 'isamchk']}),
+)
+# Server options specified in documentation
+SERVER_OPTIONS = (
+    'mysqld',
+    'https://dev.mysql.com/doc/refman/5.7/en/mysqld-option-tables.html',
+    YAML_TEMPLATE
+)
+
+
+def read_url(url):
+    """ Read the given URL and decode the response as UTF-8.
+    """
+    request = urllib.request.Request(url)
+    response = urllib.request.urlopen(request)
+    return response.read().decode('utf-8')
+
+
+def read_first_table(url):
+    """ Read the given URL, parse the result, and return the first table.
+    """
+    xhtml = read_url(url)
+    parser = HTMLTableParser()
+    parser.feed(xhtml)
+    return parser.tables[0]  # Use first table on the page
+
+
+def parse_anchors(url, section):
+    """ Return parsed options from option anchors at the given URL.
+    """
+    return re.findall(OPTION_REGEX % section, read_url(url))
+
+
+def parse_tables(url, section):
+    """ Return arsed options from HTML tables at the given URL.
+
+    This matches the given option regex, and ensures that the
+    first row of the table is ignored; it contains headings only.
+    """
+    table = read_first_table(url)
+    return [re.match(OPTION_TABLE_REGEX, row[0]).groups()[1]
+            for row in table[1:]]
+
+
+def parse_mysqld(url, section):
+    """ Return the parsed options from the huge mysqld table.
+
+    The massive options table shows variables and options and
+    highlights where they can be used.  The following code only
+    pulls out those that are marked as 'Yes' for use in an option file.
+    """
+    table = read_first_table(url)
+    # Find which column holds the option file data
+    option_index = table[0].index('Option File')
+    # Only pull out options able to be used in an options file
+    return [re.match(OPTION_TABLE_REGEX, row[0]).groups()[1]
+            for row in table[1:]
+            if len(row) >= option_index + 1 and
+            row[option_index].strip().lower() == 'yes']
+
+
+def print_yaml_options(sections, parser, file=sys.stdout):
+    """ Perform really basic templating for output.
+
+    A YAML library could be used, but we avoid extra dependencies by
+    just using string formatting.
+    """
+    for section, url, yaml in sections:
+        options = parser(url, section)
+        print(yaml % {'section': section,
+                      'options': '\n  - '.join(options),
+                      'url': url}, end='', file=file)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        description='Scrape the MySQL documentation to obtain'
+                    ' all the supported parameters for different utilities.')
+    parser.add_argument('--output',
+                        '-o',
+                        help='File output location',
+                        default=sys.stdout)
+    config = parser.parse_args()
+    output = open(config.output, 'w') if isinstance(config.output, str) \
+        else config.output
+
+    print(FILE_HEADER, end='', file=output)
+    print_yaml_options(SECTIONS, parse_anchors, file=output)
+    print_yaml_options(SECTIONS_VIA_TABLE, parse_tables, file=output)
+    print_yaml_options((SERVER_OPTIONS,), parse_mysqld, file=output)
+    print(FILE_FOOTER, end='', file=output)