Merge branch 'master' of https://github.com/saltstack-formulas/mysql-formula into feature/mysql-dev

Conflicts:
	README.rst
This commit is contained in:
Alex Hayes 2015-03-10 07:38:55 +11:00
commit 5546542b19
17 changed files with 259 additions and 1325 deletions

2
.gitignore vendored Normal file
View file

@ -0,0 +1,2 @@
*.pyc
*~

View file

@ -45,6 +45,11 @@ debconf.
cryptographically insecure, future formula versions should use the cryptographically insecure, future formula versions should use the
newly available ``random.get_str`` method. newly available ``random.get_str`` method.
``mysql.disabled``
------------------
Ensure that the MySQL service is not running.
``mysql.database`` ``mysql.database``
------------------ ------------------
@ -91,26 +96,3 @@ Install the MySQL development libraries and header files.
Note that this state is not installed by the mysql meta-state unless you set Note that this state is not installed by the mysql meta-state unless you set
your pillar data accordingly. your pillar data accordingly.
Updating the supported parameters
=================================
The ``supported_params.yaml`` file contains the full listing of options that
are acceptable in the MySQL options file. On occassion, especially on new
releases of MySQL, this file may need to be updated. To update, run the
supplied script (requires Python 3.x)::
./scripts/parse_supported_params.py -o ./mysql/supported_params.yaml
This script will scrape the options from the official MySQL documentation
online, and thus requires web access. Scraping is inherently brittle, though
this script has been defensively coded, where possible.
Once the ``supported_params.yaml`` file has been updated, commit the result to
the repository.
Support for new applications
----------------------------
To add support for configuration of other MySQL applications, add the URL and
section identifier into the relevant section of the script. Consult the
comments in the code to determine where your section should be added.

View file

@ -1,7 +1,6 @@
{% from "mysql/defaults.yaml" import rawmap with context %} {% from "mysql/defaults.yaml" import rawmap with context %}
{%- set mysql = salt['grains.filter_by'](rawmap, grain='os', merge=salt['pillar.get']('mysql:server:lookup')) %} {%- set mysql = salt['grains.filter_by'](rawmap, grain='os', merge=salt['pillar.get']('mysql:lookup')) %}
mysql: mysql:
pkg: pkg.installed:
- installed
- name: {{ mysql.client }} - name: {{ mysql.client }}

View file

@ -1,5 +1,5 @@
{% from "mysql/defaults.yaml" import rawmap with context %} {% from "mysql/defaults.yaml" import rawmap with context %}
{%- set mysql = salt['grains.filter_by'](rawmap, grain='os', merge=salt['pillar.get']('mysql:server:lookup')) %} {%- set mysql = salt['grains.filter_by'](rawmap, grain='os', merge=salt['pillar.get']('mysql:lookup')) %}
{% set mysql_root_pass = salt['pillar.get']('mysql:server:root_password', salt['grains.get']('server_id')) %} {% set mysql_root_pass = salt['pillar.get']('mysql:server:root_password', salt['grains.get']('server_id')) %}
{% set db_states = [] %} {% set db_states = [] %}

View file

@ -1,4 +1,7 @@
# vim: sts=2 ts=2 sw=2 et ai # vim: sts=2 ts=2 sw=2 et ai
#
# SET ALL PARAMS IN CONFIG SECTION USING UNDERSCORE, NOT HYPHEN
# so that it works correctly
{% load_yaml as rawmap %} {% load_yaml as rawmap %}
Ubuntu: Ubuntu:
server: mysql-server server: mysql-server
@ -18,20 +21,20 @@ Ubuntu:
nice: 0 nice: 0
mysqld: mysqld:
user: mysql user: mysql
pid-file: /var/run/mysqld/mysqld.pid pid_file: /var/run/mysqld/mysqld.pid
socket: /var/run/mysqld/mysqld.sock socket: /var/run/mysqld/mysqld.sock
port: 3306 port: 3306
basedir: /usr basedir: /usr
datadir: /var/lib/mysql datadir: /var/lib/mysql
tmpdir: /tmp tmpdir: /tmp
lc-messages-dir: /usr/share/mysql lc_messages_dir: /usr/share/mysql
skip-external-locking: noarg_present skip_external_locking: noarg_present
bind-address: 127.0.0.1 bind_address: 127.0.0.1
key_buffer_size: 16M key_buffer_size: 16M
max_allowed_packet: 16M max_allowed_packet: 16M
thread_stack: 192K thread_stack: 192K
thread_cache_size: 8 thread_cache_size: 8
myisam-recover: BACKUP myisam_recover: BACKUP
query_cache_limit: 1M query_cache_limit: 1M
query_cache_size: 16M query_cache_size: 16M
log_error: /var/log/mysql/error.log log_error: /var/log/mysql/error.log
@ -39,10 +42,10 @@ Ubuntu:
max_binlog_size: 100M max_binlog_size: 100M
mysqldump: mysqldump:
quick: noarg_present quick: noarg_present
quote-names: noarg_present quote_names: noarg_present
max_allowed_packet: 16M max_allowed_packet: 16M
isamchk: isamchk:
key_buffer: 16M key_buffer_size: 16M
append: | append: |
!includedir /etc/mysql/conf.d/ !includedir /etc/mysql/conf.d/
Debian: Debian:
@ -63,30 +66,30 @@ Debian:
nice: 0 nice: 0
mysqld: mysqld:
user: mysql user: mysql
pid-file: /var/run/mysqld/mysqld.pid pid_file: /var/run/mysqld/mysqld.pid
socket: /var/run/mysqld/mysqld.sock socket: /var/run/mysqld/mysqld.sock
port: 3306 port: 3306
basedir: /usr basedir: /usr
datadir: /var/lib/mysql datadir: /var/lib/mysql
tmpdir: /tmp tmpdir: /tmp
lc-messages-dir: /usr/share/mysql lc_messages_dir: /usr/share/mysql
skip-external-locking: noarg_present skip_external_locking: noarg_present
bind-address: 127.0.0.1 bind_address: 127.0.0.1
key_buffer_size: 16M key_buffer_size: 16M
max_allowed_packet: 16M max_allowed_packet: 16M
thread_stack: 192K thread_stack: 192K
thread_cache_size: 8 thread_cache_size: 8
myisam-recover: BACKUP myisam_recover: BACKUP
query_cache_limit: 1M query_cache_limit: 1M
query_cache_size: 16M query_cache_size: 16M
expire_logs_days: 10 expire_logs_days: 10
max_binlog_size: 100M max_binlog_size: 100M
mysqldump: mysqldump:
quick: noarg_present quick: noarg_present
quote-names: noarg_present quote_names: noarg_present
max_allowed_packet: 16M max_allowed_packet: 16M
isamchk: isamchk:
key_buffer: 16M key_buffer_size: 16M
append: | append: |
!includedir /etc/mysql/conf.d/ !includedir /etc/mysql/conf.d/
CentOS: CentOS:
@ -99,15 +102,15 @@ CentOS:
file: /etc/my.cnf file: /etc/my.cnf
sections: sections:
mysqld_safe: mysqld_safe:
log-error: /var/log/mysqld.log log_error: /var/log/mysqld.log
pid-file: /var/run/mysqld/mysqld.pid pid_file: /var/run/mysqld/mysqld.pid
mysqld: mysqld:
datadir: /var/lib/mysql datadir: /var/lib/mysql
socket: /var/lib/mysql/mysql.sock socket: /var/lib/mysql/mysql.sock
user: mysql user: mysql
port: 3306 port: 3306
bind-address: 127.0.0.1 bind_address: 127.0.0.1
symbolic-links: 0 symbolic_links: 0
RedHat: RedHat:
server: mysql-server server: mysql-server
client: mysql client: mysql
@ -118,15 +121,33 @@ RedHat:
file: /etc/my.cnf file: /etc/my.cnf
sections: sections:
mysqld_safe: mysqld_safe:
log-error: /var/log/mysqld.log log_error: /var/log/mysqld.log
pid-file: /var/run/mysqld/mysqld.pid pid_file: /var/run/mysqld/mysqld.pid
mysqld: mysqld:
datadir: /var/lib/mysql datadir: /var/lib/mysql
socket: /var/lib/mysql/mysql.sock socket: /var/lib/mysql/mysql.sock
user: mysql user: mysql
port: 3306 port: 3306
bind-address: 127.0.0.1 bind_address: 127.0.0.1
symbolic-links: 0 symbolic_links: 0
Fedora:
server: mariadb-server
client: mariadb
service: mariadb
python: MySQL-python
config:
file: /etc/my.cnf.d/server.cnf
sections:
mysqld_safe:
log_error: /var/log/mysqld.log
pid_file: /var/run/mysqld/mysqld.pid
mysqld:
datadir: /var/lib/mysql
socket: /var/lib/mysql/mysql.sock
user: mysql
port: 3306
bind_address: 127.0.0.1
symbolic_links: 0
Amazon: Amazon:
server: mysql-server server: mysql-server
client: mysql client: mysql
@ -137,15 +158,15 @@ Amazon:
file: /etc/my.cnf file: /etc/my.cnf
sections: sections:
mysqld_safe: mysqld_safe:
log-error: /var/log/mysqld.log log_error: /var/log/mysqld.log
pid-file: /var/run/mysqld/mysqld.pid pid_file: /var/run/mysqld/mysqld.pid
mysqld: mysqld:
datadir: /var/lib/mysql datadir: /var/lib/mysql
socket: /var/lib/mysql/mysql.sock socket: /var/lib/mysql/mysql.sock
user: mysql user: mysql
port: 3306 port: 3306
bind-address: 127.0.0.1 bind_address: 127.0.0.1
symbolic-links: 0 symbolic_links: 0
Gentoo: Gentoo:
server: dev-db/mysql server: dev-db/mysql
client: dev-db/mysql client: dev-db/mysql
@ -159,43 +180,43 @@ Gentoo:
port: 3306 port: 3306
socket: /var/run/mysqld/mysqld.sock socket: /var/run/mysqld/mysqld.sock
mysql: mysql:
character-sets-dir: /usr/share/mysql/charsets character_sets_dir: /usr/share/mysql/charsets
default-character-set: utf8 default_character_set: utf8
mysqladmin: mysqladmin:
character-sets-dir: /usr/share/mysql/charsets character_sets_dir: /usr/share/mysql/charsets
default-character-set: utf8 default_character_set: utf8
mysqlcheck: mysqlcheck:
character-sets-dir: /usr/share/mysql/charsets character_sets_dir: /usr/share/mysql/charsets
default-character-set: utf8 default_character_set: utf8
mysqldump: mysqldump:
character-sets-dir: /usr/share/mysql/charsets character_sets_dir: /usr/share/mysql/charsets
default-character-set: utf8 default_character_set: utf8
mysqlimport: mysqlimport:
character-sets-dir: /usr/share/mysql/charsets character_sets_dir: /usr/share/mysql/charsets
default-character-set: utf8 default_character_set: utf8
mysqlshow: mysqlshow:
character-sets-dir: /usr/share/mysql/charsets character_sets_dir: /usr/share/mysql/charsets
default-character-set: utf8 default_character_set: utf8
myisamchk: myisamchk:
character-sets-dir: /usr/share/mysql/charsets character_sets_dir: /usr/share/mysql/charsets
key_buffer: 20M key_buffer: 20M
sort_buffer_size: 20M sort_buffer_size: 20M
read_buffer: 2M read_buffer: 2M
write_buffer: 2M write_buffer: 2M
myisampack: myisampack:
character-sets-dir: /usr/share/mysql/charsets character_sets_dir: /usr/share/mysql/charsets
mysqld_safe: mysqld_safe:
err-log: /var/log/mysql/mysql.err err_log: /var/log/mysql/mysql.err
mysqld: mysqld:
character-set-server: utf8 character_set_server: utf8
user: mysql user: mysql
port: 3306 port: 3306
socket: /var/run/mysqld/mysqld.sock socket: /var/run/mysqld/mysqld.sock
pid-file: /var/run/mysqld/mysqld.pid pid_file: /var/run/mysqld/mysqld.pid
log-error: /var/log/mysql/mysqld.err log_error: /var/log/mysql/mysqld.err
basedir: /usr basedir: /usr
datadir: /var/lib/mysql datadir: /var/lib/mysql
skip-external-locking: noarg_present skip_external_locking: noarg_present
key_buffer_size: 16M key_buffer_size: 16M
max_allowed_packet: 1M max_allowed_packet: 1M
table_open_cache: 64 table_open_cache: 64
@ -205,9 +226,9 @@ Gentoo:
read_rnd_buffer_size: 512K read_rnd_buffer_size: 512K
myisam_sort_buffer_size: 8M myisam_sort_buffer_size: 8M
language: /usr/share/mysql/english language: /usr/share/mysql/english
bind-address: 127.0.0.1 bind_address: 127.0.0.1
log-bin: noarg_present log_bin: noarg_present
server-id: 1 server_id: 1
tmpdir: /tmp/ tmpdir: /tmp/
innodb_buffer_pool_size: 16M innodb_buffer_pool_size: 16M
innodb_additional_mem_pool_size: 2M innodb_additional_mem_pool_size: 2M
@ -227,7 +248,7 @@ Gentoo:
read_buffer: 2M read_buffer: 2M
write_buffer: 2M write_buffer: 2M
mysqlhotcopy: mysqlhotcopy:
interactive-timeout: noarg_present interactive_timeout: noarg_present
FreeBSD: FreeBSD:
server: mysql56-server server: mysql56-server
client: mysql56-client client: mysql56-client

7
mysql/disabled.sls Normal file
View file

@ -0,0 +1,7 @@
{% from "mysql/defaults.yaml" import rawmap with context %}
{%- set mysql = salt['grains.filter_by'](rawmap, grain='os', merge=salt['pillar.get']('mysql:server:lookup')) %}
mysql:
service.dead:
- name: {{ mysql.service }}
- enable: False

View file

@ -1,30 +1,40 @@
# DO NOT CHANGE THIS FILE! # DO NOT CHANGE THIS FILE!
# This config is generated by SALTSTACK # This config is generated by SALTSTACK
# and all change will be overrided on next salt call # and all change will be overrided on next salt call
{% from "mysql/defaults.yaml" import rawmap with context %} {#-
{% from "mysql/supported_params.yaml" import supported_params with context %} ===== FETCH DATA =====
{%- set datamap = salt['grains.filter_by'](rawmap, grain='os', merge=salt['pillar.get']('mysql:server:lookup')) %} -#}
{%- for section_name, supparams in supported_params.items() %} {%- from "mysql/defaults.yaml" import rawmap with context -%}
[{{ section_name }}] {%- from "mysql/supported_sections.yaml" import supported_sections with context -%}
{%- for allowedparam in supparams|default([]) %} {%- set datamap = salt['grains.filter_by'](rawmap, grain='os', merge=salt['pillar.get']('mysql:server:lookup')) -%}
{%- set indents = 40 - allowedparam|count %} {#-
{%- set mparam = salt['pillar.get']('mysql:server:'+section_name+':'+allowedparam, false) %} ===== COMBINE DATA =====
{%- if mparam %} -#}
{%- if mparam == "noarg_present" %} {%- set goodParamList = datamap.config.sections -%}
{{ allowedparam }} {%- for section_name in supported_sections -%}
{%- else %} {%- set sectdict = datamap.config.sections[section_name] | default({}) -%}
{{ allowedparam }}{{ '='|indent(indents, true) }} {{ mparam }} {%- for mparam, mvalue in salt['pillar.get']('mysql:server:'+section_name, {}).items() -%}
{%- endif %} {%- set mparamUnderscore = mparam | replace('-','_') -%}
{%- else %} {%- do sectdict.update({mparamUnderscore:mvalue}) -%}
{%- if datamap.config.sections[section_name] is defined %} {%- endfor -%}
{%- if datamap.config.sections[section_name][allowedparam] is defined %} {%- do goodParamList.update({section_name:sectdict}) -%}
{%- if datamap.config.sections[section_name][allowedparam] == "noarg_present" %} {%- endfor -%}
{{ allowedparam }} {#-
{%- else %} ===== PRINT DATA =====
{{ allowedparam }}{{ '='|indent(indents, true) }} {{ datamap.config.sections[section_name][allowedparam] }} -#}
{%- endif %} {%- for sname,sdata in goodParamList.items() -%}
{%- endif %} {%- if sdata %}
{%- endif %}
{%- endif %} [{{ sname }}]
{%- endfor %} {%- for mparam, mvalue in sdata.items()|default([])|sort -%}
{% endfor %} {%- set indents = 40 - mparam|count %}
{% if mvalue == "noarg_present" -%}
{{ mparam }}
{%- else -%}
{{ mparam }}{{ '='|indent(indents, true) }} {{ mvalue }}
{%- endif -%}
{%- endfor -%}
{%- endif -%}
{%- endfor %}
{{ datamap.config.append | default('') }}

7
mysql/files/usr-my.cnf Normal file
View file

@ -0,0 +1,7 @@
# DO NOT CHANGE THIS FILE!
# This config is generated by SALTSTACK
# and all change will be overrided on next salt call
#
# this file was created by mysql-server package
# and replaced with saltstack
# to prevent default my.cnf parameters overriding

View file

@ -2,6 +2,5 @@
{%- set mysql = salt['grains.filter_by'](rawmap, grain='os', merge=salt['pillar.get']('mysql:server:lookup')) %} {%- set mysql = salt['grains.filter_by'](rawmap, grain='os', merge=salt['pillar.get']('mysql:server:lookup')) %}
mysql_python: mysql_python:
pkg: pkg.installed:
- installed
- name: {{ mysql.python }} - name: {{ mysql.python }}

View file

@ -1,5 +1,5 @@
{% from "mysql/defaults.yaml" import rawmap with context %} {% from "mysql/defaults.yaml" import rawmap with context %}
{%- set mysql = salt['grains.filter_by'](rawmap, grain='os', merge=salt['pillar.get']('mysql:server:lookup')) %} {%- set mysql = salt['grains.filter_by'](rawmap, grain='os', merge=salt['pillar.get']('mysql:lookup')) %}
{% set os = salt['grains.get']('os', None) %} {% set os = salt['grains.get']('os', None) %}
{% set os_family = salt['grains.get']('os_family', None) %} {% set os_family = salt['grains.get']('os_family', None) %}
@ -56,7 +56,7 @@ mysql_delete_anonymous_user_{{ host }}:
mysqld: mysqld:
pkg.installed: pkg.installed:
- name: {{ mysql.server }} - name: {{ mysql.server }}
{% if os_family == 'Debian' %} {% if os_family == 'Debian' and mysql_root_password %}
- require: - require:
- debconf: mysql_debconf - debconf: mysql_debconf
{% endif %} {% endif %}
@ -78,3 +78,14 @@ mysql_config:
- group: root - group: root
- mode: 644 - mode: 644
{% endif %} {% endif %}
# official oracle mysql repo
# creates this file, that rewrites /etc/mysql/my.cnf setting
# so, make it empty
mysql_additional_config:
file.managed:
- name: /usr/my.cnf
- source: salt://mysql/files/usr-my.cnf
- create: False
- watch_in:
- service: mysqld

View file

@ -1,980 +0,0 @@
# vim
{#- Do not edit this YAML file by hand. See README.rst for how to update -#}
{% load_yaml as supported_params %}
# From https://dev.mysql.com/doc/refman/5.7/en/mysql-command-options.html
{% for section in ['client', 'mysql'] %}
{{ section }}:
- help
- auto-rehash
- auto-vertical-output
- batch
- binary-mode
- bind-address
- character-sets-dir
- column-names
- column-type-info
- comments
- compress
- connect-expired-password
- database
- debug
- debug-check
- debug-info
- default-auth
- default-character-set
- defaults-extra-file
- defaults-file
- defaults-group-suffix
- delimiter
- disable-named-commands
- enable-cleartext-plugin
- execute
- force
- histignore
- host
- html
- ignore-spaces
- init-command
- line-numbers
- local-infile
- login-path
- named-commands
- no-auto-rehash
- no-beep
- no-defaults
- one-database
- pager
- password
- pipe
- plugin-dir
- port
- print-defaults
- prompt
- protocol
- quick
- raw
- reconnect
- safe-updates
- secure-auth
- server-public-key-path
- shared-memory-base-name
- show-warnings
- sigint-ignore
- silent
- skip-column-names
- skip-line-numbers
- socket
- ssl
- syslog
- table
- tee
- unbuffered
- user
- verbose
- version
- vertical
- wait
- xml
- connect_timeout
- max_allowed_packet
- max_join_size
- net_buffer_length
- select_limit
{% endfor %}
# From https://dev.mysql.com/doc/refman/5.7/en/mysqldump.html
mysqldump:
- bind-address
- compress
- default-auth
- host
- login-path
- password
- pipe
- plugin-dir
- port
- protocol
- secure-auth
- socket
- ssl
- user
- max_allowed_packet
- net_buffer_length
- defaults-extra-file
- defaults-file
- defaults-group-suffix
- no-defaults
- print-defaults
- add-drop-database
- add-drop-table
- add-drop-trigger
- all-tablespaces
- no-create-db
- no-create-info
- no-tablespaces
- replace
- allow-keywords
- comments
- debug
- debug-check
- debug-info
- dump-date
- force
- log-error
- skip-comments
- verbose
- help
- version
- character-sets-dir
- default-character-set
- no-set-names
- set-charset
- apply-slave-statements
- delete-master-logs
- dump-slave
- include-master-host-port
- master-data
- set-gtid-purged
- compact
- compatible
- complete-insert
- create-options
- fields
- hex-blob
- lines-terminated-by
- quote-names
- result-file
- tab
- tz-utc
- xml
- all-databases
- databases
- events
- ignore-error
- ignore-table
- no-data
- routines
- tables
- triggers
- where
- disable-keys
- extended-insert
- insert-ignore
- opt
- quick
- skip-opt
- add-locks
- flush-logs
- flush-privileges
- lock-all-tables
- lock-tables
- no-autocommit
- order-by-primary
- shared-memory-base-name
- single-transaction
# From https://dev.mysql.com/doc/refman/5.7/en/mysqld-safe.html
mysqld_safe:
- help
- basedir
- core-file-size
- datadir
- defaults-extra-file
- defaults-file
- ledir
- log-error
- malloc-lib
- mysqld
- mysqld-version
- nice
- no-defaults
- open-files-limit
- pid-file
- plugin-dir
- port
- skip-kill-mysqld
- socket
- syslog
- syslog-tag
- timezone
- user
# From http://dev.mysql.com/doc/refman/5.6/en/mysqlhotcopy.html
mysqlhotcopy:
- help
- addtodest
- allowold
- checkpoint
- chroot
- debug
- dryrun
- flushlog
- host
- keepold
- method
- noindices
- password
- port
- old_server
- quiet
- record_log_pos
- regexp
- resetmaster
- resetslave
- socket
- suffix
- tmpdir
- user
# From http://dev.mysql.com/doc/refman/5.7/en/mysqladmin.html
mysqladmin:
- help
- bind-address
- character-sets-dir
- compress
- count
- debug
- debug-check
- debug-info
- default-auth
- default-character-set
- defaults-extra-file
- defaults-file
- defaults-group-suffix
- enable-cleartext-plugin
- force
- host
- login-path
- no-beep
- no-defaults
- password
- pipe
- plugin-dir
- port
- print-defaults
- protocol
- relative
- show-warnings
- secure-auth
- shared-memory-base-name
- silent
- sleep
- socket
- ssl
- user
- verbose
- version
- vertical
- wait
- connect_timeout
- shutdown_timeout
# From http://dev.mysql.com/doc/refman/5.7/en/mysqlcheck.html
mysqlcheck:
- help
- all-databases
- all-in-1
- analyze
- auto-repair
- bind-address
- character-sets-dir
- check
- check-only-changed
- check-upgrade
- compress
- databases
- debug
- debug-check
- debug-info
- default-character-set
- defaults-extra-file
- defaults-file
- defaults-group-suffix
- extended
- default-auth
- fast
- fix-db-names
- fix-table-names
- force
- host
- login-path
- medium-check
- no-defaults
- optimize
- password
- pipe
- plugin-dir
- port
- print-defaults
- protocol
- quick
- repair
- secure-auth
- shared-memory-base-name
- silent
- skip-database
- socket
- ssl
- tables
- use-frm
- user
- verbose
- version
- write-binlog
# From http://dev.mysql.com/doc/refman/5.7/en/mysqlimport.html
mysqlimport:
- help
- bind-address
- character-sets-dir
- columns
- compress
- debug
- debug-check
- debug-info
- default-character-set
- default-auth
- defaults-extra-file
- defaults-file
- defaults-group-suffix
- delete
- fields
- force
- host
- ignore
- ignore-lines
- lines-terminated-by
- local
- lock-tables
- login-path
- low-priority
- no-defaults
- password
- pipe
- plugin-dir
- port
- print-defaults
- protocol
- replace
- secure-auth
- shared-memory-base-name
- silent
- socket
- ssl
- user
- use-threads
- verbose
- version
# From http://dev.mysql.com/doc/refman/5.7/en/mysqlshow.html
mysqlshow:
- help
- bind-address
- character-sets-dir
- compress
- count
- debug
- debug-check
- debug-info
- default-character-set
- default-auth
- defaults-extra-file
- defaults-file
- defaults-group-suffix
- host
- keys
- login-path
- no-defaults
- password
- pipe
- plugin-dir
- port
- print-defaults
- protocol
- secure-auth
- shared-memory-base-name
- show-table-type
- socket
- ssl
- status
- user
- verbose
- version
# From http://dev.mysql.com/doc/refman/5.7/en/myisampack.html
myisampack:
- help
- backup
- character-sets-dir
- debug
- force
- join
- silent
- test
- tmpdir
- verbose
- version
- wait
# From https://dev.mysql.com/doc/refman/5.7/en/myisamchk.html
{% for section in ['myisamchk', 'isamchk'] %}
{{ section }}:
- analyze
- backup
- block-search
- check
- check-only-changed
- correct-checksum
- data-file-length
- debug
- decode_bits
- defaults-extra-file
- defaults-file
- defaults-group-suffix
- description
- extend-check
- fast
- force
- force
- ft_max_word_len
- ft_min_word_len
- ft_stopword_file
- HELP
- help
- information
- key_buffer_size
- keys-used
- max-record-length
- medium-check
- myisam_block_size
- myisam_sort_buffer_size
- no-defaults
- parallel-recover
- print-defaults
- quick
- read_buffer_size
- read-only
- recover
- safe-recover
- set-auto-increment
- set-collation
- silent
- sort_buffer_size
- sort-index
- sort_key_blocks
- sort-records
- sort-recover
- stats_method
- tmpdir
- unpack
- update-state
- verbose
- version
- write_buffer_size
{% endfor %}
# From https://dev.mysql.com/doc/refman/5.7/en/mysqld-option-tables.html
mysqld:
- abort-slave-event-count
- allow-suspicious-udfs
- ansi
- autocommit
- basedir
- big-tables
- bind-address
- binlog_cache_size
- binlog-checksum
- binlog_direct_non_transactional_updates
- binlog-do-db
- binlog-format
- binlog_group_commit_sync_delay
- binlog_group_commit_sync_no_delay_count
- binlog-ignore-db
- binlog-row-event-max-size
- binlog_row_image
- binlog-rows-query-log-events
- binlog_stmt_cache_size
- binlogging_impossible_mode
- block_encryption_mode
- bootstrap
- bulk_insert_buffer_size
- character-set-client-handshake
- character-set-filesystem
- character-set-server
- character-sets-dir
- chroot
- collation-server
- completion_type
- concurrent_insert
- connect_timeout
- console
- core-file
- daemon_memcached_enable_binlog
- daemon_memcached_engine_lib_name
- daemon_memcached_engine_lib_path
- daemon_memcached_option
- daemon_memcached_r_batch_size
- daemon_memcached_w_batch_size
- datadir
- debug
- debug-sync-timeout
- default-authentication-plugin
- default_authentication_plugin
- default_password_lifetime
- default-storage-engine
- default-time-zone
- default_tmp_storage_engine
- default_week_format
- delay-key-write
- delayed_insert_limit
- delayed_insert_timeout
- delayed_queue_size
- des-key-file
- disconnect_on_expired_password
- disconnect-slave-event-count
- div_precision_increment
- enable-named-pipe
- enforce_gtid_consistency
- enforce-gtid-consistency
- event-scheduler
- executed-gtids-compression-period
- exit-info
- expire_logs_days
- explicit_defaults_for_timestamp
- external-locking
- federated
- flush
- flush_time
- ft_boolean_syntax
- ft_max_word_len
- ft_min_word_len
- ft_query_expansion_limit
- ft_stopword_file
- gdb
- general-log
- general_log_file
- group_concat_max_len
- gtid-mode
- help
- ignore-builtin-innodb
- ignore-db-dir
- init_connect
- init-file
- init_slave
- innodb
- innodb_adaptive_flushing
- innodb_adaptive_flushing_lwm
- innodb_adaptive_hash_index
- innodb_adaptive_max_sleep_delay
- innodb_additional_mem_pool_size
- innodb_api_bk_commit_interval
- innodb_api_disable_rowlock
- innodb_api_enable_binlog
- innodb_api_enable_mdl
- innodb_api_trx_level
- innodb_autoextend_increment
- innodb_autoinc_lock_mode
- innodb_buffer_pool_chunk_size
- innodb_buffer_pool_dump_at_shutdown
- innodb_buffer_pool_dump_now
- innodb_buffer_pool_dump_pct
- innodb_buffer_pool_filename
- innodb_buffer_pool_instances
- innodb_buffer_pool_load_abort
- innodb_buffer_pool_load_at_startup
- innodb_buffer_pool_load_now
- innodb_buffer_pool_size
- innodb_change_buffer_max_size
- innodb_change_buffering
- innodb_checksum_algorithm
- innodb_checksums
- innodb_cmp_per_index_enabled
- innodb_commit_concurrency
- innodb_compression_failure_threshold_pct
- innodb_compression_level
- innodb_compression_pad_pct_max
- innodb_concurrency_tickets
- innodb_create_intrinsic
- innodb_data_file_path
- innodb_data_home_dir
- innodb_disable_sort_file_cache
- innodb_doublewrite
- innodb_fast_shutdown
- innodb_file_format
- innodb_file_format_check
- innodb_file_format_max
- innodb_file_per_table
- innodb_fill_factor
- innodb_flush_log_at_trx_commit
- innodb_flush_method
- innodb_flush_neighbors
- innodb_flushing_avg_loops
- innodb_force_load_corrupted
- innodb_force_recovery
- innodb_ft_aux_table
- innodb_ft_cache_size
- innodb_ft_enable_diag_print
- innodb_ft_enable_stopword
- innodb_ft_max_token_size
- innodb_ft_min_token_size
- innodb_ft_num_word_optimize
- innodb_ft_result_cache_limit
- innodb_ft_server_stopword_table
- innodb_ft_sort_pll_degree
- innodb_ft_total_cache_size
- innodb_ft_user_stopword_table
- innodb_io_capacity
- innodb_io_capacity_max
- innodb_large_prefix
- innodb_lock_wait_timeout
- innodb_locks_unsafe_for_binlog
- innodb_log_buffer_size
- innodb_log_compressed_pages
- innodb_log_file_size
- innodb_log_files_in_group
- innodb_log_group_home_dir
- innodb_log_write_ahead_size
- innodb_lru_scan_depth
- innodb_max_dirty_pages_pct
- innodb_max_dirty_pages_pct_lwm
- innodb_max_purge_lag
- innodb_max_purge_lag_delay
- innodb_max_undo_log_size
- innodb_monitor_disable
- innodb_monitor_enable
- innodb_monitor_reset
- innodb_monitor_reset_all
- innodb_old_blocks_pct
- innodb_old_blocks_time
- innodb_online_alter_log_max_size
- innodb_open_files
- innodb_optimize_fulltext_only
- innodb_optimize_point_storage
- innodb_page_cleaners
- innodb_page_size
- innodb_print_all_deadlocks
- innodb_purge_batch_size
- innodb_purge_rseg_truncate_frequency
- innodb_purge_threads
- innodb_random_read_ahead
- innodb_read_ahead_threshold
- innodb_read_io_threads
- innodb_read_only
- innodb_replication_delay
- innodb_rollback_on_timeout
- innodb_rollback_segments
- innodb_sort_buffer_size
- innodb_spin_wait_delay
- innodb_stats_auto_recalc
- innodb_stats_method
- innodb_stats_on_metadata
- innodb_stats_persistent
- innodb_stats_persistent_sample_pages
- innodb_stats_sample_pages
- innodb_stats_transient_sample_pages
- innodb-status-file
- innodb_status_output
- innodb_status_output_locks
- innodb_strict_mode
- innodb_support_xa
- innodb_sync_array_size
- innodb_sync_spin_loops
- innodb_table_locks
- innodb_temp_data_file_path
- innodb_thread_concurrency
- innodb_thread_sleep_delay
- innodb_undo_directory
- innodb_undo_log_truncate
- innodb_undo_logs
- innodb_undo_tablespaces
- innodb_use_native_aio
- innodb_use_sys_malloc
- innodb_write_io_threads
- interactive_timeout
- internal_tmp_disk_storage_engine
- join_buffer_size
- keep_files_on_create
- key_buffer_size
- key_cache_age_threshold
- key_cache_block_size
- key_cache_division_limit
- language
- large-pages
- lc-messages
- lc-messages-dir
- lock_wait_timeout
- log-bin
- log-bin-index
- log-bin-trust-function-creators
- log_bin_use_v
- log-bin-use-v
- log-error
- log_error_verbosity
- log-isam
- log-output
- log-queries-not-using-indexes
- log-raw
- log-short-format
- log-slave-updates
- log_slave_updates
- log-slow-admin-statements
- log-slow-slave-statements
- log_syslog
- log_syslog_facility
- log_syslog_include_pid
- log_syslog_tag
- log-tc
- log-tc-size
- log_timestamps
- log-warnings
- long_query_time
- low-priority-updates
- lower_case_table_names
- master-info-file
- master_info_repository
- master-info-repository
- master-retry-count
- master-verify-checksum
- max_allowed_packet
- max_binlog_cache_size
- max-binlog-dump-events
- max_binlog_size
- max_binlog_stmt_cache_size
- max_connect_errors
- max_connections
- max_delayed_threads
- max_error_count
- max_heap_table_size
- max_join_size
- max_length_for_sort_data
- max_prepared_stmt_count
- max_relay_log_size
- max_seeks_for_key
- max_sort_length
- max_sp_recursion_depth
- max_statement_time
- max_user_connections
- max_write_lock_count
- memlock
- min-examined-row-limit
- myisam-block-size
- myisam_data_pointer_size
- myisam_max_sort_file_size
- myisam_mmap_size
- myisam-recover-options
- myisam_repair_threads
- myisam_sort_buffer_size
- myisam_stats_method
- myisam_use_mmap
- ndb-batch-size
- ndb-blob-write-batch-bytes
- ndb-cluster-connection-pool
- ndb-connectstring
- ndb-deferred-constraints
- ndb_deferred_constraints
- ndb_distribution
- ndb-distribution
- ndb_eventbuffer_max_alloc
- ndb_force_send
- ndb_index_stat_enable
- ndb_index_stat_option
- ndb-log-apply-status
- ndb_log_apply_status
- ndb-log-empty-epochs
- ndb-log-transaction-id
- ndb_log_updated_only
- ndb-mgmd-host
- ndb_optimized_node_selection
- ndb_report_thresh_binlog_epoch_slip
- ndb_report_thresh_binlog_mem_usage
- ndb_show_foreign_key_mock_tables
- ndb_use_transactions
- ndb-wait-setup
- net_buffer_length
- net_read_timeout
- net_retry_count
- net_write_timeout
- new
- offline_mode
- old
- old-alter-table
- old-style-user-limits
- open-files-limit
- optimizer_prune_level
- optimizer_search_depth
- optimizer_switch
- partition
- performance_schema
- performance_schema_accounts_size
- performance-schema-consumer-events-stages-current
- performance-schema-consumer-events-stages-history
- performance-schema-consumer-events-stages-history-long
- performance-schema-consumer-events-statements-current
- performance-schema-consumer-events-statements-history
- performance-schema-consumer-events-statements-history-long
- performance-schema-consumer-events-transactions-current
- performance-schema-consumer-events-transactions-history
- performance-schema-consumer-events-transactions-history-long
- performance-schema-consumer-events-waits-current
- performance-schema-consumer-events-waits-history
- performance-schema-consumer-events-waits-history-long
- performance-schema-consumer-global-instrumentation
- performance-schema-consumer-statements-digest
- performance-schema-consumer-thread-instrumentation
- performance_schema_digests_size
- performance_schema_events_stages_history_long_size
- performance_schema_events_stages_history_size
- performance_schema_events_statements_history_long_size
- performance_schema_events_statements_history_size
- performance_schema_events_transactions_history_long_size
- performance_schema_events_transactions_history_size
- performance_schema_events_waits_history_long_size
- performance_schema_events_waits_history_size
- performance_schema_hosts_size
- performance-schema-instrument
- performance_schema_max_cond_classes
- performance_schema_max_cond_instances
- performance_schema_max_file_classes
- performance_schema_max_file_handles
- performance_schema_max_file_instances
- performance_schema_max_memory_classes
- performance_schema_max_metadata_locks
- performance_schema_max_mutex_classes
- performance_schema_max_mutex_instances
- performance_schema_max_prepared_statements_instances
- performance_schema_max_program_instances
- performance_schema_max_rwlock_classes
- performance_schema_max_rwlock_instances
- performance_schema_max_socket_classes
- performance_schema_max_socket_instances
- performance_schema_max_stage_classes
- performance_schema_max_statement_classes
- performance_schema_max_statement_stack
- performance_schema_max_table_handles
- performance_schema_max_table_instances
- performance_schema_max_thread_classes
- performance_schema_max_thread_instances
- performance_schema_session_connect_attrs_size
- performance_schema_setup_actors_size
- performance_schema_setup_objects_size
- performance_schema_users_size
- pid-file
- plugin
- plugin_dir
- plugin-load
- plugin-load-add
- port
- port-open-timeout
- preload_buffer_size
- profiling_history_size
- query_alloc_block_size
- query_cache_limit
- query_cache_min_res_unit
- query_cache_size
- query_cache_type
- query_cache_wlock_invalidate
- query_prealloc_size
- range_alloc_block_size
- read_buffer_size
- read_only
- read_rnd_buffer_size
- relay-log
- relay-log-index
- relay_log_index
- relay-log-info-file
- relay_log_info_file
- relay-log-info-repository
- relay_log_purge
- relay_log_recovery
- relay-log-recovery
- relay_log_space_limit
- replicate-do-db
- replicate-do-table
- replicate-ignore-db
- replicate-ignore-table
- replicate-rewrite-db
- replicate-same-server-id
- replicate-wild-do-table
- replicate-wild-ignore-table
- report-host
- report-password
- report-port
- report-user
- rpl_stop_slave_timeout
- safe-user-create
- secure-auth
- secure-file-priv
- server-id
- session_track_schema
- session_track_state_change
- session_track_system_variables
- shared_memory
- show-slave-auth-info
- simplified_binlog_gtid_recovery
- skip-character-set-client-handshake
- skip-concurrent-insert
- skip-event-scheduler
- skip_external_locking
- skip-grant-tables
- skip-host-cache
- skip-name-resolve
- skip-ndbcluster
- skip-networking
- skip-new
- skip-partition
- skip-show-database
- skip-slave-start
- skip-ssl
- skip-stack-trace
- slave_allow_batching
- slave_checkpoint_group
- slave-checkpoint-group
- slave_checkpoint_period
- slave-checkpoint-period
- slave_compressed_protocol
- slave_exec_mode
- slave-load-tmpdir
- slave-max-allowed-packet
- slave-net-timeout
- slave-parallel-type
- slave-parallel-workers
- slave-rows-search-algorithms
- slave-skip-errors
- slave-sql-verify-checksum
- slave_transaction_retries
- slave_type_conversions
- slow_launch_time
- slow-query-log
- slow_query_log_file
- slow-start-timeout
- socket
- sort_buffer_size
- sporadic-binlog-dump-fail
- sql-mode
- ssl
- ssl-ca
- ssl-capath
- ssl-cert
- ssl-cipher
- ssl-crl
- ssl-crlpath
- ssl-key
- standalone
- stored_program_cache
- super-large-pages
- symbolic-links
- sync_binlog
- sync_frm
- sync_master_info
- sync_relay_log
- sync_relay_log_info
- sysdate-is-now
- tc-heuristic-recover
- temp-pool
- thread_cache_size
- thread_concurrency
- thread_handling
- thread_stack
- timed_mutexes
- tmp_table_size
- tmpdir
- transaction_alloc_block_size
- transaction-isolation
- transaction_prealloc_size
- transaction-read-only
- updatable_views_with_limit
- user
- validate-password
- verbose
- wait_timeout
{% endload %}

View file

@ -0,0 +1,16 @@
# vim
{% load_yaml as supported_sections %}
- client
- mysql
- mysqldump
- mysqld_safe
- mysqlhotcopy
- mysqladmin
- mysqlcheck
- mysqlimport
- mysqlshow
- myisampack
- myisamchk
- isamchk
- mysqld
{% endload %}

View file

@ -3,16 +3,27 @@
{%- set mysql_root_pass = salt['pillar.get']('mysql:server:root_password', 'somepass') %} {%- set mysql_root_pass = salt['pillar.get']('mysql:server:root_password', 'somepass') %}
{% set user_states = [] %} {% set user_states = [] %}
{% set user_hosts = [] %}
include: include:
- mysql.python - mysql.python
{% for name, user in salt['pillar.get']('mysql:user', {}).items() %} {% for name, user in salt['pillar.get']('mysql:user', {}).items() %}
{% set state_id = 'mysql_user_' ~ loop.index0 %}
{% set user_host = salt['pillar.get']('mysql:user:%s:host'|format(name)) %}
{% if user_host != '' %}
{% set user_hosts = [user_host] %}
{% else %}
{% set user_hosts = salt['pillar.get']('mysql:user:%s:hosts'|format(name)) %}
{% endif %}
{% for host in user_hosts %}
{% set state_id = 'mysql_user_' ~ name ~ '_' ~ host%}
{{ state_id }}: {{ state_id }}:
mysql_user.present: mysql_user.present:
- name: {{ name }} - name: {{ name }}
- host: '{{ user['host'] }}' - host: '{{ host }}'
{%- if user['password_hash'] is defined %} {%- if user['password_hash'] is defined %}
- password_hash: '{{ user['password_hash'] }}' - password_hash: '{{ user['password_hash'] }}'
{%- elif user['password'] is defined and user['password'] != None %} {%- elif user['password'] is defined and user['password'] != None %}
@ -35,7 +46,7 @@ include:
- database: '{{ db['database'] }}.{{ db['table'] | default('*') }}' - database: '{{ db['database'] }}.{{ db['table'] | default('*') }}'
- grant_option: {{ db['grant_option'] | default(False) }} - grant_option: {{ db['grant_option'] | default(False) }}
- user: {{ name }} - user: {{ name }}
- host: '{{ user['host'] }}' - host: '{{ host }}'
- connection_host: localhost - connection_host: localhost
- connection_user: root - connection_user: root
{% if mysql_root_pass -%} {% if mysql_root_pass -%}
@ -48,3 +59,4 @@ include:
{% do user_states.append(state_id) %} {% do user_states.append(state_id) %}
{% endfor %} {% endfor %}
{% endfor %}

View file

@ -5,13 +5,12 @@ mysql:
user: mysql user: mysql
# my.cnf sections changes # my.cnf sections changes
mysqld: mysqld:
# you can use either underscore or hyphen in param names
bind-address: 0.0.0.0 bind-address: 0.0.0.0
log_bin: /var/log/mysql/mysql-bin.log
port: 3307 port: 3307
server-id: 1 binlog_do_db: foo
log-bin: mysql-bin
binlog-do-db: foo
auto_increment_increment: 5 auto_increment_increment: 5
max_connect_errors: 4294967295
mysql: mysql:
# my.cnf param that not require value # my.cnf param that not require value
no-auto-rehash: noarg_present no-auto-rehash: noarg_present
@ -28,7 +27,7 @@ mysql:
load: False load: False
# Manage users # Manage users
# you can get pillar for existent server using import_users.py script # you can get pillar for existent server using scripts/import_users.py script
user: user:
frank: frank:
password: 'somepass' password: 'somepass'
@ -58,10 +57,9 @@ mysql:
server: mysql-server server: mysql-server
client: mysql-client client: mysql-client
service: mysql-service service: mysql-service
config: /etc/mysql/my.cnf
python: python-mysqldb python: python-mysqldb
# Install MySQL headers # Install MySQL headers
dev: dev:
# Install dev package - defaults to False # Install dev package - defaults to False
install: False install: False

View file

@ -1,61 +0,0 @@
#-------------------------------------------------------------------------------
# Name: html_table_parser
# Purpose: Simple class for parsing an (x)html string to extract tables.
# Written in python3
#
# Author: Josua Schmid
#
# Created: 05.03.2014
# Copyright: (c) Josua Schmid 2014
# Licence: GPLv3
#-------------------------------------------------------------------------------
from html.parser import HTMLParser
class HTMLTableParser(HTMLParser):
""" This class serves as a html table parser. It is able to parse multiple
tables which you feed in. You can access the result per .tables field.
"""
def __init__(self):
HTMLParser.__init__(self)
self.__in_td = False
self.__in_th = False
self.__current_table = []
self.__current_row = []
self.__current_cell = []
self.tables = []
def handle_starttag(self, tag, attrs):
""" We need to remember the opening point for the content of interest.
The other tags (<table>, <tr>) are only handled at the closing point.
"""
if tag == 'td':
self.__in_td = True
if tag == 'th':
self.__in_th = True
def handle_data(self, data):
""" This is where we save content to a cell """
if self.__in_td ^ self.__in_th:
self.__current_cell.append(data.strip())
def handle_endtag(self, tag):
""" Here we exit the tags. If the closing tag is </tr>, we know that we
can save our currently parsed cells to the current table as a row and
prepare for a new row. If the closing tag is </table>, we save the
current table and prepare for a new one.
"""
if tag == 'td':
self.__in_td = False
if tag == 'th':
self.__in_th = False
if (tag == 'td') ^ (tag == 'th'):
final_cell = " ".join(self.__current_cell).strip()
self.__current_row.append(final_cell)
self.__current_cell = []
if tag == 'tr':
self.__current_table.append(self.__current_row)
self.__current_row = []
if tag == 'table':
self.tables.append(self.__current_table)
self.__current_table = []

78
scripts/import_users.py Executable file
View file

@ -0,0 +1,78 @@
#!/usr/bin/env python
"This script helps you to get mysql.user pillar from existent mysql server"
import argparse
import MySQLdb
import re
__author__ = "Egor Potiomkin"
__version__ = "1.0"
__email__ = "eg13reg@gmail.com"
parser = argparse.ArgumentParser()
parser.add_argument('host', metavar='IP', help='host where you want to get users')
parser.add_argument('user', metavar='user', help='mysql user that can show grants')
parser.add_argument('password', metavar='password', help='user password')
args = parser.parse_args()
# PARSE GRANTS
mysqlcon = MySQLdb.connect(host=args.host,user=args.user,passwd=args.password,db="mysql",use_unicode=True, charset='utf8')
mysqlCur = mysqlcon.cursor(MySQLdb.cursors.DictCursor)
mysqlCur.execute(r'''select user,host from mysql.user;''')
rows = mysqlCur.fetchall()
users = []
for row in rows:
users.append({'name': row['user'], 'host': row['host']});
mysqlCur = mysqlcon.cursor()
grants = []
for user in users:
q = r'''show grants for '%s'@'%s';''' % (user['name'], user['host'])
try:
user['grants'] = []
mysqlCur.execute(q)
rows = mysqlCur.fetchall()
for row in rows:
mpass = re.search(
r"""GRANT USAGE ON \*\.\* TO .* IDENTIFIED BY PASSWORD '(\*[A-F0-9]*)\'""",
row[0])
if mpass is None:
mgrant = re.search(
r"""GRANT ([\s,A-Z]+) ON `?([a-zA-Z0-9_\-*\\]*)`?\.`?([a-zA-Z0-9_\-*\\]*)`? TO .*""",
row[0])
if mgrant is not None:
user['grants'].append({'grant': [x.strip() for x in mgrant.group(1).split(',')], 'database': mgrant.group(2).replace('\\',''), 'table': mgrant.group(3).replace('\\','')})
else:
print "ERROR: CAN NOT PARSE GRANTS: ",row[0]
else:
user['password'] = mpass.group(1)
except MySQLdb.DatabaseError:
print "Error while getting grants for '%s'@'%s'" % (user['name'], user['host'])
#raise SystemExit
# PRINT RESULT
""" PRINT EXAMPLE
mysql:
user:
username:
host: host
password_hash: '*2792A97371B2D17789364A22A9B35D180166571A'
databases:
- database: testbase
table: table1
grants: ['select']
"""
print "mysql:"
print " user:"
for user in users:
print " %s:" % user['name']
print " host: '%s'" % user['host']
if ('password' in user):
print " password_hash: '%s'" % user['password']
print " databases:"
for grant in user['grants']:
print " - database: '%s'" % grant['database']
print " table: '%s'" % grant['table']
print " grants: ['%s']" % "','".join(grant['grant']).lower()

View file

@ -1,167 +0,0 @@
#!/usr/bin/python3
# coding: utf-8
import argparse
import re
import sys
import urllib.request
from html_table_parser import HTMLTableParser
# Regex for parsing options on MySQL documentation pages
# Options are (normally) specified as command-line options
# as anchor tags on the page. Certain documentation pages only
# show options in table listings, however.
OPTION_REGEX = '<a name="option_%s_(.*?)"></a>'
OPTION_TABLE_REGEX = '^(--)?([A-Za-z_-]+).*$'
# File heading, as per the original supported_params file
FILE_HEADER = """# vim
{#- Do not edit this YAML file by hand. See README.rst for how to update -#}
{% load_yaml as supported_params %}
"""
FILE_FOOTER = """{% endload %}"""
# Standard YAML template for options for a section
YAML_TEMPLATE = """# From %(url)s
%(section)s:
- %(options)s
"""
# For rendering Jinja that handles multiple sections
# Several MySQL utilities use exactly the same options
# Note this variable is string formatted twice, hence the double-double % signs
YAML_TEMPLATE_MULTI = """# From %%(url)s
{%%%% for section in %(sections)r %%%%}
{{ section }}:
- %%(options)s
{%%%% endfor %%%%}
"""
# Options specified in HTML documentation as command-line options
# like so <a name="option_mysql_help"></a>.
# Structure is (section_id, documentation_url, yaml_template_str)
SECTIONS = (
('mysql',
'https://dev.mysql.com/doc/refman/5.7/en/mysql-command-options.html',
YAML_TEMPLATE_MULTI % {'sections': ['client', 'mysql']}),
('mysqldump',
'https://dev.mysql.com/doc/refman/5.7/en/mysqldump.html',
YAML_TEMPLATE),
('mysqld_safe',
'https://dev.mysql.com/doc/refman/5.7/en/mysqld-safe.html',
YAML_TEMPLATE),
# Removed in MySQL 5.7
('mysqlhotcopy',
'http://dev.mysql.com/doc/refman/5.6/en/mysqlhotcopy.html',
YAML_TEMPLATE),
('mysqladmin',
'http://dev.mysql.com/doc/refman/5.7/en/mysqladmin.html',
YAML_TEMPLATE),
('mysqlcheck',
'http://dev.mysql.com/doc/refman/5.7/en/mysqlcheck.html',
YAML_TEMPLATE),
('mysqlimport',
'http://dev.mysql.com/doc/refman/5.7/en/mysqlimport.html',
YAML_TEMPLATE),
('mysqlshow',
'http://dev.mysql.com/doc/refman/5.7/en/mysqlshow.html',
YAML_TEMPLATE),
('myisampack',
'http://dev.mysql.com/doc/refman/5.7/en/myisampack.html',
YAML_TEMPLATE),
)
# Options specified in documentation as command-line and
# option file values in a table only.
SECTIONS_VIA_TABLE = (
('myisamchk',
'https://dev.mysql.com/doc/refman/5.7/en/myisamchk.html',
YAML_TEMPLATE_MULTI % {'sections': ['myisamchk', 'isamchk']}),
)
# Server options specified in documentation
SERVER_OPTIONS = (
'mysqld',
'https://dev.mysql.com/doc/refman/5.7/en/mysqld-option-tables.html',
YAML_TEMPLATE
)
def read_url(url):
""" Read the given URL and decode the response as UTF-8.
"""
request = urllib.request.Request(url)
response = urllib.request.urlopen(request)
return response.read().decode('utf-8')
def read_first_table(url):
""" Read the given URL, parse the result, and return the first table.
"""
xhtml = read_url(url)
parser = HTMLTableParser()
parser.feed(xhtml)
return parser.tables[0] # Use first table on the page
def parse_anchors(url, section):
""" Return parsed options from option anchors at the given URL.
"""
return re.findall(OPTION_REGEX % section, read_url(url))
def parse_tables(url, section):
""" Return arsed options from HTML tables at the given URL.
This matches the given option regex, and ensures that the
first row of the table is ignored; it contains headings only.
"""
table = read_first_table(url)
return [re.match(OPTION_TABLE_REGEX, row[0]).groups()[1]
for row in table[1:]]
def parse_mysqld(url, section):
""" Return the parsed options from the huge mysqld table.
The massive options table shows variables and options and
highlights where they can be used. The following code only
pulls out those that are marked as 'Yes' for use in an option file.
"""
table = read_first_table(url)
# Find which column holds the option file data
option_index = table[0].index('Option File')
# Only pull out options able to be used in an options file
return [re.match(OPTION_TABLE_REGEX, row[0]).groups()[1]
for row in table[1:]
if len(row) >= option_index + 1 and
row[option_index].strip().lower() == 'yes']
def print_yaml_options(sections, parser, file=sys.stdout):
""" Perform really basic templating for output.
A YAML library could be used, but we avoid extra dependencies by
just using string formatting.
"""
for section, url, yaml in sections:
options = parser(url, section)
print(yaml % {'section': section,
'options': '\n - '.join(options),
'url': url}, end='', file=file)
if __name__ == '__main__':
parser = argparse.ArgumentParser(
description='Scrape the MySQL documentation to obtain'
' all the supported parameters for different utilities.')
parser.add_argument('--output',
'-o',
help='File output location',
default=sys.stdout)
config = parser.parse_args()
output = open(config.output, 'w') if isinstance(config.output, str) \
else config.output
print(FILE_HEADER, end='', file=output)
print_yaml_options(SECTIONS, parse_anchors, file=output)
print_yaml_options(SECTIONS_VIA_TABLE, parse_tables, file=output)
print_yaml_options((SERVER_OPTIONS,), parse_mysqld, file=output)
print(FILE_FOOTER, end='', file=output)