From f03d88a759753613fdbdec7ad128666c02e51267 Mon Sep 17 00:00:00 2001 From: root Date: Fri, 12 Dec 2014 10:05:39 +0800 Subject: [PATCH] 1) Supported params list deleted, because it's hard to keep it fresh (there was cool parser by davidjb, but it does not take care of underscores and hiphen in options names). Now all parameters considering supported, and all hiphen will be replaced with underscore when salt creates my.cnf. Thats why all default parameters changed to underscore type. 2) Fixed bug with empty config append parameter 3) Fixed bug with empty root password applying --- README.rst | 23 -------- mysql/defaults.yaml | 101 +++++++++++++++++----------------- mysql/files/my.cnf | 64 ++++++++++++--------- mysql/server.sls | 2 +- mysql/supported_sections.yaml | 16 ++++++ pillar.example | 10 ++-- scripts/import_users.py | 78 ++++++++++++++++++++++++++ 7 files changed, 188 insertions(+), 106 deletions(-) create mode 100644 mysql/supported_sections.yaml create mode 100755 scripts/import_users.py diff --git a/README.rst b/README.rst index 88daca4..0722748 100644 --- a/README.rst +++ b/README.rst @@ -82,26 +82,3 @@ Remove the database called ``test``, normally created as part of a default MySQL installation. This state is **not** included as part of the meta-state above as this name may conflict with a real database. -Updating the supported parameters -================================= - -The ``supported_params.yaml`` file contains the full listing of options that -are acceptable in the MySQL options file. On occassion, especially on new -releases of MySQL, this file may need to be updated. To update, run the -supplied script (requires Python 3.x):: - - ./scripts/parse_supported_params.py -o ./mysql/supported_params.yaml - -This script will scrape the options from the official MySQL documentation -online, and thus requires web access. Scraping is inherently brittle, though -this script has been defensively coded, where possible. - -Once the ``supported_params.yaml`` file has been updated, commit the result to -the repository. - -Support for new applications ----------------------------- - -To add support for configuration of other MySQL applications, add the URL and -section identifier into the relevant section of the script. Consult the -comments in the code to determine where your section should be added. diff --git a/mysql/defaults.yaml b/mysql/defaults.yaml index 8a3a16b..7b62c33 100644 --- a/mysql/defaults.yaml +++ b/mysql/defaults.yaml @@ -1,4 +1,7 @@ # vim: sts=2 ts=2 sw=2 et ai +# +# SET ALL PARAMS IN CONFIG SECTION USING UNDERSCORE, NOT HYPHEN +# so that it works correctly {% load_yaml as rawmap %} Ubuntu: server: mysql-server @@ -17,20 +20,20 @@ Ubuntu: nice: 0 mysqld: user: mysql - pid-file: /var/run/mysqld/mysqld.pid + pid_file: /var/run/mysqld/mysqld.pid socket: /var/run/mysqld/mysqld.sock port: 3306 basedir: /usr datadir: /var/lib/mysql tmpdir: /tmp - lc-messages-dir: /usr/share/mysql - skip-external-locking: noarg_present - bind-address: 127.0.0.1 + lc_messages_dir: /usr/share/mysql + skip_external_locking: noarg_present + bind_address: 127.0.0.1 key_buffer_size: 16M max_allowed_packet: 16M thread_stack: 192K thread_cache_size: 8 - myisam-recover: BACKUP + myisam_recover: BACKUP query_cache_limit: 1M query_cache_size: 16M log_error: /var/log/mysql/error.log @@ -38,10 +41,10 @@ Ubuntu: max_binlog_size: 100M mysqldump: quick: noarg_present - quote-names: noarg_present + quote_names: noarg_present max_allowed_packet: 16M isamchk: - key_buffer: 16M + key_buffer_size: 16M append: | !includedir /etc/mysql/conf.d/ Debian: @@ -61,30 +64,30 @@ Debian: nice: 0 mysqld: user: mysql - pid-file: /var/run/mysqld/mysqld.pid + pid_file: /var/run/mysqld/mysqld.pid socket: /var/run/mysqld/mysqld.sock port: 3306 basedir: /usr datadir: /var/lib/mysql tmpdir: /tmp - lc-messages-dir: /usr/share/mysql - skip-external-locking: noarg_present - bind-address: 127.0.0.1 + lc_messages_dir: /usr/share/mysql + skip_external_locking: noarg_present + bind_address: 127.0.0.1 key_buffer_size: 16M max_allowed_packet: 16M thread_stack: 192K thread_cache_size: 8 - myisam-recover: BACKUP + myisam_recover: BACKUP query_cache_limit: 1M query_cache_size: 16M expire_logs_days: 10 max_binlog_size: 100M mysqldump: quick: noarg_present - quote-names: noarg_present + quote_names: noarg_present max_allowed_packet: 16M isamchk: - key_buffer: 16M + key_buffer_size: 16M append: | !includedir /etc/mysql/conf.d/ CentOS: @@ -96,15 +99,15 @@ CentOS: file: /etc/my.cnf sections: mysqld_safe: - log-error: /var/log/mysqld.log - pid-file: /var/run/mysqld/mysqld.pid + log_error: /var/log/mysqld.log + pid_file: /var/run/mysqld/mysqld.pid mysqld: datadir: /var/lib/mysql socket: /var/lib/mysql/mysql.sock user: mysql port: 3306 - bind-address: 127.0.0.1 - symbolic-links: 0 + bind_address: 127.0.0.1 + symbolic_links: 0 RedHat: server: mysql-server client: mysql @@ -114,15 +117,15 @@ RedHat: file: /etc/my.cnf sections: mysqld_safe: - log-error: /var/log/mysqld.log - pid-file: /var/run/mysqld/mysqld.pid + log_error: /var/log/mysqld.log + pid_file: /var/run/mysqld/mysqld.pid mysqld: datadir: /var/lib/mysql socket: /var/lib/mysql/mysql.sock user: mysql port: 3306 - bind-address: 127.0.0.1 - symbolic-links: 0 + bind_address: 127.0.0.1 + symbolic_links: 0 Amazon: server: mysql-server client: mysql @@ -132,15 +135,15 @@ Amazon: file: /etc/my.cnf sections: mysqld_safe: - log-error: /var/log/mysqld.log - pid-file: /var/run/mysqld/mysqld.pid + log_error: /var/log/mysqld.log + pid_file: /var/run/mysqld/mysqld.pid mysqld: datadir: /var/lib/mysql socket: /var/lib/mysql/mysql.sock user: mysql port: 3306 - bind-address: 127.0.0.1 - symbolic-links: 0 + bind_address: 127.0.0.1 + symbolic_links: 0 Gentoo: server: dev-db/mysql client: dev-db/mysql @@ -153,43 +156,43 @@ Gentoo: port: 3306 socket: /var/run/mysqld/mysqld.sock mysql: - character-sets-dir: /usr/share/mysql/charsets - default-character-set: utf8 + character_sets_dir: /usr/share/mysql/charsets + default_character_set: utf8 mysqladmin: - character-sets-dir: /usr/share/mysql/charsets - default-character-set: utf8 + character_sets_dir: /usr/share/mysql/charsets + default_character_set: utf8 mysqlcheck: - character-sets-dir: /usr/share/mysql/charsets - default-character-set: utf8 + character_sets_dir: /usr/share/mysql/charsets + default_character_set: utf8 mysqldump: - character-sets-dir: /usr/share/mysql/charsets - default-character-set: utf8 + character_sets_dir: /usr/share/mysql/charsets + default_character_set: utf8 mysqlimport: - character-sets-dir: /usr/share/mysql/charsets - default-character-set: utf8 + character_sets_dir: /usr/share/mysql/charsets + default_character_set: utf8 mysqlshow: - character-sets-dir: /usr/share/mysql/charsets - default-character-set: utf8 + character_sets_dir: /usr/share/mysql/charsets + default_character_set: utf8 myisamchk: - character-sets-dir: /usr/share/mysql/charsets + character_sets_dir: /usr/share/mysql/charsets key_buffer: 20M sort_buffer_size: 20M read_buffer: 2M write_buffer: 2M myisampack: - character-sets-dir: /usr/share/mysql/charsets + character_sets_dir: /usr/share/mysql/charsets mysqld_safe: - err-log: /var/log/mysql/mysql.err + err_log: /var/log/mysql/mysql.err mysqld: - character-set-server: utf8 + character_set_server: utf8 user: mysql port: 3306 socket: /var/run/mysqld/mysqld.sock - pid-file: /var/run/mysqld/mysqld.pid - log-error: /var/log/mysql/mysqld.err + pid_file: /var/run/mysqld/mysqld.pid + log_error: /var/log/mysql/mysqld.err basedir: /usr datadir: /var/lib/mysql - skip-external-locking: noarg_present + skip_external_locking: noarg_present key_buffer_size: 16M max_allowed_packet: 1M table_open_cache: 64 @@ -199,9 +202,9 @@ Gentoo: read_rnd_buffer_size: 512K myisam_sort_buffer_size: 8M language: /usr/share/mysql/english - bind-address: 127.0.0.1 - log-bin: noarg_present - server-id: 1 + bind_address: 127.0.0.1 + log_bin: noarg_present + server_id: 1 tmpdir: /tmp/ innodb_buffer_pool_size: 16M innodb_additional_mem_pool_size: 2M @@ -221,7 +224,7 @@ Gentoo: read_buffer: 2M write_buffer: 2M mysqlhotcopy: - interactive-timeout: noarg_present + interactive_timeout: noarg_present FreeBSD: server: mysql56-server client: mysql56-client diff --git a/mysql/files/my.cnf b/mysql/files/my.cnf index cfb1146..86c892e 100644 --- a/mysql/files/my.cnf +++ b/mysql/files/my.cnf @@ -1,30 +1,40 @@ # DO NOT CHANGE THIS FILE! # This config is generated by SALTSTACK # and all change will be overrided on next salt call -{% from "mysql/defaults.yaml" import rawmap with context %} -{% from "mysql/supported_params.yaml" import supported_params with context %} -{%- set datamap = salt['grains.filter_by'](rawmap, grain='os', merge=salt['pillar.get']('mysql:server:lookup')) %} -{%- for section_name, supparams in supported_params.items() %} -[{{ section_name }}] - {%- for allowedparam in supparams|default([]) %} - {%- set indents = 40 - allowedparam|count %} - {%- set mparam = salt['pillar.get']('mysql:server:'+section_name+':'+allowedparam, false) %} - {%- if mparam %} - {%- if mparam == "noarg_present" %} -{{ allowedparam }} - {%- else %} -{{ allowedparam }}{{ '='|indent(indents, true) }} {{ mparam }} - {%- endif %} - {%- else %} - {%- if datamap.config.sections[section_name] is defined %} - {%- if datamap.config.sections[section_name][allowedparam] is defined %} - {%- if datamap.config.sections[section_name][allowedparam] == "noarg_present" %} -{{ allowedparam }} - {%- else %} -{{ allowedparam }}{{ '='|indent(indents, true) }} {{ datamap.config.sections[section_name][allowedparam] }} - {%- endif %} - {%- endif %} - {%- endif %} - {%- endif %} - {%- endfor %} -{% endfor %} +{#- +===== FETCH DATA ===== +-#} +{%- from "mysql/defaults.yaml" import rawmap with context -%} +{%- from "mysql/supported_sections.yaml" import supported_sections with context -%} +{%- set datamap = salt['grains.filter_by'](rawmap, grain='os', merge=salt['pillar.get']('mysql:server:lookup')) -%} +{#- +===== COMBINE DATA ===== +-#} +{%- set goodParamList = datamap.config.sections -%} +{%- for section_name in supported_sections -%} + {%- set sectdict = datamap.config.sections[section_name] | default({}) -%} + {%- for mparam, mvalue in salt['pillar.get']('mysql:server:'+section_name, {}).items() -%} + {%- set mparamUnderscore = mparam | replace('-','_') -%} + {%- do sectdict.update({mparamUnderscore:mvalue}) -%} + {%- endfor -%} + {%- do goodParamList.update({section_name:sectdict}) -%} +{%- endfor -%} +{#- +===== PRINT DATA ===== +-#} +{%- for sname,sdata in goodParamList.items() -%} +{%- if sdata %} + +[{{ sname }}] +{%- for mparam, mvalue in sdata.items()|default([]) -%} +{%- set indents = 40 - mparam|count %} +{% if mvalue == "noarg_present" -%} +{{ mparam }} +{%- else -%} +{{ mparam }}{{ '='|indent(indents, true) }} {{ mvalue }} +{%- endif -%} +{%- endfor -%} +{%- endif -%} +{%- endfor %} + +{{ datamap.config.append | default('') }} diff --git a/mysql/server.sls b/mysql/server.sls index 5b9ac2d..5983e15 100644 --- a/mysql/server.sls +++ b/mysql/server.sls @@ -56,7 +56,7 @@ mysql_delete_anonymous_user_{{ host }}: mysqld: pkg.installed: - name: {{ mysql.server }} -{% if os_family == 'Debian' %} +{% if os_family == 'Debian' and mysql_root_password %} - require: - debconf: mysql_debconf {% endif %} diff --git a/mysql/supported_sections.yaml b/mysql/supported_sections.yaml new file mode 100644 index 0000000..e2e9d2a --- /dev/null +++ b/mysql/supported_sections.yaml @@ -0,0 +1,16 @@ +# vim +{% load_yaml as supported_sections %} +- client +- mysql +- mysqldump +- mysqld_safe +- mysqlhotcopy +- mysqladmin +- mysqlcheck +- mysqlimport +- mysqlshow +- myisampack +- myisamchk +- isamchk +- mysqld +{% endload %} diff --git a/pillar.example b/pillar.example index e6cf278..3dfa6c6 100644 --- a/pillar.example +++ b/pillar.example @@ -5,13 +5,12 @@ mysql: user: mysql # my.cnf sections changes mysqld: + # you can use either underscore or hyphen in param names bind-address: 0.0.0.0 + log_bin: /var/log/mysql/mysql-bin.log port: 3307 - server-id: 1 - log-bin: mysql-bin - binlog-do-db: foo + binlog_do_db: foo auto_increment_increment: 5 - max_connect_errors: 4294967295 mysql: # my.cnf param that not require value no-auto-rehash: noarg_present @@ -28,7 +27,7 @@ mysql: load: False # Manage users - # you can get pillar for existent server using import_users.py script + # you can get pillar for existent server using scripts/import_users.py script user: frank: password: 'somepass' @@ -58,5 +57,4 @@ mysql: server: mysql-server client: mysql-client service: mysql-service - config: /etc/mysql/my.cnf python: python-mysqldb diff --git a/scripts/import_users.py b/scripts/import_users.py new file mode 100755 index 0000000..456aed0 --- /dev/null +++ b/scripts/import_users.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python +"This script helps you to get mysql.user pillar from existent mysql server" + +import argparse +import MySQLdb +import re + +__author__ = "Egor Potiomkin" +__version__ = "1.0" +__email__ = "eg13reg@gmail.com" + +parser = argparse.ArgumentParser() +parser.add_argument('host', metavar='IP', help='host where you want to get users') +parser.add_argument('user', metavar='user', help='mysql user that can show grants') +parser.add_argument('password', metavar='password', help='user password') +args = parser.parse_args() + +# PARSE GRANTS +mysqlcon = MySQLdb.connect(host=args.host,user=args.user,passwd=args.password,db="mysql",use_unicode=True, charset='utf8') +mysqlCur = mysqlcon.cursor(MySQLdb.cursors.DictCursor) + +mysqlCur.execute(r'''select user,host from mysql.user;''') +rows = mysqlCur.fetchall() +users = [] + +for row in rows: + users.append({'name': row['user'], 'host': row['host']}); + +mysqlCur = mysqlcon.cursor() +grants = [] +for user in users: + q = r'''show grants for '%s'@'%s';''' % (user['name'], user['host']) + try: + user['grants'] = [] + mysqlCur.execute(q) + rows = mysqlCur.fetchall() + for row in rows: + mpass = re.search( + r"""GRANT USAGE ON \*\.\* TO .* IDENTIFIED BY PASSWORD '(\*[A-F0-9]*)\'""", + row[0]) + if mpass is None: + mgrant = re.search( + r"""GRANT ([\s,A-Z]+) ON `?([a-zA-Z0-9_\-*\\]*)`?\.`?([a-zA-Z0-9_\-*\\]*)`? TO .*""", + row[0]) + if mgrant is not None: + user['grants'].append({'grant': [x.strip() for x in mgrant.group(1).split(',')], 'database': mgrant.group(2).replace('\\',''), 'table': mgrant.group(3).replace('\\','')}) + else: + print "ERROR: CAN NOT PARSE GRANTS: ",row[0] + else: + user['password'] = mpass.group(1) + + except MySQLdb.DatabaseError: + print "Error while getting grants for '%s'@'%s'" % (user['name'], user['host']) +#raise SystemExit +# PRINT RESULT +""" PRINT EXAMPLE +mysql: + user: + username: + host: host + password_hash: '*2792A97371B2D17789364A22A9B35D180166571A' + databases: + - database: testbase + table: table1 + grants: ['select'] +""" +print "mysql:" +print " user:" +for user in users: + print " %s:" % user['name'] + print " host: '%s'" % user['host'] + if ('password' in user): + print " password_hash: '%s'" % user['password'] + print " databases:" + for grant in user['grants']: + print " - database: '%s'" % grant['database'] + print " table: '%s'" % grant['table'] + print " grants: ['%s']" % "','".join(grant['grant']).lower()