diff --git a/README.rst b/README.rst index d1edad2..a8653cd 100644 --- a/README.rst +++ b/README.rst @@ -6,8 +6,8 @@ Install the MySQL client and/or server. .. note:: - See the full `Salt Formulas installation and usage instructions - `_. + See the full `Salt Formulas installation and usage instructions + `_. Available states ================ @@ -53,14 +53,41 @@ Create and manage MySQL databases. Install mysql python bindings. ``mysql.user`` ----------------- +-------------- Create and manage MySQL database users with definable GRANT privileges. -The state accepts MySQL hashed passwords or clear text. Hashed password have priority. +The state accepts MySQL hashed passwords or clear text. Hashed password have +priority. .. note:: - See the `salt.states.mysql_user `_ docs for additional information on configuring hashed passwords. - + See the `salt.states.mysql_user + `_ + docs for additional information on configuring hashed passwords. + Make sure to **quote the passwords** in the pillar so YAML doesn't throw an exception. + +Updating the supported parameters +================================= + +The ``supported_params.yaml`` file contains the full listing of options that +are acceptable in the MySQL options file. On occassion, especially on new +releases of MySQL, this file may need to be updated. To update, run the +supplied script (requires Python 3.x):: + + ./scripts/parse_supported_params.py -o ./mysql/supported_params.yaml + +This script will scrape the options from the official MySQL documentation +online, and thus requires web access. Scraping is inherently brittle, though +this script has been defensively coded, where possible. + +Once the ``supported_params.yaml`` file has been updated, commit the result to +the repository. + +Support for new applications +---------------------------- + +To add support for configuration of other MySQL applications, add the URL and +section identifier into the relevant section of the script. Consult the +comments in the code to determine where your section should be added. diff --git a/mysql/supported_params.yaml b/mysql/supported_params.yaml index 1a68612..ff2496b 100644 --- a/mysql/supported_params.yaml +++ b/mysql/supported_params.yaml @@ -1,121 +1,980 @@ # vim +{#- Do not edit this YAML file by hand. See README.rst for how to update -#} {% load_yaml as supported_params %} -client: - - port - - socket -mysqld_safe: - - socket - - nice - - log-error - - pid-file - - err-log -mysqld: - - user - - pid-file - - socket - - port - - basedir - - datadir - - tmpdir - - lc-messages-dir - - skip-external-locking +# From https://dev.mysql.com/doc/refman/5.7/en/mysql-command-options.html +{% for section in ['client', 'mysql'] %} +{{ section }}: + - help + - auto-rehash + - auto-vertical-output + - batch + - binary-mode - bind-address - - key_buffer - - key_buffer_size + - character-sets-dir + - column-names + - column-type-info + - comments + - compress + - connect-expired-password + - database + - debug + - debug-check + - debug-info + - default-auth + - default-character-set + - defaults-extra-file + - defaults-file + - defaults-group-suffix + - delimiter + - disable-named-commands + - enable-cleartext-plugin + - execute + - force + - histignore + - host + - html + - ignore-spaces + - init-command + - line-numbers + - local-infile + - login-path + - named-commands + - no-auto-rehash + - no-beep + - no-defaults + - one-database + - pager + - password + - pipe + - plugin-dir + - port + - print-defaults + - prompt + - protocol + - quick + - raw + - reconnect + - safe-updates + - secure-auth + - server-public-key-path + - shared-memory-base-name + - show-warnings + - sigint-ignore + - silent + - skip-column-names + - skip-line-numbers + - socket + - ssl + - syslog + - table + - tee + - unbuffered + - user + - verbose + - version + - vertical + - wait + - xml + - connect_timeout - max_allowed_packet - - thread_stack - - thread_cache_size - - myisam-recover - - query_cache_limit - - query_cache_size - - max_connections - - table_cache - - thread_concurrency - - wait_timeout - - tmp_table_size - - max_heap_table_size - - open_files_limit - - general_log_file - - general_log - - log_error - - log_slow_queries - - long_query_time - - log-queries-not-using-indexes - - expire_logs_days - - max_binlog_size - - server-id - - log_bin - - binlog_do_db - - replicate-do-db - - auto_increment_increment - - auto_increment_offset - - max_connect_errors - - join_buffer_size + - max_join_size + - net_buffer_length + - select_limit +{% endfor %} +# From https://dev.mysql.com/doc/refman/5.7/en/mysqldump.html +mysqldump: + - bind-address + - compress + - default-auth + - host + - login-path + - password + - pipe + - plugin-dir + - port + - protocol + - secure-auth + - socket + - ssl + - user + - max_allowed_packet + - net_buffer_length + - defaults-extra-file + - defaults-file + - defaults-group-suffix + - no-defaults + - print-defaults + - add-drop-database + - add-drop-table + - add-drop-trigger + - all-tablespaces + - no-create-db + - no-create-info + - no-tablespaces + - replace + - allow-keywords + - comments + - debug + - debug-check + - debug-info + - dump-date + - force + - log-error + - skip-comments + - verbose + - help + - version + - character-sets-dir + - default-character-set + - no-set-names + - set-charset + - apply-slave-statements + - delete-master-logs + - dump-slave + - include-master-host-port + - master-data + - set-gtid-purged + - compact + - compatible + - complete-insert + - create-options + - fields + - hex-blob + - lines-terminated-by + - quote-names + - result-file + - tab + - tz-utc + - xml + - all-databases + - databases + - events + - ignore-error + - ignore-table + - no-data + - routines + - tables + - triggers + - where + - disable-keys + - extended-insert + - insert-ignore + - opt + - quick + - skip-opt + - add-locks + - flush-logs + - flush-privileges + - lock-all-tables + - lock-tables + - no-autocommit + - order-by-primary + - shared-memory-base-name + - single-transaction +# From https://dev.mysql.com/doc/refman/5.7/en/mysqld-safe.html +mysqld_safe: + - help + - basedir + - core-file-size + - datadir + - defaults-extra-file + - defaults-file + - ledir + - log-error + - malloc-lib + - mysqld + - mysqld-version + - nice + - no-defaults + - open-files-limit + - pid-file + - plugin-dir + - port + - skip-kill-mysqld + - socket + - syslog + - syslog-tag + - timezone + - user +# From http://dev.mysql.com/doc/refman/5.6/en/mysqlhotcopy.html +mysqlhotcopy: + - help + - addtodest + - allowold + - checkpoint - chroot - - ssl-ca - - ssl-cert - - ssl-key - - sort_buffer_size - - read_buffer_size - - read_rnd_buffer_size + - debug + - dryrun + - flushlog + - host + - keepold + - method + - noindices + - password + - port + - old_server + - quiet + - record_log_pos + - regexp + - resetmaster + - resetslave + - socket + - suffix + - tmpdir + - user +# From http://dev.mysql.com/doc/refman/5.7/en/mysqladmin.html +mysqladmin: + - help + - bind-address + - character-sets-dir + - compress + - count + - debug + - debug-check + - debug-info + - default-auth + - default-character-set + - defaults-extra-file + - defaults-file + - defaults-group-suffix + - enable-cleartext-plugin + - force + - host + - login-path + - no-beep + - no-defaults + - password + - pipe + - plugin-dir + - port + - print-defaults + - protocol + - relative + - show-warnings + - secure-auth + - shared-memory-base-name + - silent + - sleep + - socket + - ssl + - user + - verbose + - version + - vertical + - wait + - connect_timeout + - shutdown_timeout +# From http://dev.mysql.com/doc/refman/5.7/en/mysqlcheck.html +mysqlcheck: + - help + - all-databases + - all-in-1 + - analyze + - auto-repair + - bind-address + - character-sets-dir + - check + - check-only-changed + - check-upgrade + - compress + - databases + - debug + - debug-check + - debug-info + - default-character-set + - defaults-extra-file + - defaults-file + - defaults-group-suffix + - extended + - default-auth + - fast + - fix-db-names + - fix-table-names + - force + - host + - login-path + - medium-check + - no-defaults + - optimize + - password + - pipe + - plugin-dir + - port + - print-defaults + - protocol + - quick + - repair + - secure-auth + - shared-memory-base-name + - silent + - skip-database + - socket + - ssl + - tables + - use-frm + - user + - verbose + - version + - write-binlog +# From http://dev.mysql.com/doc/refman/5.7/en/mysqlimport.html +mysqlimport: + - help + - bind-address + - character-sets-dir + - columns + - compress + - debug + - debug-check + - debug-info + - default-character-set + - default-auth + - defaults-extra-file + - defaults-file + - defaults-group-suffix + - delete + - fields + - force + - host + - ignore + - ignore-lines + - lines-terminated-by + - local + - lock-tables + - login-path + - low-priority + - no-defaults + - password + - pipe + - plugin-dir + - port + - print-defaults + - protocol + - replace + - secure-auth + - shared-memory-base-name + - silent + - socket + - ssl + - user + - use-threads + - verbose + - version +# From http://dev.mysql.com/doc/refman/5.7/en/mysqlshow.html +mysqlshow: + - help + - bind-address + - character-sets-dir + - compress + - count + - debug + - debug-check + - debug-info + - default-character-set + - default-auth + - defaults-extra-file + - defaults-file + - defaults-group-suffix + - host + - keys + - login-path + - no-defaults + - password + - pipe + - plugin-dir + - port + - print-defaults + - protocol + - secure-auth + - shared-memory-base-name + - show-table-type + - socket + - ssl + - status + - user + - verbose + - version +# From http://dev.mysql.com/doc/refman/5.7/en/myisampack.html +myisampack: + - help + - backup + - character-sets-dir + - debug + - force + - join + - silent + - test + - tmpdir + - verbose + - version + - wait +# From https://dev.mysql.com/doc/refman/5.7/en/myisamchk.html +{% for section in ['myisamchk', 'isamchk'] %} +{{ section }}: + - analyze + - backup + - block-search + - check + - check-only-changed + - correct-checksum + - data-file-length + - debug + - decode_bits + - defaults-extra-file + - defaults-file + - defaults-group-suffix + - description + - extend-check + - fast + - force + - force + - ft_max_word_len + - ft_min_word_len + - ft_stopword_file + - HELP + - help + - information + - key_buffer_size + - keys-used + - max-record-length + - medium-check + - myisam_block_size - myisam_sort_buffer_size - - slow_query_log - - slow_query_log_file - - innodb_file_per_table - - innodb_data_home_dir - - innodb_data_file_path - - innodb_log_group_home_dir - - innodb_buffer_pool_size + - no-defaults + - parallel-recover + - print-defaults + - quick + - read_buffer_size + - read-only + - recover + - safe-recover + - set-auto-increment + - set-collation + - silent + - sort_buffer_size + - sort-index + - sort_key_blocks + - sort-records + - sort-recover + - stats_method + - tmpdir + - unpack + - update-state + - verbose + - version + - write_buffer_size +{% endfor %} +# From https://dev.mysql.com/doc/refman/5.7/en/mysqld-option-tables.html +mysqld: + - abort-slave-event-count + - allow-suspicious-udfs + - ansi + - autocommit + - basedir + - big-tables + - bind-address + - binlog_cache_size + - binlog-checksum + - binlog_direct_non_transactional_updates + - binlog-do-db + - binlog-format + - binlog_group_commit_sync_delay + - binlog_group_commit_sync_no_delay_count + - binlog-ignore-db + - binlog-row-event-max-size + - binlog_row_image + - binlog-rows-query-log-events + - binlog_stmt_cache_size + - binlogging_impossible_mode + - block_encryption_mode + - bootstrap + - bulk_insert_buffer_size + - character-set-client-handshake + - character-set-filesystem + - character-set-server + - character-sets-dir + - chroot + - collation-server + - completion_type + - concurrent_insert + - connect_timeout + - console + - core-file + - daemon_memcached_enable_binlog + - daemon_memcached_engine_lib_name + - daemon_memcached_engine_lib_path + - daemon_memcached_option + - daemon_memcached_r_batch_size + - daemon_memcached_w_batch_size + - datadir + - debug + - debug-sync-timeout + - default-authentication-plugin + - default_authentication_plugin + - default_password_lifetime + - default-storage-engine + - default-time-zone + - default_tmp_storage_engine + - default_week_format + - delay-key-write + - delayed_insert_limit + - delayed_insert_timeout + - delayed_queue_size + - des-key-file + - disconnect_on_expired_password + - disconnect-slave-event-count + - div_precision_increment + - enable-named-pipe + - enforce_gtid_consistency + - enforce-gtid-consistency + - event-scheduler + - executed-gtids-compression-period + - exit-info + - expire_logs_days + - explicit_defaults_for_timestamp + - external-locking + - federated + - flush + - flush_time + - ft_boolean_syntax + - ft_max_word_len + - ft_min_word_len + - ft_query_expansion_limit + - ft_stopword_file + - gdb + - general-log + - general_log_file + - group_concat_max_len + - gtid-mode + - help + - ignore-builtin-innodb + - ignore-db-dir + - init_connect + - init-file + - init_slave + - innodb + - innodb_adaptive_flushing + - innodb_adaptive_flushing_lwm + - innodb_adaptive_hash_index + - innodb_adaptive_max_sleep_delay - innodb_additional_mem_pool_size - - innodb_log_file_size - - innodb_log_buffer_size + - innodb_api_bk_commit_interval + - innodb_api_disable_rowlock + - innodb_api_enable_binlog + - innodb_api_enable_mdl + - innodb_api_trx_level + - innodb_autoextend_increment + - innodb_autoinc_lock_mode + - innodb_buffer_pool_chunk_size + - innodb_buffer_pool_dump_at_shutdown + - innodb_buffer_pool_dump_now + - innodb_buffer_pool_dump_pct + - innodb_buffer_pool_filename + - innodb_buffer_pool_instances + - innodb_buffer_pool_load_abort + - innodb_buffer_pool_load_at_startup + - innodb_buffer_pool_load_now + - innodb_buffer_pool_size + - innodb_change_buffer_max_size + - innodb_change_buffering + - innodb_checksum_algorithm + - innodb_checksums + - innodb_cmp_per_index_enabled + - innodb_commit_concurrency + - innodb_compression_failure_threshold_pct + - innodb_compression_level + - innodb_compression_pad_pct_max + - innodb_concurrency_tickets + - innodb_create_intrinsic + - innodb_data_file_path + - innodb_data_home_dir + - innodb_disable_sort_file_cache + - innodb_doublewrite + - innodb_fast_shutdown + - innodb_file_format + - innodb_file_format_check + - innodb_file_format_max + - innodb_file_per_table + - innodb_fill_factor - innodb_flush_log_at_trx_commit - innodb_flush_method - - innodb_thread_concurrency - - symbolic-links - - character-set-server - - log-error - - table_open_cache - - net_buffer_length - - language - - log-bin - - innodb_log_files_in_group + - innodb_flush_neighbors + - innodb_flushing_avg_loops + - innodb_force_load_corrupted + - innodb_force_recovery + - innodb_ft_aux_table + - innodb_ft_cache_size + - innodb_ft_enable_diag_print + - innodb_ft_enable_stopword + - innodb_ft_max_token_size + - innodb_ft_min_token_size + - innodb_ft_num_word_optimize + - innodb_ft_result_cache_limit + - innodb_ft_server_stopword_table + - innodb_ft_sort_pll_degree + - innodb_ft_total_cache_size + - innodb_ft_user_stopword_table + - innodb_io_capacity + - innodb_io_capacity_max + - innodb_large_prefix - innodb_lock_wait_timeout -mysqldump: - - quick - - quote-names + - innodb_locks_unsafe_for_binlog + - innodb_log_buffer_size + - innodb_log_compressed_pages + - innodb_log_file_size + - innodb_log_files_in_group + - innodb_log_group_home_dir + - innodb_log_write_ahead_size + - innodb_lru_scan_depth + - innodb_max_dirty_pages_pct + - innodb_max_dirty_pages_pct_lwm + - innodb_max_purge_lag + - innodb_max_purge_lag_delay + - innodb_max_undo_log_size + - innodb_monitor_disable + - innodb_monitor_enable + - innodb_monitor_reset + - innodb_monitor_reset_all + - innodb_old_blocks_pct + - innodb_old_blocks_time + - innodb_online_alter_log_max_size + - innodb_open_files + - innodb_optimize_fulltext_only + - innodb_optimize_point_storage + - innodb_page_cleaners + - innodb_page_size + - innodb_print_all_deadlocks + - innodb_purge_batch_size + - innodb_purge_rseg_truncate_frequency + - innodb_purge_threads + - innodb_random_read_ahead + - innodb_read_ahead_threshold + - innodb_read_io_threads + - innodb_read_only + - innodb_replication_delay + - innodb_rollback_on_timeout + - innodb_rollback_segments + - innodb_sort_buffer_size + - innodb_spin_wait_delay + - innodb_stats_auto_recalc + - innodb_stats_method + - innodb_stats_on_metadata + - innodb_stats_persistent + - innodb_stats_persistent_sample_pages + - innodb_stats_sample_pages + - innodb_stats_transient_sample_pages + - innodb-status-file + - innodb_status_output + - innodb_status_output_locks + - innodb_strict_mode + - innodb_support_xa + - innodb_sync_array_size + - innodb_sync_spin_loops + - innodb_table_locks + - innodb_temp_data_file_path + - innodb_thread_concurrency + - innodb_thread_sleep_delay + - innodb_undo_directory + - innodb_undo_log_truncate + - innodb_undo_logs + - innodb_undo_tablespaces + - innodb_use_native_aio + - innodb_use_sys_malloc + - innodb_write_io_threads + - interactive_timeout + - internal_tmp_disk_storage_engine + - join_buffer_size + - keep_files_on_create + - key_buffer_size + - key_cache_age_threshold + - key_cache_block_size + - key_cache_division_limit + - language + - large-pages + - lc-messages + - lc-messages-dir + - lock_wait_timeout + - log-bin + - log-bin-index + - log-bin-trust-function-creators + - log_bin_use_v + - log-bin-use-v + - log-error + - log_error_verbosity + - log-isam + - log-output + - log-queries-not-using-indexes + - log-raw + - log-short-format + - log-slave-updates + - log_slave_updates + - log-slow-admin-statements + - log-slow-slave-statements + - log_syslog + - log_syslog_facility + - log_syslog_include_pid + - log_syslog_tag + - log-tc + - log-tc-size + - log_timestamps + - log-warnings + - long_query_time + - low-priority-updates + - lower_case_table_names + - master-info-file + - master_info_repository + - master-info-repository + - master-retry-count + - master-verify-checksum - max_allowed_packet - - character-sets-dir - - default-character-set -mysql: - - no-auto-rehash - - character-sets-dir - - default-character-set -isamchk: - - key_buffer + - max_binlog_cache_size + - max-binlog-dump-events + - max_binlog_size + - max_binlog_stmt_cache_size + - max_connect_errors + - max_connections + - max_delayed_threads + - max_error_count + - max_heap_table_size + - max_join_size + - max_length_for_sort_data + - max_prepared_stmt_count + - max_relay_log_size + - max_seeks_for_key + - max_sort_length + - max_sp_recursion_depth + - max_statement_time + - max_user_connections + - max_write_lock_count + - memlock + - min-examined-row-limit + - myisam-block-size + - myisam_data_pointer_size + - myisam_max_sort_file_size + - myisam_mmap_size + - myisam-recover-options + - myisam_repair_threads + - myisam_sort_buffer_size + - myisam_stats_method + - myisam_use_mmap + - ndb-batch-size + - ndb-blob-write-batch-bytes + - ndb-cluster-connection-pool + - ndb-connectstring + - ndb-deferred-constraints + - ndb_deferred_constraints + - ndb_distribution + - ndb-distribution + - ndb_eventbuffer_max_alloc + - ndb_force_send + - ndb_index_stat_enable + - ndb_index_stat_option + - ndb-log-apply-status + - ndb_log_apply_status + - ndb-log-empty-epochs + - ndb-log-transaction-id + - ndb_log_updated_only + - ndb-mgmd-host + - ndb_optimized_node_selection + - ndb_report_thresh_binlog_epoch_slip + - ndb_report_thresh_binlog_mem_usage + - ndb_show_foreign_key_mock_tables + - ndb_use_transactions + - ndb-wait-setup + - net_buffer_length + - net_read_timeout + - net_retry_count + - net_write_timeout + - new + - offline_mode + - old + - old-alter-table + - old-style-user-limits + - open-files-limit + - optimizer_prune_level + - optimizer_search_depth + - optimizer_switch + - partition + - performance_schema + - performance_schema_accounts_size + - performance-schema-consumer-events-stages-current + - performance-schema-consumer-events-stages-history + - performance-schema-consumer-events-stages-history-long + - performance-schema-consumer-events-statements-current + - performance-schema-consumer-events-statements-history + - performance-schema-consumer-events-statements-history-long + - performance-schema-consumer-events-transactions-current + - performance-schema-consumer-events-transactions-history + - performance-schema-consumer-events-transactions-history-long + - performance-schema-consumer-events-waits-current + - performance-schema-consumer-events-waits-history + - performance-schema-consumer-events-waits-history-long + - performance-schema-consumer-global-instrumentation + - performance-schema-consumer-statements-digest + - performance-schema-consumer-thread-instrumentation + - performance_schema_digests_size + - performance_schema_events_stages_history_long_size + - performance_schema_events_stages_history_size + - performance_schema_events_statements_history_long_size + - performance_schema_events_statements_history_size + - performance_schema_events_transactions_history_long_size + - performance_schema_events_transactions_history_size + - performance_schema_events_waits_history_long_size + - performance_schema_events_waits_history_size + - performance_schema_hosts_size + - performance-schema-instrument + - performance_schema_max_cond_classes + - performance_schema_max_cond_instances + - performance_schema_max_file_classes + - performance_schema_max_file_handles + - performance_schema_max_file_instances + - performance_schema_max_memory_classes + - performance_schema_max_metadata_locks + - performance_schema_max_mutex_classes + - performance_schema_max_mutex_instances + - performance_schema_max_prepared_statements_instances + - performance_schema_max_program_instances + - performance_schema_max_rwlock_classes + - performance_schema_max_rwlock_instances + - performance_schema_max_socket_classes + - performance_schema_max_socket_instances + - performance_schema_max_stage_classes + - performance_schema_max_statement_classes + - performance_schema_max_statement_stack + - performance_schema_max_table_handles + - performance_schema_max_table_instances + - performance_schema_max_thread_classes + - performance_schema_max_thread_instances + - performance_schema_session_connect_attrs_size + - performance_schema_setup_actors_size + - performance_schema_setup_objects_size + - performance_schema_users_size + - pid-file + - plugin + - plugin_dir + - plugin-load + - plugin-load-add + - port + - port-open-timeout + - preload_buffer_size + - profiling_history_size + - query_alloc_block_size + - query_cache_limit + - query_cache_min_res_unit + - query_cache_size + - query_cache_type + - query_cache_wlock_invalidate + - query_prealloc_size + - range_alloc_block_size + - read_buffer_size + - read_only + - read_rnd_buffer_size + - relay-log + - relay-log-index + - relay_log_index + - relay-log-info-file + - relay_log_info_file + - relay-log-info-repository + - relay_log_purge + - relay_log_recovery + - relay-log-recovery + - relay_log_space_limit + - replicate-do-db + - replicate-do-table + - replicate-ignore-db + - replicate-ignore-table + - replicate-rewrite-db + - replicate-same-server-id + - replicate-wild-do-table + - replicate-wild-ignore-table + - report-host + - report-password + - report-port + - report-user + - rpl_stop_slave_timeout + - safe-user-create + - secure-auth + - secure-file-priv + - server-id + - session_track_schema + - session_track_state_change + - session_track_system_variables + - shared_memory + - show-slave-auth-info + - simplified_binlog_gtid_recovery + - skip-character-set-client-handshake + - skip-concurrent-insert + - skip-event-scheduler + - skip_external_locking + - skip-grant-tables + - skip-host-cache + - skip-name-resolve + - skip-ndbcluster + - skip-networking + - skip-new + - skip-partition + - skip-show-database + - skip-slave-start + - skip-ssl + - skip-stack-trace + - slave_allow_batching + - slave_checkpoint_group + - slave-checkpoint-group + - slave_checkpoint_period + - slave-checkpoint-period + - slave_compressed_protocol + - slave_exec_mode + - slave-load-tmpdir + - slave-max-allowed-packet + - slave-net-timeout + - slave-parallel-type + - slave-parallel-workers + - slave-rows-search-algorithms + - slave-skip-errors + - slave-sql-verify-checksum + - slave_transaction_retries + - slave_type_conversions + - slow_launch_time + - slow-query-log + - slow_query_log_file + - slow-start-timeout + - socket - sort_buffer_size - - read_buffer - - write_buffer - - character-sets-dir -myisamchk: - - key_buffer - - sort_buffer_size - - read_buffer - - write_buffer -mysqlhotcopy: - - interactive-timeout -mysqladmin: - - character-sets-dir - - default-character-set -mysqlcheck: - - character-sets-dir - - default-character-set -mysqlimport: - - character-sets-dir - - default-character-set -mysqlshow: - - character-sets-dir - - default-character-set -myisampack: - - character-sets-dir -{% endload %} + - sporadic-binlog-dump-fail + - sql-mode + - ssl + - ssl-ca + - ssl-capath + - ssl-cert + - ssl-cipher + - ssl-crl + - ssl-crlpath + - ssl-key + - standalone + - stored_program_cache + - super-large-pages + - symbolic-links + - sync_binlog + - sync_frm + - sync_master_info + - sync_relay_log + - sync_relay_log_info + - sysdate-is-now + - tc-heuristic-recover + - temp-pool + - thread_cache_size + - thread_concurrency + - thread_handling + - thread_stack + - timed_mutexes + - tmp_table_size + - tmpdir + - transaction_alloc_block_size + - transaction-isolation + - transaction_prealloc_size + - transaction-read-only + - updatable_views_with_limit + - user + - validate-password + - verbose + - wait_timeout +{% endload %} \ No newline at end of file diff --git a/scripts/html_table_parser.py b/scripts/html_table_parser.py new file mode 100644 index 0000000..e376f92 --- /dev/null +++ b/scripts/html_table_parser.py @@ -0,0 +1,61 @@ +#------------------------------------------------------------------------------- +# Name: html_table_parser +# Purpose: Simple class for parsing an (x)html string to extract tables. +# Written in python3 +# +# Author: Josua Schmid +# +# Created: 05.03.2014 +# Copyright: (c) Josua Schmid 2014 +# Licence: GPLv3 +#------------------------------------------------------------------------------- +from html.parser import HTMLParser + +class HTMLTableParser(HTMLParser): + """ This class serves as a html table parser. It is able to parse multiple + tables which you feed in. You can access the result per .tables field. + """ + def __init__(self): + HTMLParser.__init__(self) + self.__in_td = False + self.__in_th = False + self.__current_table = [] + self.__current_row = [] + self.__current_cell = [] + self.tables = [] + + def handle_starttag(self, tag, attrs): + """ We need to remember the opening point for the content of interest. + The other tags (, ) are only handled at the closing point. + """ + if tag == 'td': + self.__in_td = True + if tag == 'th': + self.__in_th = True + + def handle_data(self, data): + """ This is where we save content to a cell """ + if self.__in_td ^ self.__in_th: + self.__current_cell.append(data.strip()) + + def handle_endtag(self, tag): + """ Here we exit the tags. If the closing tag is , we know that we + can save our currently parsed cells to the current table as a row and + prepare for a new row. If the closing tag is
, we save the + current table and prepare for a new one. + """ + if tag == 'td': + self.__in_td = False + if tag == 'th': + self.__in_th = False + + if (tag == 'td') ^ (tag == 'th'): + final_cell = " ".join(self.__current_cell).strip() + self.__current_row.append(final_cell) + self.__current_cell = [] + if tag == 'tr': + self.__current_table.append(self.__current_row) + self.__current_row = [] + if tag == 'table': + self.tables.append(self.__current_table) + self.__current_table = [] \ No newline at end of file diff --git a/scripts/parse_supported_params.py b/scripts/parse_supported_params.py new file mode 100755 index 0000000..33c4ee6 --- /dev/null +++ b/scripts/parse_supported_params.py @@ -0,0 +1,167 @@ +#!/usr/bin/python3 +# coding: utf-8 +import argparse +import re +import sys +import urllib.request +from html_table_parser import HTMLTableParser + +# Regex for parsing options on MySQL documentation pages +# Options are (normally) specified as command-line options +# as anchor tags on the page. Certain documentation pages only +# show options in table listings, however. +OPTION_REGEX = '' +OPTION_TABLE_REGEX = '^(--)?([A-Za-z_-]+).*$' + +# File heading, as per the original supported_params file +FILE_HEADER = """# vim +{#- Do not edit this YAML file by hand. See README.rst for how to update -#} +{% load_yaml as supported_params %} +""" +FILE_FOOTER = """{% endload %}""" + +# Standard YAML template for options for a section +YAML_TEMPLATE = """# From %(url)s +%(section)s: + - %(options)s +""" + +# For rendering Jinja that handles multiple sections +# Several MySQL utilities use exactly the same options +# Note this variable is string formatted twice, hence the double-double % signs +YAML_TEMPLATE_MULTI = """# From %%(url)s +{%%%% for section in %(sections)r %%%%} +{{ section }}: + - %%(options)s +{%%%% endfor %%%%} +""" + +# Options specified in HTML documentation as command-line options +# like so . +# Structure is (section_id, documentation_url, yaml_template_str) +SECTIONS = ( + ('mysql', + 'https://dev.mysql.com/doc/refman/5.7/en/mysql-command-options.html', + YAML_TEMPLATE_MULTI % {'sections': ['client', 'mysql']}), + ('mysqldump', + 'https://dev.mysql.com/doc/refman/5.7/en/mysqldump.html', + YAML_TEMPLATE), + ('mysqld_safe', + 'https://dev.mysql.com/doc/refman/5.7/en/mysqld-safe.html', + YAML_TEMPLATE), + # Removed in MySQL 5.7 + ('mysqlhotcopy', + 'http://dev.mysql.com/doc/refman/5.6/en/mysqlhotcopy.html', + YAML_TEMPLATE), + ('mysqladmin', + 'http://dev.mysql.com/doc/refman/5.7/en/mysqladmin.html', + YAML_TEMPLATE), + ('mysqlcheck', + 'http://dev.mysql.com/doc/refman/5.7/en/mysqlcheck.html', + YAML_TEMPLATE), + ('mysqlimport', + 'http://dev.mysql.com/doc/refman/5.7/en/mysqlimport.html', + YAML_TEMPLATE), + ('mysqlshow', + 'http://dev.mysql.com/doc/refman/5.7/en/mysqlshow.html', + YAML_TEMPLATE), + ('myisampack', + 'http://dev.mysql.com/doc/refman/5.7/en/myisampack.html', + YAML_TEMPLATE), +) +# Options specified in documentation as command-line and +# option file values in a table only. +SECTIONS_VIA_TABLE = ( + ('myisamchk', + 'https://dev.mysql.com/doc/refman/5.7/en/myisamchk.html', + YAML_TEMPLATE_MULTI % {'sections': ['myisamchk', 'isamchk']}), +) +# Server options specified in documentation +SERVER_OPTIONS = ( + 'mysqld', + 'https://dev.mysql.com/doc/refman/5.7/en/mysqld-option-tables.html', + YAML_TEMPLATE +) + + +def read_url(url): + """ Read the given URL and decode the response as UTF-8. + """ + request = urllib.request.Request(url) + response = urllib.request.urlopen(request) + return response.read().decode('utf-8') + + +def read_first_table(url): + """ Read the given URL, parse the result, and return the first table. + """ + xhtml = read_url(url) + parser = HTMLTableParser() + parser.feed(xhtml) + return parser.tables[0] # Use first table on the page + + +def parse_anchors(url, section): + """ Return parsed options from option anchors at the given URL. + """ + return re.findall(OPTION_REGEX % section, read_url(url)) + + +def parse_tables(url, section): + """ Return arsed options from HTML tables at the given URL. + + This matches the given option regex, and ensures that the + first row of the table is ignored; it contains headings only. + """ + table = read_first_table(url) + return [re.match(OPTION_TABLE_REGEX, row[0]).groups()[1] + for row in table[1:]] + + +def parse_mysqld(url, section): + """ Return the parsed options from the huge mysqld table. + + The massive options table shows variables and options and + highlights where they can be used. The following code only + pulls out those that are marked as 'Yes' for use in an option file. + """ + table = read_first_table(url) + # Find which column holds the option file data + option_index = table[0].index('Option File') + # Only pull out options able to be used in an options file + return [re.match(OPTION_TABLE_REGEX, row[0]).groups()[1] + for row in table[1:] + if len(row) >= option_index + 1 and + row[option_index].strip().lower() == 'yes'] + + +def print_yaml_options(sections, parser, file=sys.stdout): + """ Perform really basic templating for output. + + A YAML library could be used, but we avoid extra dependencies by + just using string formatting. + """ + for section, url, yaml in sections: + options = parser(url, section) + print(yaml % {'section': section, + 'options': '\n - '.join(options), + 'url': url}, end='', file=file) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description='Scrape the MySQL documentation to obtain' + ' all the supported parameters for different utilities.') + parser.add_argument('--output', + '-o', + help='File output location', + default=sys.stdout) + config = parser.parse_args() + output = open(config.output, 'w') if isinstance(config.output, str) \ + else config.output + + print(FILE_HEADER, end='', file=output) + print_yaml_options(SECTIONS, parse_anchors, file=output) + print_yaml_options(SECTIONS_VIA_TABLE, parse_tables, file=output) + print_yaml_options((SERVER_OPTIONS,), parse_mysqld, file=output) + print(FILE_FOOTER, end='', file=output)