From 773681bb524453f21703d0be290fdb99e391ce36 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 25 Jul 2019 11:40:12 +0200 Subject: [PATCH] Testing MySQL tuning --- CHANGELOG.md | 11 ++ wo/cli/plugins/stack.py | 224 +++++++++++++++-------------- wo/cli/templates/upstream.mustache | 10 ++ 3 files changed, 134 insertions(+), 111 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 087e42c..cf90ba2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,17 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### v3.9.x - [Unreleased] +#### Added + +- MySQL configuration tuning +- Cronjob to optimize MySQL databases weekly + +#### Changed + +- `wo maintenance` refactored +- Improved debug log +- Updated Nginx configuration process to not overwrite files with custom data (htpasswd-wo, acl.conf etc..) + ### v3.9.6.2 - 2019-07-24 #### Changed diff --git a/wo/cli/plugins/stack.py b/wo/cli/plugins/stack.py index c7134f7..f8cca41 100644 --- a/wo/cli/plugins/stack.py +++ b/wo/cli/plugins/stack.py @@ -24,7 +24,6 @@ from wo.core.addswap import WOSwap from wo.core.apt_repo import WORepo from wo.core.aptget import WOAptGet from wo.core.cron import WOCron -from wo.core.checkfqdn import check_fqdn from wo.core.download import WODownload from wo.core.extract import WOExtract from wo.core.fileutils import WOFileUtils @@ -244,6 +243,7 @@ class WOStackController(CementBaseController): """Post activity after installation of packages""" if (apt_packages): + # Nginx configuration if set(WOVariables.wo_nginx).issubset(set(apt_packages)): if set(["nginx"]).issubset(set(apt_packages)): # Fix for white screen death with NGINX PLUS @@ -254,16 +254,18 @@ class WOStackController(CementBaseController): wo_nginx.write('fastcgi_param \tSCRIPT_FILENAME ' '\t$request_filename;\n') - if not (os.path.isfile('/etc/nginx/common/wpfc-php72.conf')): - # Change WordOpsVersion in nginx.conf file - WOFileUtils.searchreplace(self, "/etc/nginx/nginx.conf", - "# add_header", - "add_header") + if os.path.isfile('/etc/nginx/nginx.conf'): + if WOFileUtils.grep(self, '/etc/nginx/fastcgi_params', + '# add_header'): + # Change WordOpsVersion in nginx.conf file + WOFileUtils.searchreplace(self, "/etc/nginx/nginx.conf", + "# add_header", + "add_header") - WOFileUtils.searchreplace(self, "/etc/nginx/nginx.conf", - "\"WordOps\"", - "\"WordOps v{0}\"" - .format(WOVariables.wo_version)) + WOFileUtils.searchreplace(self, "/etc/nginx/nginx.conf", + "\"WordOps\"", + "\"WordOps v{0}\"" + .format(WOVariables.wo_version)) data = dict() Log.debug(self, 'Writting the nginx configuration to ' 'file /etc/nginx/conf.d/blockips.conf') @@ -291,7 +293,6 @@ class WOStackController(CementBaseController): (data), 'upstream.mustache', out=wo_nginx) wo_nginx.close() - if not (os.path.isfile('/etc/nginx/conf.d/stub_status.conf')): data = dict(phpconf=True if WOAptGet.is_installed(self, 'php7.2-fpm') else False) @@ -303,7 +304,6 @@ class WOStackController(CementBaseController): (data), 'stub_status.mustache', out=wo_nginx) wo_nginx.close() - if not (os.path.isfile('/etc/nginx/conf.d/webp.conf')): data = dict() Log.debug(self, 'Writting the nginx configuration to ' 'file /etc/nginx/conf.d/webp.conf') @@ -329,15 +329,8 @@ class WOStackController(CementBaseController): '/etc/nginx/common') os.makedirs('/etc/nginx/common') + if os.path.exists('/etc/nginx/common'): data = dict(webroot=WOVariables.wo_webroot) - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/acl.conf') - wo_nginx = open('/etc/nginx/common/acl.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'acl.mustache', - out=wo_nginx) - wo_nginx.close() - Log.debug(self, 'Writting the nginx configuration to ' 'file /etc/nginx/common/locations-wo.conf') wo_nginx = open('/etc/nginx/common/locations-wo.conf', @@ -387,7 +380,6 @@ class WOStackController(CementBaseController): wo_nginx.close() # php73 conf - if not os.path.isfile("/etc/nginx/common/php73.conf"): # data = dict() Log.debug(self, 'Writting the nginx configuration to ' 'file /etc/nginx/common/php73.conf') @@ -429,22 +421,60 @@ class WOStackController(CementBaseController): out=wo_nginx) wo_nginx.close() - if not os.path.isfile("/etc/nginx/common/locations-wo.conf"): + # create redis conf + data = dict() Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/locations-wo.conf') - wo_nginx = open('/etc/nginx/common/locations-wo.conf', + 'file /etc/nginx/common/redis-php72.conf') + wo_nginx = open('/etc/nginx/common/redis-php72.conf', encoding='utf-8', mode='w') - self.app.render((data), 'locations.mustache', + self.app.render((data), 'redis.mustache', + out=wo_nginx) + wo_nginx.close() + + data = dict() + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/redis-php73.conf') + wo_nginx = open('/etc/nginx/common/redis-php73.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'redis-php7.mustache', out=wo_nginx) wo_nginx.close() - if not os.path.isfile("/etc/nginx/common/release"): with open("/etc/nginx/common/release", "a") as release_file: release_file.write("v{0}" .format(WOVariables.wo_version)) release_file.close() + # Following files should not be overwrited + + if not os.path.isfile('/etc/nginx/common/acl.conf'): + data = dict(webroot=WOVariables.wo_webroot) + Log.debug(self, 'Writting the nginx configuration to ' + 'file /etc/nginx/common/acl.conf') + wo_nginx = open('/etc/nginx/common/acl.conf', + encoding='utf-8', mode='w') + self.app.render((data), 'acl.mustache', + out=wo_nginx) + wo_nginx.close() + + # add redis cache format if not already done + if (os.path.isfile("/etc/nginx/nginx.conf") and + not os.path.isfile("/etc/nginx/conf.d" + "/redis.conf")): + with open("/etc/nginx/conf.d/" + "redis.conf", "a") as redis_file: + redis_file.write("# Log format Settings\n" + "log_format rt_cache_redis " + "'$remote_addr " + "$upstream_response_time " + "$srcache_fetch_status " + "[$time_local] '\n" + "'$http_host \"$request\" $status" + " $body_bytes_sent '\n" + "'\"$http_referer\" " + "\"$http_user_agent\"';\n") + # Nginx-Plus does not have nginx # package structure like this # So creating directories @@ -590,64 +620,9 @@ class WOStackController(CementBaseController): else: WOService.restart_service(self, 'nginx') - # create redis conf is redis is installed - if WOAptGet.is_installed(self, 'redis-server'): - if (os.path.isfile("/etc/nginx/nginx.conf") and - not os.path.isfile("/etc/nginx/common/" - "redis-php72.conf")): - - data = dict() - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/redis-php72.conf') - wo_nginx = open('/etc/nginx/common/redis-php72.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'redis.mustache', - out=wo_nginx) - wo_nginx.close() - - if (os.path.isfile("/etc/nginx/nginx.conf") and - not os.path.isfile("/etc/nginx/common/" - "redis-php73.conf")): - data = dict() - Log.debug(self, 'Writting the nginx configuration to ' - 'file /etc/nginx/common/redis-php73.conf') - wo_nginx = open('/etc/nginx/common/redis-php73.conf', - encoding='utf-8', mode='w') - self.app.render((data), 'redis-php7.mustache', - out=wo_nginx) - wo_nginx.close() - - # add redis upstream if not available in upstream.conf - if os.path.isfile("/etc/nginx/conf.d/upstream.conf"): - if not WOFileUtils.grep(self, "/etc/nginx/conf.d/" - "upstream.conf", - "redis"): - with open("/etc/nginx/conf.d/upstream.conf", - "a") as redis_file: - redis_file.write("upstream redis {\n" - " server 127.0.0.1:6379;\n" - " keepalive 10;\n}\n") - - # add redis cache format if not already done - if (os.path.isfile("/etc/nginx/nginx.conf") and - not os.path.isfile("/etc/nginx/conf.d" - "/redis.conf")): - with open("/etc/nginx/conf.d/" - "redis.conf", "a") as redis_file: - redis_file.write("# Log format Settings\n" - "log_format rt_cache_redis " - "'$remote_addr " - "$upstream_response_time " - "$srcache_fetch_status " - "[$time_local] '\n" - "'$http_host \"$request\" $status" - " $body_bytes_sent '\n" - "'\"$http_referer\" " - "\"$http_user_agent\"';\n") # setup nginx common folder for php7 if self.app.pargs.php73: - if (os.path.isdir("/etc/nginx/common") and - not os.path.isfile("/etc/nginx/common/php73.conf")): + if os.path.isdir("/etc/nginx/common"): data = dict() Log.debug(self, 'Writting the nginx configuration to ' @@ -682,9 +657,6 @@ class WOStackController(CementBaseController): out=wo_nginx) wo_nginx.close() - if (os.path.isdir("/etc/nginx/common") and - not os.path.isfile("/etc/nginx/common/" - "redis-php73.conf")): data = dict() Log.debug(self, 'Writting the nginx configuration to ' 'file /etc/nginx/common/redis-php73.conf') @@ -694,22 +666,9 @@ class WOStackController(CementBaseController): out=wo_nginx) wo_nginx.close() - if os.path.isfile("/etc/nginx/conf.d/upstream.conf"): - if not WOFileUtils.grep(self, "/etc/nginx/conf.d/" - "upstream.conf", - "php73"): - with open("/etc/nginx/conf.d/" - "upstream.conf", "a") as php_file: - php_file.write("upstream php73 {\nserver unix:" - "/var/run/php/php73-fpm.sock;\n}\n" - "upstream debug73 {\nserver " - "127.0.0.1:9173;\n}\n") # create nginx configuration for redis if set(WOVariables.wo_redis).issubset(set(apt_packages)): - if (os.path.isfile("/etc/nginx/nginx.conf") and - not os.path.isfile("/etc/nginx/common/" - "redis-php72.conf")): - + if os.path.isdir('/etc/nginx/common'): data = dict() Log.debug(self, 'Writting the nginx configuration to ' 'file /etc/nginx/common/redis-php72.conf') @@ -1076,17 +1035,6 @@ class WOStackController(CementBaseController): encoding='utf-8', mode='w') config_file.write(config) config_file.close() - else: - try: - WOShellExec.cmd_exec(self, "sed -i \"/#max_conn" - "ections/a wait_timeout = 30 \\n" - "interactive_timeout = 60 \\n" - "performance_schema = 0\\n" - "query_cache_type = 1 \" " - "/etc/mysql/my.cnf") - except CommandExecutionError as e: - Log.debug(self, "{0}".format(e)) - Log.error(self, "Unable to update MySQL file") WOFileUtils.chmod(self, "/usr/bin/mysqltuner", 0o775) WOCron.setcron_weekly(self, 'mysqlcheck -Aos --auto-repair ' @@ -1789,7 +1737,6 @@ class WOStackController(CementBaseController): WOShellExec.cmd_exec(self, "systemctl enable redis-server") if os.path.isfile("/etc/redis/redis.conf"): wo_ram = psutil.virtual_memory().total / (1024 * 1024) - wo_swap = psutil.swap_memory().total / (1024 * 1024) if wo_ram < 512: Log.debug(self, "Setting maxmemory variable to " "{0} in redis.conf" @@ -1823,6 +1770,61 @@ class WOStackController(CementBaseController): "allkeys-lru/' " "/etc/redis/redis.conf") WOService.restart_service(self, 'redis-server') + if 'mariadb-server' in apt_packages: + # setting innodb memory usage + wo_ram = psutil.virtual_memory().total / (1024 * 1024) + wo_ram_innodb = int(wo_ram*0.3) + wo_ram_log_buffer = int(wo_ram_innodb*0.25) + wo_ram_log_size = int(wo_ram_log_buffer*0.5) + # replacing default values + if os.path.isfile("/etc/mysql/my.cnf"): + Log.debug(self, "Tuning MySQL configuration") + WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", + "innodb_buffer_pool_size = 256M", + "innodb_buffer_pool_size = {0}M" + .format(wo_ram_innodb)) + WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", + "innodb_log_buffer_size = 8M", + "innodb_log_buffer_size = {0}M" + .format(wo_ram_log_buffer)) + WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", + "#innodb_log_file_size = 50M", + "innodb_log_file_size = {0}M" + .format(wo_ram_log_size)) + WOFileUtils.searchreplace(self, + "/etc/mysql/my.cnf", + "wait_timeout " + " = 600", + "wait_timeout " + " = 120") + # disabling mariadb binlog + WOFileUtils.searchreplace(self, + "/etc/mysql/my.cnf", + "log_bin " + " = /var/log/" + "mysql/mariadb-bin", + "#log_bin " + " = /var/log/" + "mysql/mariadb-bin") + WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", + "log_bin_index " + "= /var/log/mysql/" + "mariadb-bin.index", + "#log_bin_index " + "= /var/log/mysql/" + "mariadb-bin.index") + WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", + "expire_logs_days " + "= 10", + "#expire_logs_days " + "= 10") + WOFileUtils.searchreplace(self, "/etc/mysql/my.cnf", + "max_binlog_size " + "= 100M", + "#max_binlog_size " + "= 100M") + WOService.restart_service(self, 'mysql') + if disp_msg: if (self.msg): for msg in self.msg: diff --git a/wo/cli/templates/upstream.mustache b/wo/cli/templates/upstream.mustache index 26e06b9..686a6a5 100644 --- a/wo/cli/templates/upstream.mustache +++ b/wo/cli/templates/upstream.mustache @@ -74,3 +74,13 @@ upstream netdata { server 127.0.0.1:19999; keepalive 64; } + +#------------------------------- +# Redis +#------------------------------- + +# Redis cache upstream +upstream redis { + server 127.0.0.1:6379; + keepalive 10; +}