From 22501d1616d7619e882d691129bc263381efb54b Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Fri, 16 Oct 2020 13:33:58 +0200 Subject: [PATCH 01/25] Fix php versions not installed with WordOps - Remove all php versions not installed with WordOps without removing php versions installed with EasyEngine or manually (check if php-fpm executable exist) --- install | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/install b/install index a30a82c..ac684ef 100755 --- a/install +++ b/install @@ -704,13 +704,18 @@ wo_init() { wo_php_fix() { local php_versions="5.6 7.0 7.1 7.2 7.3 7.4" - apt-get autoremove --assume-yes --purge php8.0-* php-igbinary php-memcached php-msgpack php-redis php-imagick php-xdebug + apt-get autoremove --assume-yes --purge php8.0-* php-igbinary php-memcached php-msgpack php-redis php-imagick php-xdebug php-memcache for php_version in $php_versions; do - if [ -f "/usr/sbin/php-fpm$php_version" ]; then - local php_extensions="igbinary memcached msgpack redis imagick xdebug" - for php_ext in $php_extensions; do - apt-get -option=Dpkg::options::=--force-confmiss --option=Dpkg::options::=--force-confold --assume-yes install "php$php_version-$php_ext" - done + if [ -f "/usr/bin/php$php_version" ]; then + if [ -f "/usr/sbin/php-fpm$php_version" ]; then + local php_extensions="igbinary memcached msgpack redis imagick xdebug" + for php_ext in $php_extensions; do + apt-get -option=Dpkg::options::=--force-confmiss --option=Dpkg::options::=--force-confold --assume-yes install "php$php_version-$php_ext" + done + else + apt-get autoremove --assume-yes --purge "php$php_version-*" + fi + fi done } From fd66b5475f8b16321fc7497975d350f9c07aa84f Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Sat, 17 Oct 2020 01:48:50 +0200 Subject: [PATCH 02/25] Fix Redis upgrade on Ubuntu 20.04 LTS * Fix typo in bash completion --- config/bash_completion.d/wo_auto.rc | 6 +++--- tests/travis.sh | 6 +++--- wo/cli/plugins/stack_upgrade.py | 12 +++--------- 3 files changed, 9 insertions(+), 15 deletions(-) diff --git a/config/bash_completion.d/wo_auto.rc b/config/bash_completion.d/wo_auto.rc index e7642dd..85ea252 100644 --- a/config/bash_completion.d/wo_auto.rc +++ b/config/bash_completion.d/wo_auto.rc @@ -74,17 +74,17 @@ _wo_complete() # HANDLE EVERYTHING AFTER THE THIRD LEVEL NAMESPACE "install" | "purge" | "remove" ) COMPREPLY=( $(compgen \ - -W "--web --admin --security --nginx --php --php73 --php74 --mysql --wpcli --phpmyadmin --adminer --utils --redis --phpredisadmin --composer --netdata --fail2ban --ufw --dashboard --proftpd --clamav --sendmail --ngxblocker --mysqlclient --mysqltuner --extplorer --nanorc --cheat --all --force" \ + -W "--web --admin --security --nginx --php72 --php73 --php74 --mysql --wpcli --phpmyadmin --adminer --utils --redis --phpredisadmin --composer --netdata --fail2ban --ufw --dashboard --proftpd --clamav --sendmail --ngxblocker --mysqlclient --mysqltuner --extplorer --nanorc --cheat --all --force" \ -- $cur) ) ;; "upgrade" ) COMPREPLY=( $(compgen \ - -W "--web --admin --utils --nginx --php --php73 --php74 --mysql --all --netdata --composer --phpmyadmin --adminer --dashboard --mysqtuner --wpcli --force" \ + -W "--web --admin --utils --nginx --php72 --php73 --php74 --mysql --all --netdata --composer --phpmyadmin --adminer --dashboard --mysqltuner --wpcli --force" \ -- $cur) ) ;; "start" | "stop" | "reload" | "restart" | "status") COMPREPLY=( $(compgen \ - -W "--nginx --php --php73 --php74 --mysql --redis --fail2ban --ufw --netdata -proftpd" \ + -W "--nginx --php --php72 --php73 --php74 --mysql --redis --fail2ban --ufw --netdata -proftpd" \ -- $cur) ) ;; "list") diff --git a/tests/travis.sh b/tests/travis.sh index 073d75a..41077c3 100644 --- a/tests/travis.sh +++ b/tests/travis.sh @@ -80,7 +80,7 @@ echo echo -e "${CGREEN}#############################################${CEND}" echo -e ' wo site update --php74 ' echo -e "${CGREEN}#############################################${CEND}" -other_site_types='mysql php72 php74 wp wpfc wpsc wpredis wpce wprocket wpsubdomain wpsubdir ngxblocker' +other_site_types='mysql php72 php73 wp wpfc wpsc wpredis wpce wprocket wpsubdomain wpsubdir' for site in $other_site_types; do echo -ne " Updating site to $site php74 [..]\r" if { @@ -103,7 +103,7 @@ echo echo -e "${CGREEN}#############################################${CEND}" echo -e ' wo site update --php73 ' echo -e "${CGREEN}#############################################${CEND}" -other_site_types='html mysql wp php72 php73 wpfc wpsc wpredis wpce wprocket wpsubdomain wpsubdir ngxblocker' +other_site_types='html mysql wp php72 php73 wpfc wpsc wpredis wpce wprocket wpsubdomain wpsubdir' for site in $other_site_types; do echo -ne " Updating site to $site php73 [..]\r" if { @@ -126,7 +126,7 @@ echo echo -e "${CGREEN}#############################################${CEND}" echo -e ' wo site update --php72 ' echo -e "${CGREEN}#############################################${CEND}" -other_site_types='mysql php72 php73 php74 wp wpfc wpsc wpredis wpce wprocket wpsubdomain wpsubdir ngxblocker' +other_site_types='mysql php72 php73 wp wpfc wpsc wpredis wpce wprocket wpsubdomain wpsubdir' for site in $other_site_types; do echo -ne " Updating site to $site php72 [..]\r" if { diff --git a/wo/cli/plugins/stack_upgrade.py b/wo/cli/plugins/stack_upgrade.py index f73df85..4c86a91 100644 --- a/wo/cli/plugins/stack_upgrade.py +++ b/wo/cli/plugins/stack_upgrade.py @@ -298,19 +298,13 @@ class WOStackUpgradeController(CementBaseController): # additional pre_pref if "nginx-custom" in apt_packages: pre_pref(self, WOVar.wo_nginx) - if "php7.2-fpm" in apt_packages: - WOAptGet.remove(self, ['php7.2-fpm'], - auto=False, purge=True) - if "php7.3-fpm" in apt_packages: - WOAptGet.remove(self, ['php7.3-fpm'], - auto=False, purge=True) - if "php7.4-fpm" in apt_packages: - WOAptGet.remove(self, ['php7.4-fpm'], - auto=False, purge=True) # check if nginx upgrade is blocked if os.path.isfile( '/etc/apt/preferences.d/nginx-block'): post_pref(self, WOVar.wo_nginx, [], True) + # redis pre_pref + if "redis-server" in apt_packages: + pre_pref(self, WOVar.wo_redis) # upgrade packages WOAptGet.install(self, apt_packages) Log.wait(self, "Configuring APT Packages") From dff13eb34593ae2d6b69d3a807a870f5c78ca28c Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 22 Oct 2020 11:04:23 +0200 Subject: [PATCH 03/25] Testing MariaDB 10.5 --- wo/cli/plugins/stack_pref.py | 51 +++++++++--------------------------- wo/cli/templates/my.mustache | 4 +-- wo/core/variables.py | 4 +-- 3 files changed, 16 insertions(+), 43 deletions(-) diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index 2c20e5c..ae62848 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -45,39 +45,6 @@ def pre_pref(self, apt_packages): if "mariadb-server" in apt_packages: # generate random 24 characters root password chars = ''.join(random.sample(string.ascii_letters, 24)) - - Log.debug(self, "Pre-seeding MySQL") - Log.debug(self, "echo \"mariadb-server-{0} " - "mysql-server/root_password " - "password \" | " - "debconf-set-selections" - .format(WOVar.mariadb_ver)) - try: - WOShellExec.cmd_exec(self, "echo \"mariadb-server-{0} " - "mysql-server/root_password " - "password {chars}\" | " - "debconf-set-selections" - .format(WOVar.mariadb_ver, chars=chars), - log=False) - except CommandExecutionError as e: - Log.debug(self, "{0}".format(e)) - Log.error(self, "Failed to initialize MySQL package") - - Log.debug(self, "echo \"mariadb-server-{0} " - "mysql-server/root_password_again " - "password \" | " - "debconf-set-selections" - .format(WOVar.mariadb_ver)) - try: - WOShellExec.cmd_exec(self, "echo \"mariadb-server-{0} " - "mysql-server/root_password_again " - "password {chars}\" | " - "debconf-set-selections" - .format(WOVar.mariadb_ver, chars=chars), - log=False) - except CommandExecutionError as e: - Log.debug(self, "{0}".format(e)) - Log.error(self, "Failed to initialize MySQL package") # generate my.cnf root credentials mysql_config = """ [client] @@ -966,7 +933,7 @@ def post_pref(self, apt_packages, packages, upgrade=False): config_file.close() else: # make sure root account have all privileges - if "PASSWORD" not in WOShellExec.cmd_exec_stdout( + if "IDENTIFIED BY PASSWORD" not in WOShellExec.cmd_exec_stdout( self, 'mysql -e "use mysql; show grants;"'): try: if not os.path.exists('/etc/mysql/conf.d/my.cnf'): @@ -975,10 +942,13 @@ def post_pref(self, apt_packages, packages, upgrade=False): config.read('/etc/mysql/conf.d/my.cnf') chars = config['client']['password'] WOShellExec.cmd_exec( - self, "mysql -e \"use mysql; " - "GRANT ALL PRIVILEGES on " - "*.* TO 'root'@'127.0.0.1' IDENTIFIED by " - "'{0}' WITH GRANT OPTION\"".format(chars)) + self, + 'mysql -e "ALTER USER root@localhost ' + 'IDENTIFIED VIA mysql_native_password;"') + WOShellExec.cmd_exec( + self, + 'mysql -e "SET PASSWORD = ' + 'PASSWORD(\'{0}\');"'.format(chars)) WOShellExec.cmd_exec( self, 'mysql -e "flush privileges;"') except CommandExecutionError: @@ -1001,11 +971,14 @@ def post_pref(self, apt_packages, packages, upgrade=False): elif (wo_ram > 64000): wo_innodb_instance = int(64) tmp_table_size = int(256) + mariadbconf = bool(not os.path.exists( + '/etc/mysql/mariadb.conf.d/50-server.cnf')) data = dict( tmp_table_size=tmp_table_size, inno_log=wo_ram_log_size, inno_buffer=wo_ram_innodb, inno_log_buffer=wo_ram_log_buffer, - innodb_instances=wo_innodb_instance) + innodb_instances=wo_innodb_instance, + newmariadb=mariadbconf) if os.path.exists('/etc/mysql/mariadb.conf.d/50-server.cnf'): WOTemplate.deploy( self, '/etc/mysql/my.cnf', 'my.mustache', data) diff --git a/wo/cli/templates/my.mustache b/wo/cli/templates/my.mustache index 69e22bc..19ac2a8 100644 --- a/wo/cli/templates/my.mustache +++ b/wo/cli/templates/my.mustache @@ -41,7 +41,6 @@ tmpdir = /tmp lc_messages_dir = /usr/share/mysql lc_messages = en_US skip-external-locking -skip-name-resolve = 1 # # Instead of skip-networking the default is now to listen only on # localhost which is more compatible and is not less secure. @@ -191,5 +190,6 @@ key_buffer = 16M # * IMPORTANT: Additional settings that can override those from this file! # The files must end with '.cnf', otherwise they'll be ignored. # +{{#newmariadb}} !include /etc/mysql/mariadb.cnf -!includedir /etc/mysql/conf.d/ +!includedir /etc/mysql/conf.d/{{/newmariadb}} diff --git a/wo/core/variables.py b/wo/core/variables.py index 62fd5f2..879929e 100644 --- a/wo/core/variables.py +++ b/wo/core/variables.py @@ -166,7 +166,7 @@ class WOVar(): else: mariadb_ver = '10.3' else: - mariadb_ver = '10.3' + mariadb_ver = '10.5' if wo_platform_codename == 'jessie': wo_mysql = wo_mysql + ["python3-mysql.connector"] else: @@ -185,7 +185,7 @@ class WOVar(): # APT repositories wo_mysql_repo = ("deb [arch=amd64,ppc64el] " "http://mariadb.mirrors.ovh.net/MariaDB/repo/" - "10.3/{distro} {codename} main" + "10.5/{distro} {codename} main" .format(distro=wo_distro, codename=wo_platform_codename)) if wo_distro == 'ubuntu': From c72ce642b4fbd2a8c35ae10b6ba2182a624bf4c0 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 22 Oct 2020 11:15:20 +0200 Subject: [PATCH 04/25] Fix Mariadb 10.5 setup --- wo/cli/plugins/stack_pref.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index ae62848..e91f9fd 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -54,13 +54,13 @@ def pre_pref(self, apt_packages): config = configparser.ConfigParser() config.read_string(mysql_config) Log.debug(self, 'Writting configuration into MySQL file') - conf_path = "/etc/mysql/conf.d/my.cnf" + conf_path = "/etc/mysql/conf.d/my.cnf.tmp" os.makedirs(os.path.dirname(conf_path), exist_ok=True) with open(conf_path, encoding='utf-8', mode='w') as configfile: config.write(configfile) Log.debug(self, 'Setting my.cnf permission') - WOFileUtils.chmod(self, "/etc/mysql/conf.d/my.cnf", 0o600) + WOFileUtils.chmod(self, "/etc/mysql/conf.d/my.cnf.tmp", 0o600) # add nginx repository if set(WOVar.wo_nginx).issubset(set(apt_packages)): @@ -936,10 +936,10 @@ def post_pref(self, apt_packages, packages, upgrade=False): if "IDENTIFIED BY PASSWORD" not in WOShellExec.cmd_exec_stdout( self, 'mysql -e "use mysql; show grants;"'): try: - if not os.path.exists('/etc/mysql/conf.d/my.cnf'): + if not os.path.exists('/etc/mysql/conf.d/my.cnf.tmp'): Log.error(self, 'my.cnf not found') config = configparser.ConfigParser() - config.read('/etc/mysql/conf.d/my.cnf') + config.read('/etc/mysql/conf.d/my.cnf.tmp') chars = config['client']['password'] WOShellExec.cmd_exec( self, @@ -951,6 +951,9 @@ def post_pref(self, apt_packages, packages, upgrade=False): 'PASSWORD(\'{0}\');"'.format(chars)) WOShellExec.cmd_exec( self, 'mysql -e "flush privileges;"') + WOFileUtils.mvfile( + self, '/etc/mysql/conf.d/my.cnf.tmp', + '/etc/mysql/conf.d/my.cnf') except CommandExecutionError: Log.error(self, "Unable to set MySQL password") Log.info(self, "Tuning MariaDB configuration") @@ -1001,10 +1004,10 @@ def post_pref(self, apt_packages, packages, upgrade=False): # on the amount of RAM WOService.stop_service(self, 'mysql') - WOFileUtils.mvfile(self, '/var/lib/mysql/ib_logfile0', - '/var/lib/mysql/ib_logfile0.bak') - WOFileUtils.mvfile(self, '/var/lib/mysql/ib_logfile1', - '/var/lib/mysql/ib_logfile1.bak') + # WOFileUtils.mvfile(self, '/var/lib/mysql/ib_logfile0', + # '/var/lib/mysql/ib_logfile0.bak') + # WOFileUtils.mvfile(self, '/var/lib/mysql/ib_logfile1', + # '/var/lib/mysql/ib_logfile1.bak') WOService.start_service(self, 'mysql') WOCron.setcron_weekly(self, 'mysqlcheck -Aos --auto-repair ' From 55f50f583c557f3a19b372a7710bbde66b3b5067 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 22 Oct 2020 12:05:01 +0200 Subject: [PATCH 05/25] Fix MariaDB upgrade --- wo/cli/plugins/stack_pref.py | 10 ++++------ wo/cli/templates/my.mustache | 1 + 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index e91f9fd..24c7c52 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -933,11 +933,8 @@ def post_pref(self, apt_packages, packages, upgrade=False): config_file.close() else: # make sure root account have all privileges - if "IDENTIFIED BY PASSWORD" not in WOShellExec.cmd_exec_stdout( - self, 'mysql -e "use mysql; show grants;"'): + if os.path.exists('/etc/mysql/conf.d/my.cnf.tmp'): try: - if not os.path.exists('/etc/mysql/conf.d/my.cnf.tmp'): - Log.error(self, 'my.cnf not found') config = configparser.ConfigParser() config.read('/etc/mysql/conf.d/my.cnf.tmp') chars = config['client']['password'] @@ -981,10 +978,11 @@ def post_pref(self, apt_packages, packages, upgrade=False): inno_buffer=wo_ram_innodb, inno_log_buffer=wo_ram_log_buffer, innodb_instances=wo_innodb_instance, - newmariadb=mariadbconf) + newmariadb=mariadbconf, release=WOVar.wo_version) if os.path.exists('/etc/mysql/mariadb.conf.d/50-server.cnf'): WOTemplate.deploy( - self, '/etc/mysql/my.cnf', 'my.mustache', data) + self, '/etc/mysql/mariadb.conf.d/50-server.cnf', + 'my.mustache', data) else: WOTemplate.deploy( self, '/etc/mysql/my.cnf', 'my.mustache', data) diff --git a/wo/cli/templates/my.mustache b/wo/cli/templates/my.mustache index 19ac2a8..00fc28d 100644 --- a/wo/cli/templates/my.mustache +++ b/wo/cli/templates/my.mustache @@ -1,4 +1,5 @@ # MariaDB database server configuration file. +# Optimized by WordOps {{release}} # # You can copy this file to one of: # - "/etc/mysql/my.cnf" to set global options, From c3224d39e5e68dce6b88a2f67a493dd190899836 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 22 Oct 2020 14:34:03 +0200 Subject: [PATCH 06/25] Add MariaDB upgrade from 10.3 to 10.5 --- wo/cli/plugins/stack_pref.py | 15 +++++++++++++-- wo/cli/plugins/stack_upgrade.py | 20 ++++++++++++++++---- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index 24c7c52..9c140cc 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -37,12 +37,23 @@ def pre_pref(self, apt_packages): with open('/etc/apt/preferences.d/' 'MariaDB.pref', 'w') as mysql_pref_file: mysql_pref_file.write(mysql_pref) - WORepo.add(self, repo_url=WOVar.wo_mysql_repo) + if not os.path.exists('/etc/apt/sources.list.d/wo-repo.list'): + WORepo.add(self, repo_url=WOVar.wo_mysql_repo) + else: + if not WOFileUtils.grepcheck( + self, '/etc/apt/sources.list.d/wo-repo.list', + 'MariaDB/repo/10.3'): + WORepo.add(self, repo_url=WOVar.wo_mysql_repo) + else: + WOFileUtils.searchreplace( + self, '/etc/apt/sources.list.d/wo-repo.list', + '10.3', '10.5') WORepo.add_key(self, '0xcbcb082a1bb943db', keyserver='keyserver.ubuntu.com') WORepo.add_key(self, '0xF1656F24C74CD1D8', keyserver='keyserver.ubuntu.com') - if "mariadb-server" in apt_packages: + if ("mariadb-server" in apt_packages and + not os.path.exists('/etc/mysql/conf.d/my.cnf')): # generate random 24 characters root password chars = ''.join(random.sample(string.ascii_letters, 24)) # generate my.cnf root credentials diff --git a/wo/cli/plugins/stack_upgrade.py b/wo/cli/plugins/stack_upgrade.py index 4c86a91..7ac60e4 100644 --- a/wo/cli/plugins/stack_upgrade.py +++ b/wo/cli/plugins/stack_upgrade.py @@ -12,6 +12,7 @@ from wo.core.logging import Log from wo.core.shellexec import WOShellExec from wo.core.variables import WOVar from wo.core.services import WOService +from wo.core.mysql import WOMysql class WOStackUpgradeController(CementBaseController): @@ -290,14 +291,20 @@ class WOStackUpgradeController(CementBaseController): start_upgrade = input("Do you want to continue:[y/N]") if start_upgrade != "Y" and start_upgrade != "y": Log.error(self, "Not starting package update") + # additional pre_pref + if "nginx-custom" in apt_packages: + pre_pref(self, WOVar.wo_nginx) + if "mariadb-server" in apt_packages: + mariadbmajorupgrade = bool( + WOFileUtils.grepcheck( + self, '/etc/apt/sources.list.d/wo-repo.list', + 'MariaDB/repo/10.3')) + pre_pref(self, WOVar.wo_mysql) Log.wait(self, "Updating APT cache") # apt-get update WOAptGet.update(self) Log.valide(self, "Updating APT cache") - # additional pre_pref - if "nginx-custom" in apt_packages: - pre_pref(self, WOVar.wo_nginx) # check if nginx upgrade is blocked if os.path.isfile( '/etc/apt/preferences.d/nginx-block'): @@ -305,7 +312,12 @@ class WOStackUpgradeController(CementBaseController): # redis pre_pref if "redis-server" in apt_packages: pre_pref(self, WOVar.wo_redis) - # upgrade packages + # mariadb upgrade + if ("mariadb-server" in apt_packages and + mariadbmajorupgrade is True): + WOMysql.backupAll(self) + WOAptGet.remove(self, ["mariadb-server"]) + # upgrade packages WOAptGet.install(self, apt_packages) Log.wait(self, "Configuring APT Packages") post_pref(self, apt_packages, [], True) From f3129f07742282fdd0ee67184cf40d157d1c39c4 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 22 Oct 2020 14:54:54 +0200 Subject: [PATCH 07/25] Remove innodb_buffer_pool_instances --- wo/cli/templates/my.mustache | 1 - 1 file changed, 1 deletion(-) diff --git a/wo/cli/templates/my.mustache b/wo/cli/templates/my.mustache index 00fc28d..47568ad 100644 --- a/wo/cli/templates/my.mustache +++ b/wo/cli/templates/my.mustache @@ -143,7 +143,6 @@ innodb_file_per_table = 1 innodb_open_files = 500000 innodb_io_capacity = 500000 innodb_flush_method = O_DIRECT -innodb_buffer_pool_instances = 1 # # * Security Features # From e2bfcd326e1fac60988ecf3bcd453fee6c38f540 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 22 Oct 2020 16:46:42 +0200 Subject: [PATCH 08/25] Move old my.cnf after MariaDB upgrade --- wo/cli/plugins/stack_pref.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index 9c140cc..0cec9fd 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -994,6 +994,10 @@ def post_pref(self, apt_packages, packages, upgrade=False): WOTemplate.deploy( self, '/etc/mysql/mariadb.conf.d/50-server.cnf', 'my.mustache', data) + WOFileUtils.mvfile( + self, '/etc/mysql/my.cnf', '/etc/mysql/my.cnf.old') + WOFileUtils.create_symlink( + self, ['/etc/mysql/mariadb.cnf', '/etc/mysql/my.cnf']) else: WOTemplate.deploy( self, '/etc/mysql/my.cnf', 'my.mustache', data) From cbd29a968c199221b796e8d38acac5c2ef952d53 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 22 Oct 2020 20:39:25 +0200 Subject: [PATCH 09/25] Perform a full dump before upgrading MariaDB --- install | 2 +- setup.py | 2 +- wo/cli/plugins/stack_migrate.py | 2 +- wo/cli/plugins/stack_pref.py | 9 ++++-- wo/cli/plugins/stack_upgrade.py | 2 +- wo/core/mysql.py | 56 +++++++++++++++++++++++---------- wo/core/variables.py | 4 +-- 7 files changed, 52 insertions(+), 25 deletions(-) diff --git a/install b/install index ac684ef..8cbe6b8 100755 --- a/install +++ b/install @@ -9,7 +9,7 @@ # ------------------------------------------------------------------------- # wget -qO wo wops.cc && sudo -E bash wo # ------------------------------------------------------------------------- -# Version 3.12.4 - 2020-10-14 +# Version 3.13.0 - 2020-10-22 # ------------------------------------------------------------------------- # CONTENTS diff --git a/setup.py b/setup.py index d60768b..d437072 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ if os.geteuid() == 0: os.makedirs('/var/lib/wo/tmp/') setup(name='wordops', - version='3.12.4', + version='3.13.0', description='An essential toolset that eases server administration', long_description=LONG, long_description_content_type='text/markdown', diff --git a/wo/cli/plugins/stack_migrate.py b/wo/cli/plugins/stack_migrate.py index 205714e..3740a0a 100644 --- a/wo/cli/plugins/stack_migrate.py +++ b/wo/cli/plugins/stack_migrate.py @@ -26,7 +26,7 @@ class WOStackMigrateController(CementBaseController): @expose(hide=True) def migrate_mariadb(self): # Backup all database - WOMysql.backupAll(self) + WOMysql.backupAll(self, fulldump=True) if not WOVar.wo_distro == 'raspbian': if (not WOVar.wo_platform_codename == 'jessie'): diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index 0cec9fd..a2de59c 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -964,7 +964,9 @@ def post_pref(self, apt_packages, packages, upgrade=False): '/etc/mysql/conf.d/my.cnf') except CommandExecutionError: Log.error(self, "Unable to set MySQL password") - Log.info(self, "Tuning MariaDB configuration") + WOGit.add(self, ["/etc/mysql"], + msg="Adding MySQL into Git") + Log.wait(self, "Tuning MariaDB configuration") if not os.path.isfile("/etc/mysql/my.cnf.default-pkg"): WOFileUtils.copyfile(self, "/etc/mysql/my.cnf", "/etc/mysql/my.cnf.default-pkg") @@ -1013,15 +1015,16 @@ def post_pref(self, apt_packages, packages, upgrade=False): 'mariadb.service.d/limits.conf', '[Service]\nLimitNOFILE=500000') WOShellExec.cmd_exec(self, 'systemctl daemon-reload') + Log.valide(self, "Tuning MySQL configuration") # set innodb_buffer_pool_instances depending # on the amount of RAM - WOService.stop_service(self, 'mysql') + WOService.restart_service(self, 'mysql') + # WOFileUtils.mvfile(self, '/var/lib/mysql/ib_logfile0', # '/var/lib/mysql/ib_logfile0.bak') # WOFileUtils.mvfile(self, '/var/lib/mysql/ib_logfile1', # '/var/lib/mysql/ib_logfile1.bak') - WOService.start_service(self, 'mysql') WOCron.setcron_weekly(self, 'mysqlcheck -Aos --auto-repair ' '> /dev/null 2>&1', diff --git a/wo/cli/plugins/stack_upgrade.py b/wo/cli/plugins/stack_upgrade.py index 7ac60e4..ea4cd8b 100644 --- a/wo/cli/plugins/stack_upgrade.py +++ b/wo/cli/plugins/stack_upgrade.py @@ -315,7 +315,7 @@ class WOStackUpgradeController(CementBaseController): # mariadb upgrade if ("mariadb-server" in apt_packages and mariadbmajorupgrade is True): - WOMysql.backupAll(self) + WOMysql.backupAll(self, fulldump=True) WOAptGet.remove(self, ["mariadb-server"]) # upgrade packages WOAptGet.install(self, apt_packages) diff --git a/wo/core/mysql.py b/wo/core/mysql.py index d9af2ff..cdab230 100644 --- a/wo/core/mysql.py +++ b/wo/core/mysql.py @@ -88,7 +88,7 @@ class WOMysql(): finally: connection.close() - def backupAll(self): + def backupAll(self, fulldump=False): import subprocess try: Log.info(self, "Backing up database at location: " @@ -98,24 +98,47 @@ class WOMysql(): Log.debug(self, 'Creating directory' '/var/lib/wo-backup/mysql') os.makedirs('/var/lib/wo-backup/mysql') - - db = subprocess.check_output(["/usr/bin/mysql " - "-Bse \'show databases\'"], - universal_newlines=True, - shell=True).split('\n') - for dbs in db: - if dbs == "": - continue - Log.info(self, "Backing up {0} database".format(dbs)) + if not fulldump: + db = subprocess.check_output( + ["/usr/bin/mysql " + "-Bse \'show databases\'"], + universal_newlines=True, + shell=True).split('\n') + for dbs in db: + if dbs == "": + continue + Log.info(self, "Backing up {0} database".format(dbs)) + p1 = subprocess.Popen( + "/usr/bin/mysqldump {0} --max_allowed_packet=1024M " + "--single-transaction ".format(dbs), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, shell=True) + p2 = subprocess.Popen( + "/usr/bin/zstd -T0 -c > " + "/var/lib/wo-backup/mysql/{0}{1}.sql.zst" + .format(dbs, WOVar.wo_date), + stdin=p1.stdout, shell=True) + # Allow p1 to receive a SIGPIPE if p2 exits + p1.stdout.close() + output = p1.stderr.read() + p1.wait() + if p1.returncode == 0: + Log.debug(self, "done") + else: + Log.error(self, output.decode("utf-8")) + else: + Log.info(self, "Backing up all databases") p1 = subprocess.Popen( - "/usr/bin/mysqldump {0} --max_allowed_packet=1024M " - "--single-transaction ".format(dbs), - stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + "/usr/bin/mysqldump --all-databases " + "--max_allowed_packet=1024M " + "--single-transaction --events", + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, shell=True) p2 = subprocess.Popen( "/usr/bin/zstd -T0 -c > " - "/var/lib/wo-backup/mysql/{0}{1}.sql.zst" - .format(dbs, WOVar.wo_date), stdin=p1.stdout, shell=True) - # Allow p1 to receive a SIGPIPE if p2 exits + "/var/lib/wo-backup/mysql/fulldump-{0}.sql.zst" + .format(WOVar.wo_date), + stdin=p1.stdout, shell=True) p1.stdout.close() output = p1.stderr.read() p1.wait() @@ -123,6 +146,7 @@ class WOMysql(): Log.debug(self, "done") else: Log.error(self, output.decode("utf-8")) + except Exception as e: Log.error(self, "Error: process exited with status %s" % e) diff --git a/wo/core/variables.py b/wo/core/variables.py index 879929e..6af68de 100644 --- a/wo/core/variables.py +++ b/wo/core/variables.py @@ -14,7 +14,7 @@ class WOVar(): """Intialization of core variables""" # WordOps version - wo_version = "3.12.4" + wo_version = "3.13.0" # WordOps packages versions wo_wp_cli = "2.4.0" wo_adminer = "4.7.5" @@ -83,7 +83,7 @@ class WOVar(): while not match(r"^[A-Za-z0-9\.\+_-]+@[A-Za-z0-9\._-]+\.[a-zA-Z]*$", wo_email): print("Whoops, seems like you made a typo - " - "the e-mailaddress is invalid...") + "the e-mail address is invalid...") wo_email = input("Enter your email: ") git.config("--global", "user.name", "{0}".format(wo_user)) From bed285abc4ad15c9bcf2317c40dbe5b68f558fb2 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Fri, 23 Oct 2020 12:50:31 +0200 Subject: [PATCH 10/25] Use wo stack migrate for MariaDB major upgrade --- tests/travis.sh | 18 +++++++ wo/cli/controllers/base.py | 2 +- wo/cli/plugins/stack_migrate.py | 95 ++++++++------------------------- wo/cli/plugins/stack_pref.py | 16 +----- wo/cli/plugins/stack_upgrade.py | 15 +----- 5 files changed, 44 insertions(+), 102 deletions(-) diff --git a/tests/travis.sh b/tests/travis.sh index 41077c3..d0e825e 100644 --- a/tests/travis.sh +++ b/tests/travis.sh @@ -226,6 +226,24 @@ if [ -z "$1" ]; then fi done fi + +echo -e "${CGREEN}#############################################${CEND}" +echo -e ' wo stack migrate --mariadb ' +echo -e "${CGREEN}#############################################${CEND}" + + echo -ne " Upgrading mariadb [..]\r" + if { + wo stack migrate --mariadb --force + } >>/var/log/wo/test.log; then + echo -ne " Upgrading mariadb [${CGREEN}OK${CEND}]\\r" + echo -ne '\n' + else + echo -e " Upgrading mariadb [${CRED}FAIL${CEND}]" + echo -ne '\n' + exit_script + + fi + echo -e "${CGREEN}#############################################${CEND}" echo -e ' wo clean ' echo -e "${CGREEN}#############################################${CEND}" diff --git a/wo/cli/controllers/base.py b/wo/cli/controllers/base.py index 9558442..95be6a6 100644 --- a/wo/cli/controllers/base.py +++ b/wo/cli/controllers/base.py @@ -8,7 +8,7 @@ VERSION = WOVar.wo_version BANNER = """ WordOps v%s -Copyright (c) 2019 WordOps. +Copyright (c) 2020 WordOps. """ % VERSION diff --git a/wo/cli/plugins/stack_migrate.py b/wo/cli/plugins/stack_migrate.py index 3740a0a..be1dabb 100644 --- a/wo/cli/plugins/stack_migrate.py +++ b/wo/cli/plugins/stack_migrate.py @@ -1,10 +1,8 @@ -import configparser -import os - from cement.core.controller import CementBaseController, expose -from wo.core.apt_repo import WORepo +from wo.cli.plugins.stack_pref import post_pref, pre_pref from wo.core.aptget import WOAptGet +from wo.core.fileutils import WOFileUtils from wo.core.logging import Log from wo.core.mysql import WOMysql from wo.core.shellexec import WOShellExec @@ -19,7 +17,10 @@ class WOStackMigrateController(CementBaseController): description = ('Migrate stack safely') arguments = [ (['--mariadb'], - dict(help="Migrate database to MariaDB", + dict(help="Migrate/Upgrade database to MariaDB", + action='store_true')), + (['--force'], + dict(help="Force Packages upgrade without any prompt", action='store_true')), ] @@ -28,95 +29,45 @@ class WOStackMigrateController(CementBaseController): # Backup all database WOMysql.backupAll(self, fulldump=True) - if not WOVar.wo_distro == 'raspbian': - if (not WOVar.wo_platform_codename == 'jessie'): - wo_mysql = ["mariadb-server", "percona-toolkit", - "python3-mysqldb", "mariadb-backup"] - else: - wo_mysql = ["mariadb-server", "percona-toolkit", - "python3-mysql.connector"] - else: - wo_mysql = ["mariadb-server", "percona-toolkit", - "python3-mysqldb"] - # Add MariaDB repo Log.info(self, "Adding repository for MariaDB, please wait...") - - mysql_pref = ("Package: *\nPin: origin sfo1.mirrors.digitalocean.com" - "\nPin-Priority: 1000\n") - with open('/etc/apt/preferences.d/' - 'MariaDB.pref', 'w') as mysql_pref_file: - mysql_pref_file.write(mysql_pref) - - WORepo.add(self, repo_url=WOVar.wo_mysql_repo) - Log.debug(self, 'Adding key for {0}' - .format(WOVar.wo_mysql_repo)) - WORepo.add_key(self, '0xcbcb082a1bb943db', - keyserver="keyserver.ubuntu.com") - - config = configparser.ConfigParser() - if os.path.exists('/etc/mysql/conf.d/my.cnf'): - config.read('/etc/mysql/conf.d/my.cnf') - else: - config.read(os.path.expanduser("~")+'/.my.cnf') - - try: - chars = config['client']['password'] - except Exception as e: - Log.error(self, "Error: process exited with error %s" - % e) - - Log.debug(self, "Pre-seeding MariaDB") - Log.debug(self, "echo \"mariadb-server-10.3 " - "mysql-server/root_password " - "password \" | " - "debconf-set-selections") - WOShellExec.cmd_exec(self, "echo \"mariadb-server-10.3 " - "mysql-server/root_password " - "password {chars}\" | " - "debconf-set-selections" - .format(chars=chars), - log=False) - Log.debug(self, "echo \"mariadb-server-10.3 " - "mysql-server/root_password_again " - "password \" | " - "debconf-set-selections") - WOShellExec.cmd_exec(self, "echo \"mariadb-server-10.3 " - "mysql-server/root_password_again " - "password {chars}\" | " - "debconf-set-selections" - .format(chars=chars), - log=False) + pre_pref(self, WOVar.wo_mysql) # Install MariaDB - apt_packages = wo_mysql Log.info(self, "Updating apt-cache, hang on...") WOAptGet.update(self) Log.info(self, "Installing MariaDB, hang on...") - WOAptGet.remove(self, ["mysql-common", "libmysqlclient18"]) + WOAptGet.remove(self, ["mariadb-server"]) WOAptGet.auto_remove(self) - WOAptGet.install(self, apt_packages) + WOAptGet.install(self, WOVar.wo_mysql) + post_pref(self, WOVar.wo_mysql, []) + WOShellExec.cmd_exec(self, 'systemctl daemon-reload') + WOFileUtils.mvfile( + self, '/etc/mysql/my.cnf', '/etc/mysql/my.cnf.old') + WOFileUtils.create_symlink( + self, ['/etc/mysql/mariadb.cnf', '/etc/mysql/my.cnf']) @expose(hide=True) def default(self): - if ((not self.app.pargs.mariadb)): + pargs = self.app.pargs + if ((not pargs.mariadb)): self.app.args.print_help() - if self.app.pargs.mariadb: + if pargs.mariadb: if WOVar.wo_mysql_host != "localhost": Log.error( self, "Remote MySQL server in use, skipping local install") - if (WOShellExec.cmd_exec(self, "mysqladmin ping") and - (not WOAptGet.is_installed(self, 'mariadb-server'))): + if (WOShellExec.cmd_exec(self, "mysqladmin ping")): Log.info(self, "If your database size is big, " "migration may take some time.") Log.info(self, "During migration non nginx-cached parts of " "your site may remain down") - start_migrate = input("Type \"mariadb\" to continue:") - if start_migrate != "mariadb": - Log.error(self, "Not starting migration") + if not pargs.force: + start_upgrade = input("Do you want to continue:[y/N]") + if start_upgrade != "Y" and start_upgrade != "y": + Log.error(self, "Not starting package update") self.migrate_mariadb() else: Log.error(self, "Your current MySQL is not alive or " diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index a2de59c..ca619a9 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -37,17 +37,7 @@ def pre_pref(self, apt_packages): with open('/etc/apt/preferences.d/' 'MariaDB.pref', 'w') as mysql_pref_file: mysql_pref_file.write(mysql_pref) - if not os.path.exists('/etc/apt/sources.list.d/wo-repo.list'): - WORepo.add(self, repo_url=WOVar.wo_mysql_repo) - else: - if not WOFileUtils.grepcheck( - self, '/etc/apt/sources.list.d/wo-repo.list', - 'MariaDB/repo/10.3'): - WORepo.add(self, repo_url=WOVar.wo_mysql_repo) - else: - WOFileUtils.searchreplace( - self, '/etc/apt/sources.list.d/wo-repo.list', - '10.3', '10.5') + WORepo.add(self, repo_url=WOVar.wo_mysql_repo) WORepo.add_key(self, '0xcbcb082a1bb943db', keyserver='keyserver.ubuntu.com') WORepo.add_key(self, '0xF1656F24C74CD1D8', @@ -996,10 +986,6 @@ def post_pref(self, apt_packages, packages, upgrade=False): WOTemplate.deploy( self, '/etc/mysql/mariadb.conf.d/50-server.cnf', 'my.mustache', data) - WOFileUtils.mvfile( - self, '/etc/mysql/my.cnf', '/etc/mysql/my.cnf.old') - WOFileUtils.create_symlink( - self, ['/etc/mysql/mariadb.cnf', '/etc/mysql/my.cnf']) else: WOTemplate.deploy( self, '/etc/mysql/my.cnf', 'my.mustache', data) diff --git a/wo/cli/plugins/stack_upgrade.py b/wo/cli/plugins/stack_upgrade.py index ea4cd8b..5b5bd7b 100644 --- a/wo/cli/plugins/stack_upgrade.py +++ b/wo/cli/plugins/stack_upgrade.py @@ -294,12 +294,6 @@ class WOStackUpgradeController(CementBaseController): # additional pre_pref if "nginx-custom" in apt_packages: pre_pref(self, WOVar.wo_nginx) - if "mariadb-server" in apt_packages: - mariadbmajorupgrade = bool( - WOFileUtils.grepcheck( - self, '/etc/apt/sources.list.d/wo-repo.list', - 'MariaDB/repo/10.3')) - pre_pref(self, WOVar.wo_mysql) Log.wait(self, "Updating APT cache") # apt-get update WOAptGet.update(self) @@ -312,17 +306,10 @@ class WOStackUpgradeController(CementBaseController): # redis pre_pref if "redis-server" in apt_packages: pre_pref(self, WOVar.wo_redis) - # mariadb upgrade - if ("mariadb-server" in apt_packages and - mariadbmajorupgrade is True): - WOMysql.backupAll(self, fulldump=True) - WOAptGet.remove(self, ["mariadb-server"]) - # upgrade packages + # upgrade packages WOAptGet.install(self, apt_packages) Log.wait(self, "Configuring APT Packages") post_pref(self, apt_packages, [], True) - if "mariadb-server" in apt_packages: - WOShellExec.cmd_exec(self, 'mysql_upgrade') Log.valide(self, "Configuring APT Packages") # Post Actions after package updates From be4805a21ef53724311cfefac0c3c3fa3ff5075e Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Fri, 23 Oct 2020 12:52:47 +0200 Subject: [PATCH 11/25] Update bash_completion with wo stack migrate --- config/bash_completion.d/wo_auto.rc | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/config/bash_completion.d/wo_auto.rc b/config/bash_completion.d/wo_auto.rc index 85ea252..282ee04 100644 --- a/config/bash_completion.d/wo_auto.rc +++ b/config/bash_completion.d/wo_auto.rc @@ -35,7 +35,7 @@ _wo_complete() "stack") COMPREPLY=( $(compgen \ - -W "upgrade install purge reload remove restart start status stop" \ + -W "upgrade install migrate purge reload remove restart start status stop" \ -- $cur) ) ;; @@ -82,6 +82,12 @@ _wo_complete() -W "--web --admin --utils --nginx --php72 --php73 --php74 --mysql --all --netdata --composer --phpmyadmin --adminer --dashboard --mysqltuner --wpcli --force" \ -- $cur) ) ;; + ;; + "migrate") + COMPREPLY=( $(compgen \ + -W "--mariadb --force" \ + -- $cur) ) + ;; "start" | "stop" | "reload" | "restart" | "status") COMPREPLY=( $(compgen \ -W "--nginx --php --php72 --php73 --php74 --mysql --redis --fail2ban --ufw --netdata -proftpd" \ From 0c01ebb9efee5287f4293cd5ff0ec3b964c80b46 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Fri, 23 Oct 2020 13:32:39 +0200 Subject: [PATCH 12/25] Remove previous MariaDB repository --- wo/cli/plugins/stack_migrate.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/wo/cli/plugins/stack_migrate.py b/wo/cli/plugins/stack_migrate.py index be1dabb..3d2aa14 100644 --- a/wo/cli/plugins/stack_migrate.py +++ b/wo/cli/plugins/stack_migrate.py @@ -7,7 +7,8 @@ from wo.core.logging import Log from wo.core.mysql import WOMysql from wo.core.shellexec import WOShellExec from wo.core.variables import WOVar - +from wo.core.fileutils import WOFileUtils +from wo.core.apt_repo import WORepo class WOStackMigrateController(CementBaseController): class Meta: @@ -29,6 +30,17 @@ class WOStackMigrateController(CementBaseController): # Backup all database WOMysql.backupAll(self, fulldump=True) + # Remove previous MariaDB repository + wo_mysql_old_repo = ( + "deb [arch=amd64,ppc64el] " + "http://mariadb.mirrors.ovh.net/MariaDB/repo/" + "10.3/{distro} {codename} main" + .format(distro=WOVar.wo_distro, + codename=WOVar.wo_platform_codename)) + if WOFileUtils.grepcheck( + self, '/etc/apt/sources.list.d/wo-repo.list', + wo_mysql_old_repo): + WORepo.remove(self, repo_url=wo_mysql_old_repo) # Add MariaDB repo Log.info(self, "Adding repository for MariaDB, please wait...") pre_pref(self, WOVar.wo_mysql) From 51f34e4286b21d9193aabacc0119ddb9bc95aff4 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Fri, 23 Oct 2020 13:49:14 +0200 Subject: [PATCH 13/25] Fix double import --- wo/cli/plugins/stack_migrate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wo/cli/plugins/stack_migrate.py b/wo/cli/plugins/stack_migrate.py index 3d2aa14..6403a76 100644 --- a/wo/cli/plugins/stack_migrate.py +++ b/wo/cli/plugins/stack_migrate.py @@ -7,9 +7,9 @@ from wo.core.logging import Log from wo.core.mysql import WOMysql from wo.core.shellexec import WOShellExec from wo.core.variables import WOVar -from wo.core.fileutils import WOFileUtils from wo.core.apt_repo import WORepo + class WOStackMigrateController(CementBaseController): class Meta: label = 'migrate' From 3ccdd8c383dffa4fcb4860a0200ffdb44bc81546 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Fri, 23 Oct 2020 13:51:35 +0200 Subject: [PATCH 14/25] Speed up travis --- .travis.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index d63edd6..b1049ae 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,9 +4,11 @@ jobs: include: - os: linux dist: xenial - - os: linux + - if: branch = master + os: linux dist: bionic - - os: linux + - if: branch = master + os: linux dist: focal notifications: From 1b0d9d7d8d39c8827a4c328fb66bff4079cbd99b Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Fri, 23 Oct 2020 15:42:15 +0200 Subject: [PATCH 15/25] Fix bash_completion --- config/bash_completion.d/wo_auto.rc | 1 - 1 file changed, 1 deletion(-) diff --git a/config/bash_completion.d/wo_auto.rc b/config/bash_completion.d/wo_auto.rc index 282ee04..bde5b19 100644 --- a/config/bash_completion.d/wo_auto.rc +++ b/config/bash_completion.d/wo_auto.rc @@ -82,7 +82,6 @@ _wo_complete() -W "--web --admin --utils --nginx --php72 --php73 --php74 --mysql --all --netdata --composer --phpmyadmin --adminer --dashboard --mysqltuner --wpcli --force" \ -- $cur) ) ;; - ;; "migrate") COMPREPLY=( $(compgen \ -W "--mariadb --force" \ From 51b078e86d9f34fe4376c9b4cf888e9133d90186 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Fri, 23 Oct 2020 16:33:35 +0200 Subject: [PATCH 16/25] Improve MariaDB upgrade --- CHANGELOG.md | 10 ++++++++++ wo/cli/plugins/stack_migrate.py | 13 ++++++++----- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a2e551..96e7a55 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### v3.9.x - [Unreleased] +#### Added + +- MariaDB 10.5 support (installed by default) +- Upgrade to MariaDB 10.5 with `wo stack migrate --mariadb` + +#### Fixed + +- Useless php-cli version removal +- Redis 6.0.6 not installed on Ubuntu 20.04 LTS + ### v3.12.4 - 2020-10-14 #### Changed diff --git a/wo/cli/plugins/stack_migrate.py b/wo/cli/plugins/stack_migrate.py index 6403a76..bd0ee5f 100644 --- a/wo/cli/plugins/stack_migrate.py +++ b/wo/cli/plugins/stack_migrate.py @@ -42,23 +42,26 @@ class WOStackMigrateController(CementBaseController): wo_mysql_old_repo): WORepo.remove(self, repo_url=wo_mysql_old_repo) # Add MariaDB repo - Log.info(self, "Adding repository for MariaDB, please wait...") pre_pref(self, WOVar.wo_mysql) # Install MariaDB - Log.info(self, "Updating apt-cache, hang on...") + Log.wait(self, "Updating apt-cache ") WOAptGet.update(self) - Log.info(self, "Installing MariaDB, hang on...") + Log.valide(self, "Updating apt-cache ") + Log.wait(self, "Upgrading MariaDB ") WOAptGet.remove(self, ["mariadb-server"]) WOAptGet.auto_remove(self) WOAptGet.install(self, WOVar.wo_mysql) - post_pref(self, WOVar.wo_mysql, []) - WOShellExec.cmd_exec(self, 'systemctl daemon-reload') + Log.valide(self, "Upgrading MariaDB ") WOFileUtils.mvfile( self, '/etc/mysql/my.cnf', '/etc/mysql/my.cnf.old') WOFileUtils.create_symlink( self, ['/etc/mysql/mariadb.cnf', '/etc/mysql/my.cnf']) + WOShellExec.cmd_exec(self, 'systemctl daemon-reload') + post_pref(self, WOVar.wo_mysql, []) + + @expose(hide=True) def default(self): From 665453dcbcd7c98894db2790b0c00a7b142927f3 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Fri, 23 Oct 2020 18:45:03 +0200 Subject: [PATCH 17/25] Fix formatting --- wo/cli/plugins/stack_migrate.py | 2 -- wo/cli/plugins/stack_upgrade.py | 1 - 2 files changed, 3 deletions(-) diff --git a/wo/cli/plugins/stack_migrate.py b/wo/cli/plugins/stack_migrate.py index bd0ee5f..be084a0 100644 --- a/wo/cli/plugins/stack_migrate.py +++ b/wo/cli/plugins/stack_migrate.py @@ -61,8 +61,6 @@ class WOStackMigrateController(CementBaseController): WOShellExec.cmd_exec(self, 'systemctl daemon-reload') post_pref(self, WOVar.wo_mysql, []) - - @expose(hide=True) def default(self): pargs = self.app.pargs diff --git a/wo/cli/plugins/stack_upgrade.py b/wo/cli/plugins/stack_upgrade.py index 5b5bd7b..e3f0082 100644 --- a/wo/cli/plugins/stack_upgrade.py +++ b/wo/cli/plugins/stack_upgrade.py @@ -12,7 +12,6 @@ from wo.core.logging import Log from wo.core.shellexec import WOShellExec from wo.core.variables import WOVar from wo.core.services import WOService -from wo.core.mysql import WOMysql class WOStackUpgradeController(CementBaseController): From 3d32e02c654a481e0ef469c161bb62a0330dbedb Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Sun, 25 Oct 2020 14:19:08 +0100 Subject: [PATCH 18/25] Improved query strings caching --- CHANGELOG.md | 4 ++++ wo/cli/templates/map-wp.mustache | 16 +++++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 96e7a55..86f480b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - MariaDB 10.5 support (installed by default) - Upgrade to MariaDB 10.5 with `wo stack migrate --mariadb` +#### Changed + +- Improved Nginx caching rules to cache requests with query strings related to analytics (utm_, fbclid) + #### Fixed - Useless php-cli version removal diff --git a/wo/cli/templates/map-wp.mustache b/wo/cli/templates/map-wp.mustache index c1af32c..63b45fa 100644 --- a/wo/cli/templates/map-wp.mustache +++ b/wo/cli/templates/map-wp.mustache @@ -60,11 +60,25 @@ map $http_user_agent $mobile_prefix { } # do not cache requests with query strings -map $is_args $query_no_cache { +map $is_args $is_args_no_cache { default 1; "" 0; } +# cache requests with query string related to analytics +map $args $args_to_cache { + default 0; + "~*utm_" 1; + "~*fbclid" 1; +} + +# do not cache requests with query strings excepted analytics related queries +map $is_args_no_cache$args_to_cache $query_no_cache { + defaut 1; + 00 0; + 11 0; +} + # if all previous check are passed, $skip_cache = 0 map $http_request_no_cache$cookie_no_cache$uri_no_cache$query_no_cache $skip_cache { default 1; From b5db63ff82117d04464fd9b51884b4fbdf9fab36 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Sun, 25 Oct 2020 16:03:46 +0100 Subject: [PATCH 19/25] Install WordOps inside a virtualenv --- install | 43 ++++++++++++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/install b/install index 8cbe6b8..b825a3f 100755 --- a/install +++ b/install @@ -9,7 +9,7 @@ # ------------------------------------------------------------------------- # wget -qO wo wops.cc && sudo -E bash wo # ------------------------------------------------------------------------- -# Version 3.13.0 - 2020-10-22 +# Version 3.13.0 - 2020-10-25 # ------------------------------------------------------------------------- # CONTENTS @@ -167,7 +167,7 @@ wo_check_distro() { wo_lib_echo_fail "Feel free to open a pull-request if you want to add support for another Linux distributions" exit 100 else - check_wo_linux_distro=$(lsb_release -sc | grep -E "xenial|bionic|disco|jessie|stretch|buster|focal") + check_wo_linux_distro=$(lsb_release -sc | grep -E "xenial|bionic|jessie|stretch|buster|focal") if [ -z "$check_wo_linux_distro" ]; then wo_lib_echo_fail "WordOps (wo) only supports Ubuntu 16.04/18.04/20.04 LTS, Debian 9.x/10.x and Raspbian 9.x/10x.\n You can bypass this warning by adding the flag --force to the install command" @@ -208,13 +208,13 @@ wo_install_dep() { if [ "$wo_linux_distro" == "Ubuntu" ]; then # install dependencies apt-get -option=Dpkg::options::=--force-confmiss --option=Dpkg::options::=--force-confold --assume-yes install \ - build-essential curl gzip python3-pip python3-apt gcc python3-dev sqlite3 git tar software-properties-common pigz \ + build-essential curl gzip python3-pip python3-apt python3-venv gcc python3-dev sqlite3 git tar software-properties-common pigz \ gnupg2 cron ccze rsync apt-transport-https tree haveged ufw unattended-upgrades tzdata ntp zstd idn >/dev/null 2>&1 curl -sL https://download.opensuse.org/repositories/home:/virtubox:/WordOps/xUbuntu_18.04/Release.key | apt-key add - else # install dependencies apt-get -option=Dpkg::options::=--force-confmiss --option=Dpkg::options::=--force-confold --assume-yes install \ - build-essential curl gzip dirmngr sudo python3-pip python3-apt gcc python3-dev ca-certificates sqlite3 git tar \ + build-essential curl gzip dirmngr sudo python3-pip python3-apt python3-venv gcc python3-dev ca-certificates sqlite3 git tar \ software-properties-common pigz apt-transport-https gnupg2 cron ccze rsync tree haveged ufw unattended-upgrades tzdata ntp zstd idn >/dev/null 2>&1 # add php repository gpg key [ -d /etc/apt/trusted.gpg.d ] && { wget -qO /etc/apt/trusted.gpg.d/php.gpg https://packages.sury.org/php/apt.gpg; } @@ -228,8 +228,6 @@ wo_install_dep() { fi # upgrade pip python3 -m pip install --upgrade pip - python3 -m pip install --upgrade setuptools wheel - } wo_timesync() { @@ -432,14 +430,28 @@ wo_install() { if [ -d /usr/local/lib/python3."$python_ver"/dist-packages ]; then cd /usr/local/lib/python3."$python_ver"/dist-packages || exit 1 fi + python3 -m pip uninstall -yq wo wordops ee + if [ -d /opt/wo ]; then + cd /opt/wo || exit 1 + source bin/activate + else + mkdir -p /opt/wo + cd /opt || exit 1 + python3 -m venv --system-site-packages wo + source wo/bin/activate + fi + python3 -m pip install -U pip setuptools wheel if [ "$wo_branch" = "master" ]; then python3 -m pip install -I wordops else python3 -m pip install -I "git+git://github.com/WordOps/WordOps.git@$wo_branch#egg=wordops" fi - cp -rf /usr/local/lib/python3."$python_ver"/dist-packages/usr/* /usr/ - cp -rn /usr/local/lib/python3."$python_ver"/dist-packages/etc/* /etc/ - cp -f /usr/local/lib/python3."$python_ver"/dist-packages/etc/bash_completion.d/wo_auto.rc /etc/bash_completion.d/wo_auto.rc + cp -rf /opt/wo/lib/python3."$python_ver"/site-packages/usr/* /usr/ + cp -rn /opt/wo/lib/python3."$python_ver"/site-packages/etc/* /etc/ + cp -f /opt/wo/lib/python3."$python_ver"/site-packages/etc/bash_completion.d/wo_auto.rc /etc/bash_completion.d/wo_auto.rc + rm -f /usr/local/bin/wo + ln -s /opt/wo/bin/wo /usr/local/bin/ + deactivate } # Clone Github repository if it doesn't exist @@ -451,14 +463,19 @@ wo_travis_install() { fi if [ -f ./setup.py ]; then python3 setup.py sdist bdist_wheel + mkdir -p /opt/wo + python3 -m venv --system-site-packages /opt/wo + source /opt/wo/bin/activate python3 -m pip install --upgrade dist/*.whl else python3 -m pip install -U "git+git://github.com/WordOps/WordOps.git@$wo_branch#egg=wordops" fi - cp -rf /usr/local/lib/python3."$python_ver"/dist-packages/usr/* /usr/ - cp -rn /usr/local/lib/python3."$python_ver"/dist-packages/etc/* /etc/ - cp -f /usr/local/lib/python3."$python_ver"/dist-packages/etc/bash_completion.d/wo_auto.rc /etc/bash_completion.d/wo_auto.rc - + cp -rf /opt/wo/lib/python3."$python_ver"/site-packages/usr/* /usr/ + cp -rn /opt/wo/lib/python3."$python_ver"/site-packages/etc/* /etc/ + cp -f /opt/wo/lib/python3."$python_ver"/site-packages/etc/bash_completion.d/wo_auto.rc /etc/bash_completion.d/wo_auto.rc + rm -f /usr/local/bin/wo + ln -s /opt/wo/bin/wo /usr/local/bin/ + deactivate } wo_upgrade_nginx() { From 4fe0bfadb7f7f300c1c58f31fd30d5a350f9bfb5 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Sun, 25 Oct 2020 18:30:03 +0100 Subject: [PATCH 20/25] Fix travis install --- install | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/install b/install index b825a3f..3655f4b 100755 --- a/install +++ b/install @@ -462,11 +462,17 @@ wo_travis_install() { rm -rf dist fi if [ -f ./setup.py ]; then - python3 setup.py sdist bdist_wheel - mkdir -p /opt/wo - python3 -m venv --system-site-packages /opt/wo - source /opt/wo/bin/activate - python3 -m pip install --upgrade dist/*.whl + if [ -d /opt/wo ]; then + cd /opt/wo || exit 1 + source bin/activate + else + mkdir -p /opt/wo + cd /opt || exit 1 + python3 -m venv --system-site-packages wo + source wo/bin/activate + fi + python3 -m pip install -U pip setuptools wheel + python3 -m pip install -U "git+git://github.com/WordOps/WordOps.git@$wo_branch#egg=wordops" else python3 -m pip install -U "git+git://github.com/WordOps/WordOps.git@$wo_branch#egg=wordops" fi From 4ac512038e945e77e331a6ebaf645debfe15da84 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Sun, 25 Oct 2020 18:54:33 +0100 Subject: [PATCH 21/25] Ready for release --- CHANGELOG.md | 3 +++ install | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 86f480b..c5f7b6a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### v3.9.x - [Unreleased] +#### v3.13.0 - 2020-10-25 + #### Added - MariaDB 10.5 support (installed by default) @@ -16,6 +18,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), #### Changed - Improved Nginx caching rules to cache requests with query strings related to analytics (utm_, fbclid) +- WordOps is installed inside a Python virtual environment in /opt/wo to isolate it from the system's Python libraries #### Fixed diff --git a/install b/install index 3655f4b..3e6c8a2 100755 --- a/install +++ b/install @@ -449,7 +449,7 @@ wo_install() { cp -rf /opt/wo/lib/python3."$python_ver"/site-packages/usr/* /usr/ cp -rn /opt/wo/lib/python3."$python_ver"/site-packages/etc/* /etc/ cp -f /opt/wo/lib/python3."$python_ver"/site-packages/etc/bash_completion.d/wo_auto.rc /etc/bash_completion.d/wo_auto.rc - rm -f /usr/local/bin/wo + rm -rf /usr/local/bin/wo /usr/local/lib/python3."$python_ver"/dist-packages/{etc,usr} ln -s /opt/wo/bin/wo /usr/local/bin/ deactivate } @@ -638,7 +638,7 @@ wo_uninstall() { if { python3 -m pip list | grep -q "wordops" >/dev/null 2>&1; }; then python3 -m pip uninstall wordops -y fi - rm -rf /usr/local/lib/python3."$python_ver"/dist-packages/{pystache-*,cement-2.*,wo-*,wordops-*} /usr/local/bin/wo /etc/bash_completion.d/wo_auto.rc /var/lib/wo /etc/wo /usr/lib/wo/templates + rm -rf /usr/local/lib/python3."$python_ver"/dist-packages/{pystache-*,cement-2.*,wo-*,wordops-*} /usr/local/bin/wo /etc/bash_completion.d/wo_auto.rc /var/lib/wo /etc/wo /usr/lib/wo/templates /opt/wo } wo_clean_repo() { From 869286f19fdc94bab69d1c952ada7256db2256a6 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Sun, 25 Oct 2020 19:05:29 +0100 Subject: [PATCH 22/25] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7a37251..7ff41c8 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ - **Easy to install** : One step automated installer with migration from EasyEngine v3 support - **Fast deployment** : Fast and automated WordPress, Nginx, PHP, MySQL & Redis installation - **Custom Nginx build** : Nginx 1.18.0 - TLS v1.3 Cloudflare HTTP/2 HPACK & Brotli support -- **Up-to-date** : PHP 7.2, 7.3 & 7.4, MariaDB 10.3 & Redis 6.0 +- **Up-to-date** : PHP 7.2, 7.3 & 7.4, MariaDB 10.5 & Redis 6.0 - **Secured** : Hardened WordPress security with strict Nginx location directives - **Powerful** : Optimized Nginx configurations with multiple cache backends support - **SSL** : Domain, Subdomain & Wildcard Let's Encrypt SSL certificates with DNS API support From 32b6ff5fcf27eade19cd976c1985d2683d71486b Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Sun, 25 Oct 2020 19:57:23 +0100 Subject: [PATCH 23/25] Fix travis --- install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install b/install index 3e6c8a2..249b9fb 100755 --- a/install +++ b/install @@ -472,7 +472,7 @@ wo_travis_install() { source wo/bin/activate fi python3 -m pip install -U pip setuptools wheel - python3 -m pip install -U "git+git://github.com/WordOps/WordOps.git@$wo_branch#egg=wordops" + python3 -m pip install -U . else python3 -m pip install -U "git+git://github.com/WordOps/WordOps.git@$wo_branch#egg=wordops" fi From 5a3344a830b7bf65f4fce45a13127a3abcc3be56 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Sun, 25 Oct 2020 20:15:38 +0100 Subject: [PATCH 24/25] Fix travis --- install | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/install b/install index 249b9fb..9002070 100755 --- a/install +++ b/install @@ -463,12 +463,10 @@ wo_travis_install() { fi if [ -f ./setup.py ]; then if [ -d /opt/wo ]; then - cd /opt/wo || exit 1 - source bin/activate + source /opt/wo/bin/activate else mkdir -p /opt/wo - cd /opt || exit 1 - python3 -m venv --system-site-packages wo + python3 -m venv --system-site-packages /opt/wo source wo/bin/activate fi python3 -m pip install -U pip setuptools wheel From 821eed29ef58d3e7cb095db48fd8487e2e6ebe0f Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Sun, 25 Oct 2020 20:32:05 +0100 Subject: [PATCH 25/25] Another fix for travis --- install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install b/install index 9002070..c5b8714 100755 --- a/install +++ b/install @@ -467,7 +467,7 @@ wo_travis_install() { else mkdir -p /opt/wo python3 -m venv --system-site-packages /opt/wo - source wo/bin/activate + source /opt/wo/bin/activate fi python3 -m pip install -U pip setuptools wheel python3 -m pip install -U .