diff --git a/.travis.yml b/.travis.yml index d63edd6..b1049ae 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,9 +4,11 @@ jobs: include: - os: linux dist: xenial - - os: linux + - if: branch = master + os: linux dist: bionic - - os: linux + - if: branch = master + os: linux dist: focal notifications: diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a2e551..c5f7b6a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,23 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### v3.9.x - [Unreleased] +#### v3.13.0 - 2020-10-25 + +#### Added + +- MariaDB 10.5 support (installed by default) +- Upgrade to MariaDB 10.5 with `wo stack migrate --mariadb` + +#### Changed + +- Improved Nginx caching rules to cache requests with query strings related to analytics (utm_, fbclid) +- WordOps is installed inside a Python virtual environment in /opt/wo to isolate it from the system's Python libraries + +#### Fixed + +- Useless php-cli version removal +- Redis 6.0.6 not installed on Ubuntu 20.04 LTS + ### v3.12.4 - 2020-10-14 #### Changed diff --git a/README.md b/README.md index 7a37251..7ff41c8 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ - **Easy to install** : One step automated installer with migration from EasyEngine v3 support - **Fast deployment** : Fast and automated WordPress, Nginx, PHP, MySQL & Redis installation - **Custom Nginx build** : Nginx 1.18.0 - TLS v1.3 Cloudflare HTTP/2 HPACK & Brotli support -- **Up-to-date** : PHP 7.2, 7.3 & 7.4, MariaDB 10.3 & Redis 6.0 +- **Up-to-date** : PHP 7.2, 7.3 & 7.4, MariaDB 10.5 & Redis 6.0 - **Secured** : Hardened WordPress security with strict Nginx location directives - **Powerful** : Optimized Nginx configurations with multiple cache backends support - **SSL** : Domain, Subdomain & Wildcard Let's Encrypt SSL certificates with DNS API support diff --git a/config/bash_completion.d/wo_auto.rc b/config/bash_completion.d/wo_auto.rc index e7642dd..bde5b19 100644 --- a/config/bash_completion.d/wo_auto.rc +++ b/config/bash_completion.d/wo_auto.rc @@ -35,7 +35,7 @@ _wo_complete() "stack") COMPREPLY=( $(compgen \ - -W "upgrade install purge reload remove restart start status stop" \ + -W "upgrade install migrate purge reload remove restart start status stop" \ -- $cur) ) ;; @@ -74,17 +74,22 @@ _wo_complete() # HANDLE EVERYTHING AFTER THE THIRD LEVEL NAMESPACE "install" | "purge" | "remove" ) COMPREPLY=( $(compgen \ - -W "--web --admin --security --nginx --php --php73 --php74 --mysql --wpcli --phpmyadmin --adminer --utils --redis --phpredisadmin --composer --netdata --fail2ban --ufw --dashboard --proftpd --clamav --sendmail --ngxblocker --mysqlclient --mysqltuner --extplorer --nanorc --cheat --all --force" \ + -W "--web --admin --security --nginx --php72 --php73 --php74 --mysql --wpcli --phpmyadmin --adminer --utils --redis --phpredisadmin --composer --netdata --fail2ban --ufw --dashboard --proftpd --clamav --sendmail --ngxblocker --mysqlclient --mysqltuner --extplorer --nanorc --cheat --all --force" \ -- $cur) ) ;; "upgrade" ) COMPREPLY=( $(compgen \ - -W "--web --admin --utils --nginx --php --php73 --php74 --mysql --all --netdata --composer --phpmyadmin --adminer --dashboard --mysqtuner --wpcli --force" \ + -W "--web --admin --utils --nginx --php72 --php73 --php74 --mysql --all --netdata --composer --phpmyadmin --adminer --dashboard --mysqltuner --wpcli --force" \ + -- $cur) ) + ;; + "migrate") + COMPREPLY=( $(compgen \ + -W "--mariadb --force" \ -- $cur) ) ;; "start" | "stop" | "reload" | "restart" | "status") COMPREPLY=( $(compgen \ - -W "--nginx --php --php73 --php74 --mysql --redis --fail2ban --ufw --netdata -proftpd" \ + -W "--nginx --php --php72 --php73 --php74 --mysql --redis --fail2ban --ufw --netdata -proftpd" \ -- $cur) ) ;; "list") diff --git a/install b/install index a30a82c..c5b8714 100755 --- a/install +++ b/install @@ -9,7 +9,7 @@ # ------------------------------------------------------------------------- # wget -qO wo wops.cc && sudo -E bash wo # ------------------------------------------------------------------------- -# Version 3.12.4 - 2020-10-14 +# Version 3.13.0 - 2020-10-25 # ------------------------------------------------------------------------- # CONTENTS @@ -167,7 +167,7 @@ wo_check_distro() { wo_lib_echo_fail "Feel free to open a pull-request if you want to add support for another Linux distributions" exit 100 else - check_wo_linux_distro=$(lsb_release -sc | grep -E "xenial|bionic|disco|jessie|stretch|buster|focal") + check_wo_linux_distro=$(lsb_release -sc | grep -E "xenial|bionic|jessie|stretch|buster|focal") if [ -z "$check_wo_linux_distro" ]; then wo_lib_echo_fail "WordOps (wo) only supports Ubuntu 16.04/18.04/20.04 LTS, Debian 9.x/10.x and Raspbian 9.x/10x.\n You can bypass this warning by adding the flag --force to the install command" @@ -208,13 +208,13 @@ wo_install_dep() { if [ "$wo_linux_distro" == "Ubuntu" ]; then # install dependencies apt-get -option=Dpkg::options::=--force-confmiss --option=Dpkg::options::=--force-confold --assume-yes install \ - build-essential curl gzip python3-pip python3-apt gcc python3-dev sqlite3 git tar software-properties-common pigz \ + build-essential curl gzip python3-pip python3-apt python3-venv gcc python3-dev sqlite3 git tar software-properties-common pigz \ gnupg2 cron ccze rsync apt-transport-https tree haveged ufw unattended-upgrades tzdata ntp zstd idn >/dev/null 2>&1 curl -sL https://download.opensuse.org/repositories/home:/virtubox:/WordOps/xUbuntu_18.04/Release.key | apt-key add - else # install dependencies apt-get -option=Dpkg::options::=--force-confmiss --option=Dpkg::options::=--force-confold --assume-yes install \ - build-essential curl gzip dirmngr sudo python3-pip python3-apt gcc python3-dev ca-certificates sqlite3 git tar \ + build-essential curl gzip dirmngr sudo python3-pip python3-apt python3-venv gcc python3-dev ca-certificates sqlite3 git tar \ software-properties-common pigz apt-transport-https gnupg2 cron ccze rsync tree haveged ufw unattended-upgrades tzdata ntp zstd idn >/dev/null 2>&1 # add php repository gpg key [ -d /etc/apt/trusted.gpg.d ] && { wget -qO /etc/apt/trusted.gpg.d/php.gpg https://packages.sury.org/php/apt.gpg; } @@ -228,8 +228,6 @@ wo_install_dep() { fi # upgrade pip python3 -m pip install --upgrade pip - python3 -m pip install --upgrade setuptools wheel - } wo_timesync() { @@ -432,14 +430,28 @@ wo_install() { if [ -d /usr/local/lib/python3."$python_ver"/dist-packages ]; then cd /usr/local/lib/python3."$python_ver"/dist-packages || exit 1 fi + python3 -m pip uninstall -yq wo wordops ee + if [ -d /opt/wo ]; then + cd /opt/wo || exit 1 + source bin/activate + else + mkdir -p /opt/wo + cd /opt || exit 1 + python3 -m venv --system-site-packages wo + source wo/bin/activate + fi + python3 -m pip install -U pip setuptools wheel if [ "$wo_branch" = "master" ]; then python3 -m pip install -I wordops else python3 -m pip install -I "git+git://github.com/WordOps/WordOps.git@$wo_branch#egg=wordops" fi - cp -rf /usr/local/lib/python3."$python_ver"/dist-packages/usr/* /usr/ - cp -rn /usr/local/lib/python3."$python_ver"/dist-packages/etc/* /etc/ - cp -f /usr/local/lib/python3."$python_ver"/dist-packages/etc/bash_completion.d/wo_auto.rc /etc/bash_completion.d/wo_auto.rc + cp -rf /opt/wo/lib/python3."$python_ver"/site-packages/usr/* /usr/ + cp -rn /opt/wo/lib/python3."$python_ver"/site-packages/etc/* /etc/ + cp -f /opt/wo/lib/python3."$python_ver"/site-packages/etc/bash_completion.d/wo_auto.rc /etc/bash_completion.d/wo_auto.rc + rm -rf /usr/local/bin/wo /usr/local/lib/python3."$python_ver"/dist-packages/{etc,usr} + ln -s /opt/wo/bin/wo /usr/local/bin/ + deactivate } # Clone Github repository if it doesn't exist @@ -450,15 +462,24 @@ wo_travis_install() { rm -rf dist fi if [ -f ./setup.py ]; then - python3 setup.py sdist bdist_wheel - python3 -m pip install --upgrade dist/*.whl + if [ -d /opt/wo ]; then + source /opt/wo/bin/activate + else + mkdir -p /opt/wo + python3 -m venv --system-site-packages /opt/wo + source /opt/wo/bin/activate + fi + python3 -m pip install -U pip setuptools wheel + python3 -m pip install -U . else python3 -m pip install -U "git+git://github.com/WordOps/WordOps.git@$wo_branch#egg=wordops" fi - cp -rf /usr/local/lib/python3."$python_ver"/dist-packages/usr/* /usr/ - cp -rn /usr/local/lib/python3."$python_ver"/dist-packages/etc/* /etc/ - cp -f /usr/local/lib/python3."$python_ver"/dist-packages/etc/bash_completion.d/wo_auto.rc /etc/bash_completion.d/wo_auto.rc - + cp -rf /opt/wo/lib/python3."$python_ver"/site-packages/usr/* /usr/ + cp -rn /opt/wo/lib/python3."$python_ver"/site-packages/etc/* /etc/ + cp -f /opt/wo/lib/python3."$python_ver"/site-packages/etc/bash_completion.d/wo_auto.rc /etc/bash_completion.d/wo_auto.rc + rm -f /usr/local/bin/wo + ln -s /opt/wo/bin/wo /usr/local/bin/ + deactivate } wo_upgrade_nginx() { @@ -615,7 +636,7 @@ wo_uninstall() { if { python3 -m pip list | grep -q "wordops" >/dev/null 2>&1; }; then python3 -m pip uninstall wordops -y fi - rm -rf /usr/local/lib/python3."$python_ver"/dist-packages/{pystache-*,cement-2.*,wo-*,wordops-*} /usr/local/bin/wo /etc/bash_completion.d/wo_auto.rc /var/lib/wo /etc/wo /usr/lib/wo/templates + rm -rf /usr/local/lib/python3."$python_ver"/dist-packages/{pystache-*,cement-2.*,wo-*,wordops-*} /usr/local/bin/wo /etc/bash_completion.d/wo_auto.rc /var/lib/wo /etc/wo /usr/lib/wo/templates /opt/wo } wo_clean_repo() { @@ -704,13 +725,18 @@ wo_init() { wo_php_fix() { local php_versions="5.6 7.0 7.1 7.2 7.3 7.4" - apt-get autoremove --assume-yes --purge php8.0-* php-igbinary php-memcached php-msgpack php-redis php-imagick php-xdebug + apt-get autoremove --assume-yes --purge php8.0-* php-igbinary php-memcached php-msgpack php-redis php-imagick php-xdebug php-memcache for php_version in $php_versions; do - if [ -f "/usr/sbin/php-fpm$php_version" ]; then - local php_extensions="igbinary memcached msgpack redis imagick xdebug" - for php_ext in $php_extensions; do - apt-get -option=Dpkg::options::=--force-confmiss --option=Dpkg::options::=--force-confold --assume-yes install "php$php_version-$php_ext" - done + if [ -f "/usr/bin/php$php_version" ]; then + if [ -f "/usr/sbin/php-fpm$php_version" ]; then + local php_extensions="igbinary memcached msgpack redis imagick xdebug" + for php_ext in $php_extensions; do + apt-get -option=Dpkg::options::=--force-confmiss --option=Dpkg::options::=--force-confold --assume-yes install "php$php_version-$php_ext" + done + else + apt-get autoremove --assume-yes --purge "php$php_version-*" + fi + fi done } diff --git a/setup.py b/setup.py index d60768b..d437072 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ if os.geteuid() == 0: os.makedirs('/var/lib/wo/tmp/') setup(name='wordops', - version='3.12.4', + version='3.13.0', description='An essential toolset that eases server administration', long_description=LONG, long_description_content_type='text/markdown', diff --git a/tests/travis.sh b/tests/travis.sh index 073d75a..d0e825e 100644 --- a/tests/travis.sh +++ b/tests/travis.sh @@ -80,7 +80,7 @@ echo echo -e "${CGREEN}#############################################${CEND}" echo -e ' wo site update --php74 ' echo -e "${CGREEN}#############################################${CEND}" -other_site_types='mysql php72 php74 wp wpfc wpsc wpredis wpce wprocket wpsubdomain wpsubdir ngxblocker' +other_site_types='mysql php72 php73 wp wpfc wpsc wpredis wpce wprocket wpsubdomain wpsubdir' for site in $other_site_types; do echo -ne " Updating site to $site php74 [..]\r" if { @@ -103,7 +103,7 @@ echo echo -e "${CGREEN}#############################################${CEND}" echo -e ' wo site update --php73 ' echo -e "${CGREEN}#############################################${CEND}" -other_site_types='html mysql wp php72 php73 wpfc wpsc wpredis wpce wprocket wpsubdomain wpsubdir ngxblocker' +other_site_types='html mysql wp php72 php73 wpfc wpsc wpredis wpce wprocket wpsubdomain wpsubdir' for site in $other_site_types; do echo -ne " Updating site to $site php73 [..]\r" if { @@ -126,7 +126,7 @@ echo echo -e "${CGREEN}#############################################${CEND}" echo -e ' wo site update --php72 ' echo -e "${CGREEN}#############################################${CEND}" -other_site_types='mysql php72 php73 php74 wp wpfc wpsc wpredis wpce wprocket wpsubdomain wpsubdir ngxblocker' +other_site_types='mysql php72 php73 wp wpfc wpsc wpredis wpce wprocket wpsubdomain wpsubdir' for site in $other_site_types; do echo -ne " Updating site to $site php72 [..]\r" if { @@ -226,6 +226,24 @@ if [ -z "$1" ]; then fi done fi + +echo -e "${CGREEN}#############################################${CEND}" +echo -e ' wo stack migrate --mariadb ' +echo -e "${CGREEN}#############################################${CEND}" + + echo -ne " Upgrading mariadb [..]\r" + if { + wo stack migrate --mariadb --force + } >>/var/log/wo/test.log; then + echo -ne " Upgrading mariadb [${CGREEN}OK${CEND}]\\r" + echo -ne '\n' + else + echo -e " Upgrading mariadb [${CRED}FAIL${CEND}]" + echo -ne '\n' + exit_script + + fi + echo -e "${CGREEN}#############################################${CEND}" echo -e ' wo clean ' echo -e "${CGREEN}#############################################${CEND}" diff --git a/wo/cli/controllers/base.py b/wo/cli/controllers/base.py index 9558442..95be6a6 100644 --- a/wo/cli/controllers/base.py +++ b/wo/cli/controllers/base.py @@ -8,7 +8,7 @@ VERSION = WOVar.wo_version BANNER = """ WordOps v%s -Copyright (c) 2019 WordOps. +Copyright (c) 2020 WordOps. """ % VERSION diff --git a/wo/cli/plugins/stack_migrate.py b/wo/cli/plugins/stack_migrate.py index 205714e..be084a0 100644 --- a/wo/cli/plugins/stack_migrate.py +++ b/wo/cli/plugins/stack_migrate.py @@ -1,14 +1,13 @@ -import configparser -import os - from cement.core.controller import CementBaseController, expose -from wo.core.apt_repo import WORepo +from wo.cli.plugins.stack_pref import post_pref, pre_pref from wo.core.aptget import WOAptGet +from wo.core.fileutils import WOFileUtils from wo.core.logging import Log from wo.core.mysql import WOMysql from wo.core.shellexec import WOShellExec from wo.core.variables import WOVar +from wo.core.apt_repo import WORepo class WOStackMigrateController(CementBaseController): @@ -19,104 +18,69 @@ class WOStackMigrateController(CementBaseController): description = ('Migrate stack safely') arguments = [ (['--mariadb'], - dict(help="Migrate database to MariaDB", + dict(help="Migrate/Upgrade database to MariaDB", + action='store_true')), + (['--force'], + dict(help="Force Packages upgrade without any prompt", action='store_true')), ] @expose(hide=True) def migrate_mariadb(self): # Backup all database - WOMysql.backupAll(self) - - if not WOVar.wo_distro == 'raspbian': - if (not WOVar.wo_platform_codename == 'jessie'): - wo_mysql = ["mariadb-server", "percona-toolkit", - "python3-mysqldb", "mariadb-backup"] - else: - wo_mysql = ["mariadb-server", "percona-toolkit", - "python3-mysql.connector"] - else: - wo_mysql = ["mariadb-server", "percona-toolkit", - "python3-mysqldb"] + WOMysql.backupAll(self, fulldump=True) + # Remove previous MariaDB repository + wo_mysql_old_repo = ( + "deb [arch=amd64,ppc64el] " + "http://mariadb.mirrors.ovh.net/MariaDB/repo/" + "10.3/{distro} {codename} main" + .format(distro=WOVar.wo_distro, + codename=WOVar.wo_platform_codename)) + if WOFileUtils.grepcheck( + self, '/etc/apt/sources.list.d/wo-repo.list', + wo_mysql_old_repo): + WORepo.remove(self, repo_url=wo_mysql_old_repo) # Add MariaDB repo - Log.info(self, "Adding repository for MariaDB, please wait...") - - mysql_pref = ("Package: *\nPin: origin sfo1.mirrors.digitalocean.com" - "\nPin-Priority: 1000\n") - with open('/etc/apt/preferences.d/' - 'MariaDB.pref', 'w') as mysql_pref_file: - mysql_pref_file.write(mysql_pref) - - WORepo.add(self, repo_url=WOVar.wo_mysql_repo) - Log.debug(self, 'Adding key for {0}' - .format(WOVar.wo_mysql_repo)) - WORepo.add_key(self, '0xcbcb082a1bb943db', - keyserver="keyserver.ubuntu.com") - - config = configparser.ConfigParser() - if os.path.exists('/etc/mysql/conf.d/my.cnf'): - config.read('/etc/mysql/conf.d/my.cnf') - else: - config.read(os.path.expanduser("~")+'/.my.cnf') - - try: - chars = config['client']['password'] - except Exception as e: - Log.error(self, "Error: process exited with error %s" - % e) - - Log.debug(self, "Pre-seeding MariaDB") - Log.debug(self, "echo \"mariadb-server-10.3 " - "mysql-server/root_password " - "password \" | " - "debconf-set-selections") - WOShellExec.cmd_exec(self, "echo \"mariadb-server-10.3 " - "mysql-server/root_password " - "password {chars}\" | " - "debconf-set-selections" - .format(chars=chars), - log=False) - Log.debug(self, "echo \"mariadb-server-10.3 " - "mysql-server/root_password_again " - "password \" | " - "debconf-set-selections") - WOShellExec.cmd_exec(self, "echo \"mariadb-server-10.3 " - "mysql-server/root_password_again " - "password {chars}\" | " - "debconf-set-selections" - .format(chars=chars), - log=False) + pre_pref(self, WOVar.wo_mysql) # Install MariaDB - apt_packages = wo_mysql - Log.info(self, "Updating apt-cache, hang on...") + Log.wait(self, "Updating apt-cache ") WOAptGet.update(self) - Log.info(self, "Installing MariaDB, hang on...") - WOAptGet.remove(self, ["mysql-common", "libmysqlclient18"]) + Log.valide(self, "Updating apt-cache ") + Log.wait(self, "Upgrading MariaDB ") + WOAptGet.remove(self, ["mariadb-server"]) WOAptGet.auto_remove(self) - WOAptGet.install(self, apt_packages) + WOAptGet.install(self, WOVar.wo_mysql) + Log.valide(self, "Upgrading MariaDB ") + WOFileUtils.mvfile( + self, '/etc/mysql/my.cnf', '/etc/mysql/my.cnf.old') + WOFileUtils.create_symlink( + self, ['/etc/mysql/mariadb.cnf', '/etc/mysql/my.cnf']) + WOShellExec.cmd_exec(self, 'systemctl daemon-reload') + post_pref(self, WOVar.wo_mysql, []) @expose(hide=True) def default(self): - if ((not self.app.pargs.mariadb)): + pargs = self.app.pargs + if ((not pargs.mariadb)): self.app.args.print_help() - if self.app.pargs.mariadb: + if pargs.mariadb: if WOVar.wo_mysql_host != "localhost": Log.error( self, "Remote MySQL server in use, skipping local install") - if (WOShellExec.cmd_exec(self, "mysqladmin ping") and - (not WOAptGet.is_installed(self, 'mariadb-server'))): + if (WOShellExec.cmd_exec(self, "mysqladmin ping")): Log.info(self, "If your database size is big, " "migration may take some time.") Log.info(self, "During migration non nginx-cached parts of " "your site may remain down") - start_migrate = input("Type \"mariadb\" to continue:") - if start_migrate != "mariadb": - Log.error(self, "Not starting migration") + if not pargs.force: + start_upgrade = input("Do you want to continue:[y/N]") + if start_upgrade != "Y" and start_upgrade != "y": + Log.error(self, "Not starting package update") self.migrate_mariadb() else: Log.error(self, "Your current MySQL is not alive or " diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index 2c20e5c..ca619a9 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -42,42 +42,10 @@ def pre_pref(self, apt_packages): keyserver='keyserver.ubuntu.com') WORepo.add_key(self, '0xF1656F24C74CD1D8', keyserver='keyserver.ubuntu.com') - if "mariadb-server" in apt_packages: + if ("mariadb-server" in apt_packages and + not os.path.exists('/etc/mysql/conf.d/my.cnf')): # generate random 24 characters root password chars = ''.join(random.sample(string.ascii_letters, 24)) - - Log.debug(self, "Pre-seeding MySQL") - Log.debug(self, "echo \"mariadb-server-{0} " - "mysql-server/root_password " - "password \" | " - "debconf-set-selections" - .format(WOVar.mariadb_ver)) - try: - WOShellExec.cmd_exec(self, "echo \"mariadb-server-{0} " - "mysql-server/root_password " - "password {chars}\" | " - "debconf-set-selections" - .format(WOVar.mariadb_ver, chars=chars), - log=False) - except CommandExecutionError as e: - Log.debug(self, "{0}".format(e)) - Log.error(self, "Failed to initialize MySQL package") - - Log.debug(self, "echo \"mariadb-server-{0} " - "mysql-server/root_password_again " - "password \" | " - "debconf-set-selections" - .format(WOVar.mariadb_ver)) - try: - WOShellExec.cmd_exec(self, "echo \"mariadb-server-{0} " - "mysql-server/root_password_again " - "password {chars}\" | " - "debconf-set-selections" - .format(WOVar.mariadb_ver, chars=chars), - log=False) - except CommandExecutionError as e: - Log.debug(self, "{0}".format(e)) - Log.error(self, "Failed to initialize MySQL package") # generate my.cnf root credentials mysql_config = """ [client] @@ -87,13 +55,13 @@ def pre_pref(self, apt_packages): config = configparser.ConfigParser() config.read_string(mysql_config) Log.debug(self, 'Writting configuration into MySQL file') - conf_path = "/etc/mysql/conf.d/my.cnf" + conf_path = "/etc/mysql/conf.d/my.cnf.tmp" os.makedirs(os.path.dirname(conf_path), exist_ok=True) with open(conf_path, encoding='utf-8', mode='w') as configfile: config.write(configfile) Log.debug(self, 'Setting my.cnf permission') - WOFileUtils.chmod(self, "/etc/mysql/conf.d/my.cnf", 0o600) + WOFileUtils.chmod(self, "/etc/mysql/conf.d/my.cnf.tmp", 0o600) # add nginx repository if set(WOVar.wo_nginx).issubset(set(apt_packages)): @@ -966,24 +934,29 @@ def post_pref(self, apt_packages, packages, upgrade=False): config_file.close() else: # make sure root account have all privileges - if "PASSWORD" not in WOShellExec.cmd_exec_stdout( - self, 'mysql -e "use mysql; show grants;"'): + if os.path.exists('/etc/mysql/conf.d/my.cnf.tmp'): try: - if not os.path.exists('/etc/mysql/conf.d/my.cnf'): - Log.error(self, 'my.cnf not found') config = configparser.ConfigParser() - config.read('/etc/mysql/conf.d/my.cnf') + config.read('/etc/mysql/conf.d/my.cnf.tmp') chars = config['client']['password'] WOShellExec.cmd_exec( - self, "mysql -e \"use mysql; " - "GRANT ALL PRIVILEGES on " - "*.* TO 'root'@'127.0.0.1' IDENTIFIED by " - "'{0}' WITH GRANT OPTION\"".format(chars)) + self, + 'mysql -e "ALTER USER root@localhost ' + 'IDENTIFIED VIA mysql_native_password;"') + WOShellExec.cmd_exec( + self, + 'mysql -e "SET PASSWORD = ' + 'PASSWORD(\'{0}\');"'.format(chars)) WOShellExec.cmd_exec( self, 'mysql -e "flush privileges;"') + WOFileUtils.mvfile( + self, '/etc/mysql/conf.d/my.cnf.tmp', + '/etc/mysql/conf.d/my.cnf') except CommandExecutionError: Log.error(self, "Unable to set MySQL password") - Log.info(self, "Tuning MariaDB configuration") + WOGit.add(self, ["/etc/mysql"], + msg="Adding MySQL into Git") + Log.wait(self, "Tuning MariaDB configuration") if not os.path.isfile("/etc/mysql/my.cnf.default-pkg"): WOFileUtils.copyfile(self, "/etc/mysql/my.cnf", "/etc/mysql/my.cnf.default-pkg") @@ -1001,14 +974,18 @@ def post_pref(self, apt_packages, packages, upgrade=False): elif (wo_ram > 64000): wo_innodb_instance = int(64) tmp_table_size = int(256) + mariadbconf = bool(not os.path.exists( + '/etc/mysql/mariadb.conf.d/50-server.cnf')) data = dict( tmp_table_size=tmp_table_size, inno_log=wo_ram_log_size, inno_buffer=wo_ram_innodb, inno_log_buffer=wo_ram_log_buffer, - innodb_instances=wo_innodb_instance) + innodb_instances=wo_innodb_instance, + newmariadb=mariadbconf, release=WOVar.wo_version) if os.path.exists('/etc/mysql/mariadb.conf.d/50-server.cnf'): WOTemplate.deploy( - self, '/etc/mysql/my.cnf', 'my.mustache', data) + self, '/etc/mysql/mariadb.conf.d/50-server.cnf', + 'my.mustache', data) else: WOTemplate.deploy( self, '/etc/mysql/my.cnf', 'my.mustache', data) @@ -1024,15 +1001,16 @@ def post_pref(self, apt_packages, packages, upgrade=False): 'mariadb.service.d/limits.conf', '[Service]\nLimitNOFILE=500000') WOShellExec.cmd_exec(self, 'systemctl daemon-reload') + Log.valide(self, "Tuning MySQL configuration") # set innodb_buffer_pool_instances depending # on the amount of RAM - WOService.stop_service(self, 'mysql') - WOFileUtils.mvfile(self, '/var/lib/mysql/ib_logfile0', - '/var/lib/mysql/ib_logfile0.bak') - WOFileUtils.mvfile(self, '/var/lib/mysql/ib_logfile1', - '/var/lib/mysql/ib_logfile1.bak') - WOService.start_service(self, 'mysql') + WOService.restart_service(self, 'mysql') + + # WOFileUtils.mvfile(self, '/var/lib/mysql/ib_logfile0', + # '/var/lib/mysql/ib_logfile0.bak') + # WOFileUtils.mvfile(self, '/var/lib/mysql/ib_logfile1', + # '/var/lib/mysql/ib_logfile1.bak') WOCron.setcron_weekly(self, 'mysqlcheck -Aos --auto-repair ' '> /dev/null 2>&1', diff --git a/wo/cli/plugins/stack_upgrade.py b/wo/cli/plugins/stack_upgrade.py index f73df85..e3f0082 100644 --- a/wo/cli/plugins/stack_upgrade.py +++ b/wo/cli/plugins/stack_upgrade.py @@ -290,33 +290,25 @@ class WOStackUpgradeController(CementBaseController): start_upgrade = input("Do you want to continue:[y/N]") if start_upgrade != "Y" and start_upgrade != "y": Log.error(self, "Not starting package update") + # additional pre_pref + if "nginx-custom" in apt_packages: + pre_pref(self, WOVar.wo_nginx) Log.wait(self, "Updating APT cache") # apt-get update WOAptGet.update(self) Log.valide(self, "Updating APT cache") - # additional pre_pref - if "nginx-custom" in apt_packages: - pre_pref(self, WOVar.wo_nginx) - if "php7.2-fpm" in apt_packages: - WOAptGet.remove(self, ['php7.2-fpm'], - auto=False, purge=True) - if "php7.3-fpm" in apt_packages: - WOAptGet.remove(self, ['php7.3-fpm'], - auto=False, purge=True) - if "php7.4-fpm" in apt_packages: - WOAptGet.remove(self, ['php7.4-fpm'], - auto=False, purge=True) # check if nginx upgrade is blocked if os.path.isfile( '/etc/apt/preferences.d/nginx-block'): post_pref(self, WOVar.wo_nginx, [], True) + # redis pre_pref + if "redis-server" in apt_packages: + pre_pref(self, WOVar.wo_redis) # upgrade packages WOAptGet.install(self, apt_packages) Log.wait(self, "Configuring APT Packages") post_pref(self, apt_packages, [], True) - if "mariadb-server" in apt_packages: - WOShellExec.cmd_exec(self, 'mysql_upgrade') Log.valide(self, "Configuring APT Packages") # Post Actions after package updates diff --git a/wo/cli/templates/map-wp.mustache b/wo/cli/templates/map-wp.mustache index c1af32c..63b45fa 100644 --- a/wo/cli/templates/map-wp.mustache +++ b/wo/cli/templates/map-wp.mustache @@ -60,11 +60,25 @@ map $http_user_agent $mobile_prefix { } # do not cache requests with query strings -map $is_args $query_no_cache { +map $is_args $is_args_no_cache { default 1; "" 0; } +# cache requests with query string related to analytics +map $args $args_to_cache { + default 0; + "~*utm_" 1; + "~*fbclid" 1; +} + +# do not cache requests with query strings excepted analytics related queries +map $is_args_no_cache$args_to_cache $query_no_cache { + defaut 1; + 00 0; + 11 0; +} + # if all previous check are passed, $skip_cache = 0 map $http_request_no_cache$cookie_no_cache$uri_no_cache$query_no_cache $skip_cache { default 1; diff --git a/wo/cli/templates/my.mustache b/wo/cli/templates/my.mustache index 69e22bc..47568ad 100644 --- a/wo/cli/templates/my.mustache +++ b/wo/cli/templates/my.mustache @@ -1,4 +1,5 @@ # MariaDB database server configuration file. +# Optimized by WordOps {{release}} # # You can copy this file to one of: # - "/etc/mysql/my.cnf" to set global options, @@ -41,7 +42,6 @@ tmpdir = /tmp lc_messages_dir = /usr/share/mysql lc_messages = en_US skip-external-locking -skip-name-resolve = 1 # # Instead of skip-networking the default is now to listen only on # localhost which is more compatible and is not less secure. @@ -143,7 +143,6 @@ innodb_file_per_table = 1 innodb_open_files = 500000 innodb_io_capacity = 500000 innodb_flush_method = O_DIRECT -innodb_buffer_pool_instances = 1 # # * Security Features # @@ -191,5 +190,6 @@ key_buffer = 16M # * IMPORTANT: Additional settings that can override those from this file! # The files must end with '.cnf', otherwise they'll be ignored. # +{{#newmariadb}} !include /etc/mysql/mariadb.cnf -!includedir /etc/mysql/conf.d/ +!includedir /etc/mysql/conf.d/{{/newmariadb}} diff --git a/wo/core/mysql.py b/wo/core/mysql.py index d9af2ff..cdab230 100644 --- a/wo/core/mysql.py +++ b/wo/core/mysql.py @@ -88,7 +88,7 @@ class WOMysql(): finally: connection.close() - def backupAll(self): + def backupAll(self, fulldump=False): import subprocess try: Log.info(self, "Backing up database at location: " @@ -98,24 +98,47 @@ class WOMysql(): Log.debug(self, 'Creating directory' '/var/lib/wo-backup/mysql') os.makedirs('/var/lib/wo-backup/mysql') - - db = subprocess.check_output(["/usr/bin/mysql " - "-Bse \'show databases\'"], - universal_newlines=True, - shell=True).split('\n') - for dbs in db: - if dbs == "": - continue - Log.info(self, "Backing up {0} database".format(dbs)) + if not fulldump: + db = subprocess.check_output( + ["/usr/bin/mysql " + "-Bse \'show databases\'"], + universal_newlines=True, + shell=True).split('\n') + for dbs in db: + if dbs == "": + continue + Log.info(self, "Backing up {0} database".format(dbs)) + p1 = subprocess.Popen( + "/usr/bin/mysqldump {0} --max_allowed_packet=1024M " + "--single-transaction ".format(dbs), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, shell=True) + p2 = subprocess.Popen( + "/usr/bin/zstd -T0 -c > " + "/var/lib/wo-backup/mysql/{0}{1}.sql.zst" + .format(dbs, WOVar.wo_date), + stdin=p1.stdout, shell=True) + # Allow p1 to receive a SIGPIPE if p2 exits + p1.stdout.close() + output = p1.stderr.read() + p1.wait() + if p1.returncode == 0: + Log.debug(self, "done") + else: + Log.error(self, output.decode("utf-8")) + else: + Log.info(self, "Backing up all databases") p1 = subprocess.Popen( - "/usr/bin/mysqldump {0} --max_allowed_packet=1024M " - "--single-transaction ".format(dbs), - stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + "/usr/bin/mysqldump --all-databases " + "--max_allowed_packet=1024M " + "--single-transaction --events", + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, shell=True) p2 = subprocess.Popen( "/usr/bin/zstd -T0 -c > " - "/var/lib/wo-backup/mysql/{0}{1}.sql.zst" - .format(dbs, WOVar.wo_date), stdin=p1.stdout, shell=True) - # Allow p1 to receive a SIGPIPE if p2 exits + "/var/lib/wo-backup/mysql/fulldump-{0}.sql.zst" + .format(WOVar.wo_date), + stdin=p1.stdout, shell=True) p1.stdout.close() output = p1.stderr.read() p1.wait() @@ -123,6 +146,7 @@ class WOMysql(): Log.debug(self, "done") else: Log.error(self, output.decode("utf-8")) + except Exception as e: Log.error(self, "Error: process exited with status %s" % e) diff --git a/wo/core/variables.py b/wo/core/variables.py index 62fd5f2..6af68de 100644 --- a/wo/core/variables.py +++ b/wo/core/variables.py @@ -14,7 +14,7 @@ class WOVar(): """Intialization of core variables""" # WordOps version - wo_version = "3.12.4" + wo_version = "3.13.0" # WordOps packages versions wo_wp_cli = "2.4.0" wo_adminer = "4.7.5" @@ -83,7 +83,7 @@ class WOVar(): while not match(r"^[A-Za-z0-9\.\+_-]+@[A-Za-z0-9\._-]+\.[a-zA-Z]*$", wo_email): print("Whoops, seems like you made a typo - " - "the e-mailaddress is invalid...") + "the e-mail address is invalid...") wo_email = input("Enter your email: ") git.config("--global", "user.name", "{0}".format(wo_user)) @@ -166,7 +166,7 @@ class WOVar(): else: mariadb_ver = '10.3' else: - mariadb_ver = '10.3' + mariadb_ver = '10.5' if wo_platform_codename == 'jessie': wo_mysql = wo_mysql + ["python3-mysql.connector"] else: @@ -185,7 +185,7 @@ class WOVar(): # APT repositories wo_mysql_repo = ("deb [arch=amd64,ppc64el] " "http://mariadb.mirrors.ovh.net/MariaDB/repo/" - "10.3/{distro} {codename} main" + "10.5/{distro} {codename} main" .format(distro=wo_distro, codename=wo_platform_codename)) if wo_distro == 'ubuntu':