Merge pull request #329 from WordOps/updating-configuration

v3.13.0 - WordOps stable release

.travis.yml

@@ -4,9 +4,11 @@ jobs:
   include:
     - os: linux
       dist: xenial
-    - os: linux
+    - if: branch = master
+      os: linux
       dist: bionic
-    - os: linux
+    - if: branch = master
+      os: linux
       dist: focal
 
 notifications:

CHANGELOG.md

@@ -8,6 +8,23 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ### v3.9.x - [Unreleased]
 
+#### v3.13.0 - 2020-10-25
+
+#### Added
+
+- MariaDB 10.5 support (installed by default)
+- Upgrade to MariaDB 10.5 with `wo stack migrate --mariadb`
+
+#### Changed
+
+- Improved Nginx caching rules to cache requests with query strings related to analytics (utm_, fbclid)
+- WordOps is installed inside a Python virtual environment in /opt/wo to isolate it from the system's Python libraries
+
+#### Fixed
+
+- Useless php-cli version removal
+- Redis 6.0.6 not installed on Ubuntu 20.04 LTS
+
 ### v3.12.4 - 2020-10-14
 
 #### Changed

README.md

@@ -42,7 +42,7 @@
 - **Easy to install** : One step automated installer with migration from EasyEngine v3 support
 - **Fast deployment** : Fast and automated WordPress, Nginx, PHP, MySQL & Redis installation
 - **Custom Nginx build** : Nginx 1.18.0 - TLS v1.3 Cloudflare HTTP/2 HPACK & Brotli support
-- **Up-to-date** : PHP 7.2, 7.3 & 7.4, MariaDB 10.3 & Redis 6.0
+- **Up-to-date** : PHP 7.2, 7.3 & 7.4, MariaDB 10.5 & Redis 6.0
 - **Secured** : Hardened WordPress security with strict Nginx location directives
 - **Powerful** : Optimized Nginx configurations with multiple cache backends support
 - **SSL** : Domain, Subdomain & Wildcard Let's Encrypt SSL certificates with DNS API support

config/bash_completion.d/wo_auto.rc

@@ -35,7 +35,7 @@ _wo_complete()
 
             "stack")
                 COMPREPLY=( $(compgen \
-                              -W "upgrade install purge reload remove restart start status stop" \
+                              -W "upgrade install migrate purge reload remove restart start status stop" \
                               -- $cur) )
                 ;;
 
@@ -74,17 +74,22 @@ _wo_complete()
             # HANDLE EVERYTHING AFTER THE THIRD LEVEL NAMESPACE
             "install" | "purge" | "remove" )
                 COMPREPLY=( $(compgen \
-                              -W "--web --admin --security --nginx --php --php73 --php74 --mysql --wpcli --phpmyadmin --adminer --utils --redis --phpredisadmin --composer --netdata --fail2ban --ufw --dashboard --proftpd --clamav --sendmail --ngxblocker --mysqlclient --mysqltuner --extplorer --nanorc --cheat --all --force" \
+                              -W "--web --admin --security --nginx --php72 --php73 --php74 --mysql --wpcli --phpmyadmin --adminer --utils --redis --phpredisadmin --composer --netdata --fail2ban --ufw --dashboard --proftpd --clamav --sendmail --ngxblocker --mysqlclient --mysqltuner --extplorer --nanorc --cheat --all --force" \
                               -- $cur) )
                 ;;
             "upgrade" )
                 COMPREPLY=( $(compgen \
-                              -W "--web --admin --utils --nginx --php --php73 --php74 --mysql --all --netdata --composer --phpmyadmin --adminer --dashboard --mysqtuner --wpcli --force" \
+                              -W "--web --admin --utils --nginx --php72 --php73 --php74 --mysql --all --netdata --composer --phpmyadmin --adminer --dashboard --mysqltuner --wpcli --force" \
+                              -- $cur) )
+                ;;
+            "migrate")
+                COMPREPLY=( $(compgen \
+                              -W "--mariadb --force" \
                               -- $cur) )
                 ;;
             "start" | "stop" | "reload" | "restart" | "status")
                 COMPREPLY=( $(compgen \
-                              -W "--nginx --php --php73 --php74 --mysql --redis --fail2ban --ufw --netdata -proftpd" \
+                              -W "--nginx --php --php72 --php73 --php74 --mysql --redis --fail2ban --ufw --netdata -proftpd" \
                               -- $cur) )
                 ;;
             "list")

install

@@ -9,7 +9,7 @@
 # -------------------------------------------------------------------------
 # wget -qO wo wops.cc && sudo -E bash wo
 # -------------------------------------------------------------------------
-# Version 3.12.4 - 2020-10-14
+# Version 3.13.0 - 2020-10-25
 # -------------------------------------------------------------------------
 
 # CONTENTS
@@ -167,7 +167,7 @@ wo_check_distro() {
             wo_lib_echo_fail "Feel free to open a pull-request if you want to add support for another Linux distributions"
             exit 100
         else
-            check_wo_linux_distro=$(lsb_release -sc | grep -E "xenial|bionic|disco|jessie|stretch|buster|focal")
+            check_wo_linux_distro=$(lsb_release -sc | grep -E "xenial|bionic|jessie|stretch|buster|focal")
             if [ -z "$check_wo_linux_distro" ]; then
                 wo_lib_echo_fail "WordOps (wo) only supports Ubuntu 16.04/18.04/20.04 LTS, Debian 9.x/10.x and Raspbian 9.x/10x.\n
                 You can bypass this warning by adding the flag --force to the install command"
@@ -208,13 +208,13 @@ wo_install_dep() {
     if [ "$wo_linux_distro" == "Ubuntu" ]; then
         # install dependencies
         apt-get -option=Dpkg::options::=--force-confmiss --option=Dpkg::options::=--force-confold --assume-yes install \
-            build-essential curl gzip python3-pip python3-apt gcc python3-dev sqlite3 git tar software-properties-common pigz \
+            build-essential curl gzip python3-pip python3-apt python3-venv gcc python3-dev sqlite3 git tar software-properties-common pigz \
             gnupg2 cron ccze rsync apt-transport-https tree haveged ufw unattended-upgrades tzdata ntp zstd idn >/dev/null 2>&1
         curl -sL https://download.opensuse.org/repositories/home:/virtubox:/WordOps/xUbuntu_18.04/Release.key | apt-key add -
     else
         # install dependencies
         apt-get -option=Dpkg::options::=--force-confmiss --option=Dpkg::options::=--force-confold --assume-yes install \
-            build-essential curl gzip dirmngr sudo python3-pip python3-apt gcc python3-dev ca-certificates sqlite3 git tar \
+            build-essential curl gzip dirmngr sudo python3-pip python3-apt python3-venv gcc python3-dev ca-certificates sqlite3 git tar \
             software-properties-common pigz apt-transport-https gnupg2 cron ccze rsync tree haveged ufw unattended-upgrades tzdata ntp zstd idn >/dev/null 2>&1
         # add php repository gpg key
         [ -d /etc/apt/trusted.gpg.d ] && { wget -qO /etc/apt/trusted.gpg.d/php.gpg https://packages.sury.org/php/apt.gpg; }
@@ -228,8 +228,6 @@ wo_install_dep() {
     fi
     # upgrade pip
     python3 -m pip install --upgrade pip
-    python3 -m pip install --upgrade setuptools wheel
-
 }
 
 wo_timesync() {
@@ -432,14 +430,28 @@ wo_install() {
     if [ -d /usr/local/lib/python3."$python_ver"/dist-packages ]; then
         cd /usr/local/lib/python3."$python_ver"/dist-packages || exit 1
     fi
+    python3 -m pip uninstall -yq wo wordops ee
+    if [ -d /opt/wo ]; then
+        cd /opt/wo || exit 1
+        source bin/activate
+    else
+        mkdir -p /opt/wo
+        cd /opt || exit 1
+        python3 -m venv --system-site-packages wo
+        source wo/bin/activate
+    fi
+    python3 -m pip install -U pip setuptools wheel
     if [ "$wo_branch" = "master" ]; then
         python3 -m pip install -I wordops
     else
         python3 -m pip install -I "git+git://github.com/WordOps/WordOps.git@$wo_branch#egg=wordops"
     fi
-    cp -rf /usr/local/lib/python3."$python_ver"/dist-packages/usr/* /usr/
+    cp -rf /opt/wo/lib/python3."$python_ver"/site-packages/usr/* /usr/
-    cp -rn /usr/local/lib/python3."$python_ver"/dist-packages/etc/* /etc/
+    cp -rn /opt/wo/lib/python3."$python_ver"/site-packages/etc/* /etc/
-    cp -f /usr/local/lib/python3."$python_ver"/dist-packages/etc/bash_completion.d/wo_auto.rc /etc/bash_completion.d/wo_auto.rc
+    cp -f /opt/wo/lib/python3."$python_ver"/site-packages/etc/bash_completion.d/wo_auto.rc /etc/bash_completion.d/wo_auto.rc
+    rm -rf /usr/local/bin/wo /usr/local/lib/python3."$python_ver"/dist-packages/{etc,usr}
+    ln -s /opt/wo/bin/wo /usr/local/bin/
+    deactivate
 }
 
 # Clone Github repository if it doesn't exist
@@ -450,15 +462,24 @@ wo_travis_install() {
         rm -rf dist
     fi
     if [ -f ./setup.py ]; then
-        python3 setup.py sdist bdist_wheel
+        if [ -d /opt/wo ]; then
-        python3 -m pip install --upgrade dist/*.whl
+            source /opt/wo/bin/activate
+        else
+            mkdir -p /opt/wo
+            python3 -m venv --system-site-packages /opt/wo
+            source /opt/wo/bin/activate
+        fi
+        python3 -m pip install -U pip setuptools wheel
+        python3 -m pip install -U .
     else
         python3 -m pip install -U "git+git://github.com/WordOps/WordOps.git@$wo_branch#egg=wordops"
     fi
-    cp -rf /usr/local/lib/python3."$python_ver"/dist-packages/usr/* /usr/
+    cp -rf /opt/wo/lib/python3."$python_ver"/site-packages/usr/* /usr/
-    cp -rn /usr/local/lib/python3."$python_ver"/dist-packages/etc/* /etc/
+    cp -rn /opt/wo/lib/python3."$python_ver"/site-packages/etc/* /etc/
-    cp -f /usr/local/lib/python3."$python_ver"/dist-packages/etc/bash_completion.d/wo_auto.rc /etc/bash_completion.d/wo_auto.rc
+    cp -f /opt/wo/lib/python3."$python_ver"/site-packages/etc/bash_completion.d/wo_auto.rc /etc/bash_completion.d/wo_auto.rc
-
+    rm -f /usr/local/bin/wo
+    ln -s /opt/wo/bin/wo /usr/local/bin/
+    deactivate
 }
 
 wo_upgrade_nginx() {
@@ -615,7 +636,7 @@ wo_uninstall() {
     if { python3 -m pip list | grep -q "wordops" >/dev/null 2>&1; }; then
         python3 -m pip uninstall wordops -y
     fi
-    rm -rf /usr/local/lib/python3."$python_ver"/dist-packages/{pystache-*,cement-2.*,wo-*,wordops-*} /usr/local/bin/wo /etc/bash_completion.d/wo_auto.rc /var/lib/wo /etc/wo /usr/lib/wo/templates
+    rm -rf /usr/local/lib/python3."$python_ver"/dist-packages/{pystache-*,cement-2.*,wo-*,wordops-*} /usr/local/bin/wo /etc/bash_completion.d/wo_auto.rc /var/lib/wo /etc/wo /usr/lib/wo/templates /opt/wo
 }
 
 wo_clean_repo() {
@@ -704,13 +725,18 @@ wo_init() {
 wo_php_fix() {
     local php_versions="5.6 7.0 7.1 7.2 7.3 7.4"
 
-    apt-get autoremove --assume-yes --purge php8.0-* php-igbinary php-memcached php-msgpack php-redis php-imagick php-xdebug
+    apt-get autoremove --assume-yes --purge php8.0-* php-igbinary php-memcached php-msgpack php-redis php-imagick php-xdebug php-memcache
     for php_version in $php_versions; do
-        if [ -f "/usr/sbin/php-fpm$php_version" ]; then
+        if [ -f "/usr/bin/php$php_version" ]; then
-            local php_extensions="igbinary memcached msgpack redis imagick xdebug"
+            if [ -f "/usr/sbin/php-fpm$php_version" ]; then
-            for php_ext in $php_extensions; do
+                local php_extensions="igbinary memcached msgpack redis imagick xdebug"
-                apt-get -option=Dpkg::options::=--force-confmiss --option=Dpkg::options::=--force-confold --assume-yes install "php$php_version-$php_ext"
+                for php_ext in $php_extensions; do
-            done
+                    apt-get -option=Dpkg::options::=--force-confmiss --option=Dpkg::options::=--force-confold --assume-yes install "php$php_version-$php_ext"
+                done
+            else
+                apt-get autoremove --assume-yes --purge "php$php_version-*"
+            fi
+
         fi
     done
 }

setup.py

@@ -27,7 +27,7 @@ if os.geteuid() == 0:
         os.makedirs('/var/lib/wo/tmp/')
 
 setup(name='wordops',
-      version='3.12.4',
+      version='3.13.0',
       description='An essential toolset that eases server administration',
       long_description=LONG,
       long_description_content_type='text/markdown',

tests/travis.sh

@@ -80,7 +80,7 @@ echo
 echo -e "${CGREEN}#############################################${CEND}"
 echo -e '       wo site update --php74              '
 echo -e "${CGREEN}#############################################${CEND}"
-other_site_types='mysql php72 php74 wp wpfc wpsc wpredis wpce wprocket wpsubdomain wpsubdir ngxblocker'
+other_site_types='mysql php72 php73 wp wpfc wpsc wpredis wpce wprocket wpsubdomain wpsubdir'
 for site in $other_site_types; do
     echo -ne "       Updating site to $site php74              [..]\r"
     if {
@@ -103,7 +103,7 @@ echo
 echo -e "${CGREEN}#############################################${CEND}"
 echo -e '       wo site update --php73              '
 echo -e "${CGREEN}#############################################${CEND}"
-other_site_types='html mysql wp php72 php73 wpfc wpsc wpredis wpce wprocket wpsubdomain wpsubdir ngxblocker'
+other_site_types='html mysql wp php72 php73 wpfc wpsc wpredis wpce wprocket wpsubdomain wpsubdir'
 for site in $other_site_types; do
     echo -ne "       Updating site to $site php73              [..]\r"
     if {
@@ -126,7 +126,7 @@ echo
 echo -e "${CGREEN}#############################################${CEND}"
 echo -e '       wo site update --php72              '
 echo -e "${CGREEN}#############################################${CEND}"
-other_site_types='mysql php72 php73 php74 wp wpfc wpsc wpredis wpce wprocket wpsubdomain wpsubdir ngxblocker'
+other_site_types='mysql php72 php73 wp wpfc wpsc wpredis wpce wprocket wpsubdomain wpsubdir'
 for site in $other_site_types; do
     echo -ne "       Updating site to $site php72              [..]\r"
     if {
@@ -226,6 +226,24 @@ if [ -z "$1" ]; then
         fi
     done
 fi
+
+echo -e "${CGREEN}#############################################${CEND}"
+echo -e '       wo stack migrate --mariadb              '
+echo -e "${CGREEN}#############################################${CEND}"
+
+        echo -ne "      Upgrading mariadb               [..]\r"
+        if {
+            wo stack migrate --mariadb --force
+        } >>/var/log/wo/test.log; then
+            echo -ne "       Upgrading mariadb               [${CGREEN}OK${CEND}]\\r"
+            echo -ne '\n'
+        else
+            echo -e "        Upgrading mariadb              [${CRED}FAIL${CEND}]"
+            echo -ne '\n'
+            exit_script
+
+        fi
+
 echo -e "${CGREEN}#############################################${CEND}"
 echo -e '       wo clean              '
 echo -e "${CGREEN}#############################################${CEND}"

wo/cli/controllers/base.py

@@ -8,7 +8,7 @@ VERSION = WOVar.wo_version
 
 BANNER = """
 WordOps v%s
-Copyright (c) 2019 WordOps.
+Copyright (c) 2020 WordOps.
 """ % VERSION
 
 

wo/cli/plugins/stack_migrate.py

@@ -1,14 +1,13 @@
-import configparser
-import os
-
 from cement.core.controller import CementBaseController, expose
 
-from wo.core.apt_repo import WORepo
+from wo.cli.plugins.stack_pref import post_pref, pre_pref
 from wo.core.aptget import WOAptGet
+from wo.core.fileutils import WOFileUtils
 from wo.core.logging import Log
 from wo.core.mysql import WOMysql
 from wo.core.shellexec import WOShellExec
 from wo.core.variables import WOVar
+from wo.core.apt_repo import WORepo
 
 
 class WOStackMigrateController(CementBaseController):
@@ -19,104 +18,69 @@ class WOStackMigrateController(CementBaseController):
         description = ('Migrate stack safely')
         arguments = [
             (['--mariadb'],
-                dict(help="Migrate database to MariaDB",
+                dict(help="Migrate/Upgrade database to MariaDB",
+                     action='store_true')),
+            (['--force'],
+                dict(help="Force Packages upgrade without any prompt",
                      action='store_true')),
         ]
 
     @expose(hide=True)
     def migrate_mariadb(self):
         # Backup all database
-        WOMysql.backupAll(self)
+        WOMysql.backupAll(self, fulldump=True)
-
-        if not WOVar.wo_distro == 'raspbian':
-            if (not WOVar.wo_platform_codename == 'jessie'):
-                wo_mysql = ["mariadb-server", "percona-toolkit",
-                            "python3-mysqldb", "mariadb-backup"]
-            else:
-                wo_mysql = ["mariadb-server", "percona-toolkit",
-                            "python3-mysql.connector"]
-        else:
-            wo_mysql = ["mariadb-server", "percona-toolkit",
-                        "python3-mysqldb"]
 
+        # Remove previous MariaDB repository
+        wo_mysql_old_repo = (
+            "deb [arch=amd64,ppc64el] "
+            "http://mariadb.mirrors.ovh.net/MariaDB/repo/"
+            "10.3/{distro} {codename} main"
+            .format(distro=WOVar.wo_distro,
+                    codename=WOVar.wo_platform_codename))
+        if WOFileUtils.grepcheck(
+                self, '/etc/apt/sources.list.d/wo-repo.list',
+                wo_mysql_old_repo):
+            WORepo.remove(self, repo_url=wo_mysql_old_repo)
         # Add MariaDB repo
-        Log.info(self, "Adding repository for MariaDB, please wait...")
+        pre_pref(self, WOVar.wo_mysql)
-
-        mysql_pref = ("Package: *\nPin: origin sfo1.mirrors.digitalocean.com"
-                      "\nPin-Priority: 1000\n")
-        with open('/etc/apt/preferences.d/'
-                  'MariaDB.pref', 'w') as mysql_pref_file:
-            mysql_pref_file.write(mysql_pref)
-
-        WORepo.add(self, repo_url=WOVar.wo_mysql_repo)
-        Log.debug(self, 'Adding key for {0}'
-                  .format(WOVar.wo_mysql_repo))
-        WORepo.add_key(self, '0xcbcb082a1bb943db',
-                       keyserver="keyserver.ubuntu.com")
-
-        config = configparser.ConfigParser()
-        if os.path.exists('/etc/mysql/conf.d/my.cnf'):
-            config.read('/etc/mysql/conf.d/my.cnf')
-        else:
-            config.read(os.path.expanduser("~")+'/.my.cnf')
-
-        try:
-            chars = config['client']['password']
-        except Exception as e:
-            Log.error(self, "Error: process exited with error %s"
-                            % e)
-
-        Log.debug(self, "Pre-seeding MariaDB")
-        Log.debug(self, "echo \"mariadb-server-10.3 "
-                        "mysql-server/root_password "
-                        "password \" | "
-                        "debconf-set-selections")
-        WOShellExec.cmd_exec(self, "echo \"mariadb-server-10.3 "
-                                   "mysql-server/root_password "
-                                   "password {chars}\" | "
-                                   "debconf-set-selections"
-                                   .format(chars=chars),
-                                   log=False)
-        Log.debug(self, "echo \"mariadb-server-10.3 "
-                        "mysql-server/root_password_again "
-                        "password \" | "
-                        "debconf-set-selections")
-        WOShellExec.cmd_exec(self, "echo \"mariadb-server-10.3 "
-                                   "mysql-server/root_password_again "
-                                   "password {chars}\" | "
-                                   "debconf-set-selections"
-                                   .format(chars=chars),
-                                   log=False)
 
         # Install MariaDB
-        apt_packages = wo_mysql
 
-        Log.info(self, "Updating apt-cache, hang on...")
+        Log.wait(self, "Updating apt-cache          ")
         WOAptGet.update(self)
-        Log.info(self, "Installing MariaDB, hang on...")
+        Log.valide(self, "Updating apt-cache          ")
-        WOAptGet.remove(self, ["mysql-common", "libmysqlclient18"])
+        Log.wait(self, "Upgrading MariaDB          ")
+        WOAptGet.remove(self, ["mariadb-server"])
         WOAptGet.auto_remove(self)
-        WOAptGet.install(self, apt_packages)
+        WOAptGet.install(self, WOVar.wo_mysql)
+        Log.valide(self, "Upgrading MariaDB          ")
+        WOFileUtils.mvfile(
+            self, '/etc/mysql/my.cnf', '/etc/mysql/my.cnf.old')
+        WOFileUtils.create_symlink(
+            self, ['/etc/mysql/mariadb.cnf', '/etc/mysql/my.cnf'])
+        WOShellExec.cmd_exec(self, 'systemctl daemon-reload')
+        post_pref(self, WOVar.wo_mysql, [])
 
     @expose(hide=True)
     def default(self):
-        if ((not self.app.pargs.mariadb)):
+        pargs = self.app.pargs
+        if ((not pargs.mariadb)):
             self.app.args.print_help()
-        if self.app.pargs.mariadb:
+        if pargs.mariadb:
             if WOVar.wo_mysql_host != "localhost":
                 Log.error(
                     self, "Remote MySQL server in use, skipping local install")
 
-            if (WOShellExec.cmd_exec(self, "mysqladmin ping") and
+            if (WOShellExec.cmd_exec(self, "mysqladmin ping")):
-                    (not WOAptGet.is_installed(self, 'mariadb-server'))):
 
                 Log.info(self, "If your database size is big, "
                          "migration may take some time.")
                 Log.info(self, "During migration non nginx-cached parts of "
                          "your site may remain down")
-                start_migrate = input("Type \"mariadb\" to continue:")
+                if not pargs.force:
-                if start_migrate != "mariadb":
+                    start_upgrade = input("Do you want to continue:[y/N]")
-                    Log.error(self, "Not starting migration")
+                    if start_upgrade != "Y" and start_upgrade != "y":
+                        Log.error(self, "Not starting package update")
                 self.migrate_mariadb()
             else:
                 Log.error(self, "Your current MySQL is not alive or "

wo/cli/plugins/stack_pref.py

@@ -42,42 +42,10 @@ def pre_pref(self, apt_packages):
                            keyserver='keyserver.ubuntu.com')
             WORepo.add_key(self, '0xF1656F24C74CD1D8',
                            keyserver='keyserver.ubuntu.com')
-    if "mariadb-server" in apt_packages:
+    if ("mariadb-server" in apt_packages and
+            not os.path.exists('/etc/mysql/conf.d/my.cnf')):
         # generate random 24 characters root password
         chars = ''.join(random.sample(string.ascii_letters, 24))
-
-        Log.debug(self, "Pre-seeding MySQL")
-        Log.debug(self, "echo \"mariadb-server-{0} "
-                  "mysql-server/root_password "
-                  "password \" | "
-                  "debconf-set-selections"
-                  .format(WOVar.mariadb_ver))
-        try:
-            WOShellExec.cmd_exec(self, "echo \"mariadb-server-{0} "
-                                 "mysql-server/root_password "
-                                 "password {chars}\" | "
-                                 "debconf-set-selections"
-                                 .format(WOVar.mariadb_ver, chars=chars),
-                                 log=False)
-        except CommandExecutionError as e:
-            Log.debug(self, "{0}".format(e))
-            Log.error(self, "Failed to initialize MySQL package")
-
-        Log.debug(self, "echo \"mariadb-server-{0} "
-                  "mysql-server/root_password_again "
-                  "password \" | "
-                  "debconf-set-selections"
-                  .format(WOVar.mariadb_ver))
-        try:
-            WOShellExec.cmd_exec(self, "echo \"mariadb-server-{0} "
-                                 "mysql-server/root_password_again "
-                                 "password {chars}\" | "
-                                 "debconf-set-selections"
-                                 .format(WOVar.mariadb_ver, chars=chars),
-                                 log=False)
-        except CommandExecutionError as e:
-            Log.debug(self, "{0}".format(e))
-            Log.error(self, "Failed to initialize MySQL package")
         # generate my.cnf root credentials
         mysql_config = """
             [client]
@@ -87,13 +55,13 @@ def pre_pref(self, apt_packages):
         config = configparser.ConfigParser()
         config.read_string(mysql_config)
         Log.debug(self, 'Writting configuration into MySQL file')
-        conf_path = "/etc/mysql/conf.d/my.cnf"
+        conf_path = "/etc/mysql/conf.d/my.cnf.tmp"
         os.makedirs(os.path.dirname(conf_path), exist_ok=True)
         with open(conf_path, encoding='utf-8',
                   mode='w') as configfile:
             config.write(configfile)
         Log.debug(self, 'Setting my.cnf permission')
-        WOFileUtils.chmod(self, "/etc/mysql/conf.d/my.cnf", 0o600)
+        WOFileUtils.chmod(self, "/etc/mysql/conf.d/my.cnf.tmp", 0o600)
 
     # add nginx repository
     if set(WOVar.wo_nginx).issubset(set(apt_packages)):
@@ -966,24 +934,29 @@ def post_pref(self, apt_packages, packages, upgrade=False):
                 config_file.close()
             else:
                 # make sure root account have all privileges
-                if "PASSWORD" not in WOShellExec.cmd_exec_stdout(
+                if os.path.exists('/etc/mysql/conf.d/my.cnf.tmp'):
-                        self, 'mysql -e "use mysql; show grants;"'):
                     try:
-                        if not os.path.exists('/etc/mysql/conf.d/my.cnf'):
-                            Log.error(self, 'my.cnf not found')
                         config = configparser.ConfigParser()
-                        config.read('/etc/mysql/conf.d/my.cnf')
+                        config.read('/etc/mysql/conf.d/my.cnf.tmp')
                         chars = config['client']['password']
                         WOShellExec.cmd_exec(
-                            self, "mysql -e \"use mysql; "
+                            self,
-                            "GRANT ALL PRIVILEGES on "
+                            'mysql -e "ALTER USER root@localhost '
-                            "*.* TO 'root'@'127.0.0.1' IDENTIFIED by "
+                            'IDENTIFIED VIA mysql_native_password;"')
-                            "'{0}' WITH GRANT OPTION\"".format(chars))
+                        WOShellExec.cmd_exec(
+                            self,
+                            'mysql -e "SET PASSWORD = '
+                            'PASSWORD(\'{0}\');"'.format(chars))
                         WOShellExec.cmd_exec(
                             self, 'mysql -e "flush privileges;"')
+                        WOFileUtils.mvfile(
+                            self, '/etc/mysql/conf.d/my.cnf.tmp',
+                            '/etc/mysql/conf.d/my.cnf')
                     except CommandExecutionError:
                         Log.error(self, "Unable to set MySQL password")
-                Log.info(self, "Tuning MariaDB configuration")
+                    WOGit.add(self, ["/etc/mysql"],
+                              msg="Adding MySQL into Git")
+                Log.wait(self, "Tuning MariaDB configuration")
                 if not os.path.isfile("/etc/mysql/my.cnf.default-pkg"):
                     WOFileUtils.copyfile(self, "/etc/mysql/my.cnf",
                                          "/etc/mysql/my.cnf.default-pkg")
@@ -1001,14 +974,18 @@ def post_pref(self, apt_packages, packages, upgrade=False):
                 elif (wo_ram > 64000):
                     wo_innodb_instance = int(64)
                     tmp_table_size = int(256)
+                mariadbconf = bool(not os.path.exists(
+                    '/etc/mysql/mariadb.conf.d/50-server.cnf'))
                 data = dict(
                     tmp_table_size=tmp_table_size, inno_log=wo_ram_log_size,
                     inno_buffer=wo_ram_innodb,
                     inno_log_buffer=wo_ram_log_buffer,
-                    innodb_instances=wo_innodb_instance)
+                    innodb_instances=wo_innodb_instance,
+                    newmariadb=mariadbconf, release=WOVar.wo_version)
                 if os.path.exists('/etc/mysql/mariadb.conf.d/50-server.cnf'):
                     WOTemplate.deploy(
-                        self, '/etc/mysql/my.cnf', 'my.mustache', data)
+                        self, '/etc/mysql/mariadb.conf.d/50-server.cnf',
+                        'my.mustache', data)
                 else:
                     WOTemplate.deploy(
                         self, '/etc/mysql/my.cnf', 'my.mustache', data)
@@ -1024,15 +1001,16 @@ def post_pref(self, apt_packages, packages, upgrade=False):
                             'mariadb.service.d/limits.conf',
                             '[Service]\nLimitNOFILE=500000')
                         WOShellExec.cmd_exec(self, 'systemctl daemon-reload')
+                Log.valide(self, "Tuning MySQL configuration")
                 # set innodb_buffer_pool_instances depending
                 # on the amount of RAM
 
-                WOService.stop_service(self, 'mysql')
+                WOService.restart_service(self, 'mysql')
-                WOFileUtils.mvfile(self, '/var/lib/mysql/ib_logfile0',
+
-                                   '/var/lib/mysql/ib_logfile0.bak')
+                # WOFileUtils.mvfile(self, '/var/lib/mysql/ib_logfile0',
-                WOFileUtils.mvfile(self, '/var/lib/mysql/ib_logfile1',
+                #                    '/var/lib/mysql/ib_logfile0.bak')
-                                   '/var/lib/mysql/ib_logfile1.bak')
+                # WOFileUtils.mvfile(self, '/var/lib/mysql/ib_logfile1',
-                WOService.start_service(self, 'mysql')
+                #                    '/var/lib/mysql/ib_logfile1.bak')
 
             WOCron.setcron_weekly(self, 'mysqlcheck -Aos --auto-repair '
                                   '> /dev/null 2>&1',

wo/cli/plugins/stack_upgrade.py

@@ -290,33 +290,25 @@ class WOStackUpgradeController(CementBaseController):
                     start_upgrade = input("Do you want to continue:[y/N]")
                     if start_upgrade != "Y" and start_upgrade != "y":
                         Log.error(self, "Not starting package update")
+                # additional pre_pref
+                if "nginx-custom" in apt_packages:
+                    pre_pref(self, WOVar.wo_nginx)
                 Log.wait(self, "Updating APT cache")
                 # apt-get update
                 WOAptGet.update(self)
                 Log.valide(self, "Updating APT cache")
 
-                # additional pre_pref
-                if "nginx-custom" in apt_packages:
-                    pre_pref(self, WOVar.wo_nginx)
-                if "php7.2-fpm" in apt_packages:
-                    WOAptGet.remove(self, ['php7.2-fpm'],
-                                    auto=False, purge=True)
-                if "php7.3-fpm" in apt_packages:
-                    WOAptGet.remove(self, ['php7.3-fpm'],
-                                    auto=False, purge=True)
-                if "php7.4-fpm" in apt_packages:
-                    WOAptGet.remove(self, ['php7.4-fpm'],
-                                    auto=False, purge=True)
                 # check if nginx upgrade is blocked
                 if os.path.isfile(
                         '/etc/apt/preferences.d/nginx-block'):
                     post_pref(self, WOVar.wo_nginx, [], True)
+                # redis pre_pref
+                if "redis-server" in apt_packages:
+                    pre_pref(self, WOVar.wo_redis)
                 # upgrade packages
                 WOAptGet.install(self, apt_packages)
                 Log.wait(self, "Configuring APT Packages")
                 post_pref(self, apt_packages, [], True)
-                if "mariadb-server" in apt_packages:
-                    WOShellExec.cmd_exec(self, 'mysql_upgrade')
                 Log.valide(self, "Configuring APT Packages")
                 # Post Actions after package updates
 

wo/cli/templates/map-wp.mustache

@@ -60,11 +60,25 @@ map $http_user_agent $mobile_prefix {
 }
 
 # do not cache requests with query strings
-map $is_args $query_no_cache {
+map $is_args $is_args_no_cache {
     default 1;
     "" 0;
 }
 
+# cache requests with query string related to analytics
+map $args $args_to_cache {
+    default 0;
+    "~*utm_" 1;
+    "~*fbclid" 1;
+}
+
+# do not cache requests with query strings excepted analytics related queries
+map $is_args_no_cache$args_to_cache $query_no_cache {
+    defaut 1;
+    00 0;
+    11 0;
+}
+
 # if all previous check are passed, $skip_cache = 0
 map $http_request_no_cache$cookie_no_cache$uri_no_cache$query_no_cache $skip_cache {
     default 1;

wo/cli/templates/my.mustache

@@ -1,4 +1,5 @@
 # MariaDB database server configuration file.
+# Optimized by WordOps {{release}}
 #
 # You can copy this file to one of:
 # - "/etc/mysql/my.cnf" to set global options,
@@ -41,7 +42,6 @@ tmpdir		= /tmp
 lc_messages_dir	= /usr/share/mysql
 lc_messages	= en_US
 skip-external-locking
-skip-name-resolve = 1
 #
 # Instead of skip-networking the default is now to listen only on
 # localhost which is more compatible and is not less secure.
@@ -143,7 +143,6 @@ innodb_file_per_table	= 1
 innodb_open_files	= 500000
 innodb_io_capacity	= 500000
 innodb_flush_method	= O_DIRECT
-innodb_buffer_pool_instances = 1
 #
 # * Security Features
 #
@@ -191,5 +190,6 @@ key_buffer		= 16M
 # * IMPORTANT: Additional settings that can override those from this file!
 #   The files must end with '.cnf', otherwise they'll be ignored.
 #
+{{#newmariadb}}
 !include /etc/mysql/mariadb.cnf
-!includedir /etc/mysql/conf.d/
+!includedir /etc/mysql/conf.d/{{/newmariadb}}

wo/core/mysql.py

@@ -88,7 +88,7 @@ class WOMysql():
         finally:
             connection.close()
 
-    def backupAll(self):
+    def backupAll(self, fulldump=False):
         import subprocess
         try:
             Log.info(self, "Backing up database at location: "
@@ -98,24 +98,47 @@ class WOMysql():
                 Log.debug(self, 'Creating directory'
                           '/var/lib/wo-backup/mysql')
                 os.makedirs('/var/lib/wo-backup/mysql')
-
+            if not fulldump:
-            db = subprocess.check_output(["/usr/bin/mysql "
+                db = subprocess.check_output(
-                                          "-Bse \'show databases\'"],
+                    ["/usr/bin/mysql "
-                                         universal_newlines=True,
+                     "-Bse \'show databases\'"],
-                                         shell=True).split('\n')
+                    universal_newlines=True,
-            for dbs in db:
+                    shell=True).split('\n')
-                if dbs == "":
+                for dbs in db:
-                    continue
+                    if dbs == "":
-                Log.info(self, "Backing up {0} database".format(dbs))
+                        continue
+                    Log.info(self, "Backing up {0} database".format(dbs))
+                    p1 = subprocess.Popen(
+                        "/usr/bin/mysqldump {0} --max_allowed_packet=1024M "
+                        "--single-transaction ".format(dbs),
+                        stdout=subprocess.PIPE,
+                        stderr=subprocess.PIPE, shell=True)
+                    p2 = subprocess.Popen(
+                        "/usr/bin/zstd -T0 -c > "
+                        "/var/lib/wo-backup/mysql/{0}{1}.sql.zst"
+                        .format(dbs, WOVar.wo_date),
+                        stdin=p1.stdout, shell=True)
+                    # Allow p1 to receive a SIGPIPE if p2 exits
+                    p1.stdout.close()
+                    output = p1.stderr.read()
+                    p1.wait()
+                    if p1.returncode == 0:
+                        Log.debug(self, "done")
+                    else:
+                        Log.error(self, output.decode("utf-8"))
+            else:
+                Log.info(self, "Backing up all databases")
                 p1 = subprocess.Popen(
-                    "/usr/bin/mysqldump {0} --max_allowed_packet=1024M "
+                    "/usr/bin/mysqldump --all-databases "
-                    "--single-transaction ".format(dbs),
+                    "--max_allowed_packet=1024M "
-                    stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
+                    "--single-transaction --events",
+                    stdout=subprocess.PIPE,
+                    stderr=subprocess.PIPE, shell=True)
                 p2 = subprocess.Popen(
                     "/usr/bin/zstd -T0 -c > "
-                    "/var/lib/wo-backup/mysql/{0}{1}.sql.zst"
+                    "/var/lib/wo-backup/mysql/fulldump-{0}.sql.zst"
-                    .format(dbs, WOVar.wo_date), stdin=p1.stdout, shell=True)
+                    .format(WOVar.wo_date),
-                # Allow p1 to receive a SIGPIPE if p2 exits
+                    stdin=p1.stdout, shell=True)
                 p1.stdout.close()
                 output = p1.stderr.read()
                 p1.wait()
@@ -123,6 +146,7 @@ class WOMysql():
                     Log.debug(self, "done")
                 else:
                     Log.error(self, output.decode("utf-8"))
+
         except Exception as e:
             Log.error(self, "Error: process exited with status %s"
                             % e)

wo/core/variables.py

@@ -14,7 +14,7 @@ class WOVar():
     """Intialization of core variables"""
 
     # WordOps version
-    wo_version = "3.12.4"
+    wo_version = "3.13.0"
     # WordOps packages versions
     wo_wp_cli = "2.4.0"
     wo_adminer = "4.7.5"
@@ -83,7 +83,7 @@ class WOVar():
         while not match(r"^[A-Za-z0-9\.\+_-]+@[A-Za-z0-9\._-]+\.[a-zA-Z]*$",
                         wo_email):
             print("Whoops, seems like you made a typo - "
-                  "the e-mailaddress is invalid...")
+                  "the e-mail address is invalid...")
             wo_email = input("Enter your email: ")
 
         git.config("--global", "user.name", "{0}".format(wo_user))
@@ -166,7 +166,7 @@ class WOVar():
         else:
             mariadb_ver = '10.3'
     else:
-        mariadb_ver = '10.3'
+        mariadb_ver = '10.5'
         if wo_platform_codename == 'jessie':
             wo_mysql = wo_mysql + ["python3-mysql.connector"]
         else:
@@ -185,7 +185,7 @@ class WOVar():
     # APT repositories
     wo_mysql_repo = ("deb [arch=amd64,ppc64el] "
                      "http://mariadb.mirrors.ovh.net/MariaDB/repo/"
-                     "10.3/{distro} {codename} main"
+                     "10.5/{distro} {codename} main"
                      .format(distro=wo_distro,
                              codename=wo_platform_codename))
     if wo_distro == 'ubuntu':