Fix wo clean issue

* add `wo update --mainline`
* move mariadb tweaks into wo

CHANGELOG.md

@@ -18,7 +18,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 #### Changed
 
 - Sysctl tweaks are applied during stack install and removed from install script
-- Nginx systemd tweaks during Nginx install/upgrade and removed from install script
+- Nginx & MariaDB systemd tweaks are removed from install script and applied during stacks install/upgrade
 - Initial creation of .gitconfig is displayed the first time you run the command `wo`
 - Added `/var/lib/php/sessions/` to open_basedir to allow php sessions storage
 - WordOps now check if a repository already exist before trying to adding it again.
@@ -30,6 +30,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 #### Fixed
 
 - Incorrect PHP-FPM log path is `wo log`
+- force-ssl.conf not removed after removing a site
+- `wo clean --opcache` not working with invalid SSL certificate
 
 
 ### v3.9.9.4 -  2019-10-18

install

@@ -9,7 +9,7 @@
 # -------------------------------------------------------------------------
 # wget -qO wo wops.cc && sudo bash wo
 # -------------------------------------------------------------------------
-# Version 3.10.0 - 2019-10-25
+# Version 3.10.0 - 2019-10-28
 # -------------------------------------------------------------------------
 
 # CONTENTS
@@ -70,6 +70,9 @@ while [ "$#" -gt 0 ]; do
         wo_travis="y"
         wo_force_install="y"
         ;;
+    --mainline | --beta)
+        wo_branch="mainline"
+        ;;
     -s | --silent)
         wo_force_install="y"
         ;;
@@ -102,7 +105,7 @@ export LC_ALL='C.UTF-8'
 
 # check if a command exist
 command_exists() {
-    command -v "$@" > /dev/null 2>&1
+    command -v "$@" >/dev/null 2>&1
 }
 
 # run functions and exit on failure
@@ -110,7 +113,7 @@ _run() {
     if [ -n "$2" ]; then
         echo -ne "${TPUT_ECHO}${2}${TPUT_RESET}\t"
     fi
-    if ! { "$1" >> "$wo_install_log" 2>&1; }; then
+    if ! { "$1" >>"$wo_install_log" 2>&1; }; then
         if [ -n "$2" ]; then
             echo -e "${TPUT_FAIL}[KO]${TPUT_RESET}"
         fi
@@ -205,13 +208,13 @@ wo_install_dep() {
         # install dependencies
         apt-get -option=Dpkg::options::=--force-confmiss --option=Dpkg::options::=--force-confold --assume-yes install \
             build-essential curl gzip python3-pip python3-wheel python3-apt python3-setuptools python3-dev sqlite3 git tar software-properties-common pigz \
-            gnupg2 cron ccze rsync apt-transport-https tree haveged ufw unattended-upgrades tzdata ntp > /dev/null 2>&1
+            gnupg2 cron ccze rsync apt-transport-https tree haveged ufw unattended-upgrades tzdata ntp >/dev/null 2>&1
         curl -sL https://download.opensuse.org/repositories/home:/virtubox:/WordOps/xUbuntu_18.04/Release.key | apt-key add -
     else
         # install dependencies
         apt-get -option=Dpkg::options::=--force-confmiss --option=Dpkg::options::=--force-confold --assume-yes install \
             build-essential curl gzip dirmngr sudo python3-pip python3-wheel python3-apt python3-setuptools python3-dev ca-certificates sqlite3 git tar \
-            software-properties-common pigz apt-transport-https gnupg2 cron ccze rsync tree haveged ufw unattended-upgrades tzdata ntp > /dev/null 2>&1
+            software-properties-common pigz apt-transport-https gnupg2 cron ccze rsync tree haveged ufw unattended-upgrades tzdata ntp >/dev/null 2>&1
         # add php repository gpg key
         [ -d /etc/apt/trusted.gpg.d ] && { wget -qO /etc/apt/trusted.gpg.d/php.gpg https://packages.sury.org/php/apt.gpg; }
         # add nginx repository gpg key
@@ -248,17 +251,16 @@ wo_sync_db() {
     # Switching from EE -> WO
     ###
     if [ ! -f /var/lib/wo/dbase.db ]; then
-        # Create the WordOps folder
-        mkdir -p /var/lib/wo
 
         if [ -f /var/lib/ee/ee.db ]; then
+
             # Make a backup of the EasyEngine database
             cp /var/lib/ee/ee.db /var/lib/wo/dbase-ee.db
 
             # Copy ee database
             cp /var/lib/ee/ee.db /var/lib/wo/dbase.db
         else
-            if [ -d /etc/nginx/sites-available ]; then
+            if [ -d /etc/nginx/sites-available ] && [ -d /var/www ]; then
 
                 # Create an empty database for WordOps
                 echo "CREATE TABLE sites (
@@ -423,8 +425,11 @@ wo_install_acme_sh() {
 
 # WordOps install
 wo_install() {
-
-    python3 -m pip install --upgrade wordops
+    if [ "$wo_branch" = "master" ]; then
+        python3 -m pip install --upgrade wordops
+    else
+        python3 -m pip install -U "git+git://github.com/WordOps/WordOps.git@$wo_branch#egg=wordops"
+    fi
     cp -rf /usr/local/lib/python3.*/dist-packages/usr/* /usr/
     cp -rn /usr/local/lib/python3.*/dist-packages/etc/* /etc/
     cp -f /usr/local/lib/python3.*/dist-packages/etc/bash_completion.d/wo_auto.rc /etc/bash_completion.d/wo_auto.rc
@@ -441,7 +446,7 @@ wo_travis_install() {
         python3 setup.py sdist bdist_wheel
         python3 -m pip install --upgrade dist/*.whl
     else
-        python3 -m pip install -U git+git://github.com/WordOps/WordOps.git@$wo_branch #egg=wordops
+        python3 -m pip install -U "git+git://github.com/WordOps/WordOps.git@$wo_branch#egg=wordops"
     fi
     cp -rf /usr/local/lib/python3.*/dist-packages/usr/* /usr/
     cp -rn /usr/local/lib/python3.*/dist-packages/etc/* /etc/
@@ -586,16 +591,7 @@ wo_remove_ee_cron() {
 }
 
 wo_domain_suffix() {
-    _curl https://raw.githubusercontent.com/publicsuffix/list/master/public_suffix_list.dat | sed '/^\/\//d' | sed '/^$/d' | sed 's/^\s+//g' > /var/lib/wo/public_suffix_list.dat
-}
-
-wo_mariadb_tweak() {
-    # increase mariadb open_files_limit
-    if [ -d /etc/systemd/system/mariadb.service.d ] && [ ! -f /etc/systemd/system/mariadb.service.d/limits.conf ]; then
-        echo -e '[Service]\nLimitNOFILE=500000' > /etc/systemd/system/mariadb.service.d/limits.conf
-        systemctl daemon-reload
-        service mysql restart
-    fi
+    _curl https://raw.githubusercontent.com/publicsuffix/list/master/public_suffix_list.dat | sed '/^\/\//d' | sed '/^$/d' | sed 's/^\s+//g' >/var/lib/wo/public_suffix_list.dat
 }
 
 wo_clean() {
@@ -604,33 +600,36 @@ wo_clean() {
 }
 
 wo_uninstall() {
+    if { python3 -m pip list | grep -q "wordops" >/dev/null 2>&1; }; then
+        python3 -m pip uninstall wordops -y
+    fi
     rm -rf /usr/local/lib/python3.*/dist-packages/{pystache-*,cement-2.*,wo-*,wordops-*} /usr/local/bin/wo /etc/bash_completion.d/wo_auto.rc /var/lib/wo /etc/wo /usr/lib/wo/templates
 }
 
 wo_cheat_install() {
-    curl -sL https://cht.sh/:cht.sh > /usr/local/bin/cht.sh
+    curl -sL https://cht.sh/:cht.sh >/usr/local/bin/cht.sh
     chmod +x /usr/local/bin/cht.sh
     [ ! -h /usr/local/bin/cheat ] && {
         rm -f /usr/local/bin/cheat
         ln -s /usr/local/bin/cht.sh /usr/local/bin/cheat
     }
-    curl -sL https://cheat.sh/:bash_completion > /etc/bash_completion.d/cht.sh
+    curl -sL https://cheat.sh/:bash_completion >/etc/bash_completion.d/cht.sh
 }
 
 wo_clean_repo() {
     # remove old EasyEngine Nginx repository
     if [ -f /etc/apt/sources.list.d/ee-repo.list ]; then
         cp -f /etc/apt/sources.list.d/ee-repo.list /etc/apt/sources.list.d/ee-repo.list.save
-        grep -v "/home:/rtCamp:/EasyEngine" /etc/apt/sources.list.d/ee-repo.list.save > /etc/apt/sources.list.d/ee-repo.list
+        grep -v "/home:/rtCamp:/EasyEngine" /etc/apt/sources.list.d/ee-repo.list.save >/etc/apt/sources.list.d/ee-repo.list
     fi
     if [ -f /etc/apt/sources.list.d/wo-repo.list ]; then
         local wo_linux_distro
         wo_linux_distro=$(lsb_release -is)
         cp -f /etc/apt/sources.list.d/wo-repo.list /etc/apt/sources.list.d/wo-repo.list.save
         if [ "$wo_linux_distro" = "Ubuntu" ]; then
-            grep -v "opensuse" /etc/apt/sources.list.d/wo-repo.list.save > /etc/apt/sources.list.d/wo-repo.list
+            grep -v "opensuse" /etc/apt/sources.list.d/wo-repo.list.save >/etc/apt/sources.list.d/wo-repo.list
         else
-            grep -v "/home:/rtCamp:/EasyEngine" /etc/apt/sources.list.d/wo-repo.list.save > /etc/apt/sources.list.d/wo-repo.list
+            grep -v "/home:/rtCamp:/EasyEngine" /etc/apt/sources.list.d/wo-repo.list.save >/etc/apt/sources.list.d/wo-repo.list
         fi
     fi
 }
@@ -643,12 +642,12 @@ wo_init() {
 
     if [ -z "$wo_travis" ]; then
         if ! {
-            apt-get update --allow-releaseinfo-change -qq > /dev/null 2>&1
+            apt-get update --allow-releaseinfo-change -qq >/dev/null 2>&1
         }; then
-            apt-get update -qq > /dev/null 2>&1
+            apt-get update -qq >/dev/null 2>&1
         fi
         if ! command_exists curl; then
-            apt-get -y install curl -qq > /dev/null 2>&1
+            apt-get -y install curl -qq >/dev/null 2>&1
         fi
         if ! command_exists lsb_release; then
             apt-get install lsb-release -qq
@@ -676,35 +675,26 @@ wo_init() {
 # 4 - WO MAIN SETUP
 ###
 
-###
-# 1 - Define variables for later use
-###
-
-# inital setup
-
+# create required directories
 wo_dir_init
+# install lsb_release, curl and display header
 wo_init
+# define main variables
 wo_init_variables
+# remove old repositories
 _run wo_clean_repo
+# check distribution support
 wo_check_distro
 
 # wo uninstall script
 if [ "$wo_purge" = "y" ]; then
-
     _run wo_backup_wo "Backing-up WO install"
     _run wo_uninstall "Uninstalling WordOps"
     wo_lib_echo "The WordOps backup files can be found in $WO_BACKUP_FILE"
     exit 0
 else
-    # inital checkup
-    wo_check_distro
     # 1 - WO already installed
     if [ -x /usr/local/bin/wo ]; then
-        if [ -z "$wo_force_install" ]; then
-            if { wo -v 2>&1 | grep -q "$wo_version_new"; }; then
-                wo_lib_error "You already have WordOps $wo_version_new" 1
-            fi
-        fi
         _run wo_clean
     # 2 - Migration from EEv3
     elif [ -x /usr/local/bin/ee ]; then
@@ -716,13 +706,13 @@ else
         fi
         _run wo_backup_ee "Backing-up EE install"
         _run wo_remove_ee_cron "Removing EasyEngine cronjob"
-        _run wo_sync_db "Syncing WO database"
     fi
     _run wo_install_dep "Installing wo dependencies"
     _run wo_timesync
     # skip steps if travis
     if [ -z "$wo_travis" ]; then
         #_run wo_download "Downloading WordOps"
+        _run wo_sync_db
         _run wo_install "Installing WordOps"
     else
         _run wo_travis_install "Installing WordOps"
@@ -732,16 +722,13 @@ else
         _run wo_clean_ee "Cleaning previous EasyEngine install"
     fi
     _run wo_install_acme_sh
-    if [ -d /etc/systemd/system/mariadb.service.d ]; then
-        _run wo_mariadb_tweak
-    fi
     _run wo_cheat_install "Running post-install steps"
     _run wo_domain_suffix
     _run wo_update_wp_cli
     _run wo_update_latest
     _run secure_wo_db
 
-    wo sync >> $wo_install_log 2>&1
+    wo sync >>$wo_install_log 2>&1
 
     if [ "$ee_migration" = "1" ]; then
         echo
@@ -771,6 +758,7 @@ else
     echo
     wo_lib_echo "WordOps Documentation : https://docs.wordops.net"
     wo_lib_echo "WordOps Community Forum : https://community.wordops.net"
+    wo_lib_echo "WordOps Community Chat : https://chat.wordops.net"
     echo
     wo_lib_echo "Give WordOps a GitHub star : https://github.com/WordOps/WordOps/"
     echo

wo/cli/plugins/clean.py

@@ -75,8 +75,7 @@ class WOCleanController(CementBaseController):
         try:
             Log.info(self, "Cleaning opcache")
             opgui = requests.get(
-                "https://127.0.0.1:22222/cache/opcache/opgui.php?reset=1",
-                verify=False)
+                "https://127.0.0.1:22222/cache/opcache/opgui.php?reset=1")
             if opgui.status_code != '200':
                 Log.warn(self, 'Cleaning opcache failed')
         except Exception as e:

wo/cli/plugins/stack_pref.py

@@ -835,8 +835,19 @@ def post_pref(self, apt_packages, packages, upgrade=False):
                     self, '/etc/mysql/my.cnf', 'my.mustache', data)
                 # replacing default values
                 Log.debug(self, "Tuning MySQL configuration")
+                if os.path.isdir('/etc/systemd/system/mariadb.service.d'):
+                    if not os.path.isfile(
+                            '/etc/systemd/system/'
+                            'mariadb.service.d/limits.conf'):
+                        WOFileUtils.textwrite(
+                            self,
+                            '/etc/systemd/system/'
+                            'mariadb.service.d/limits.conf',
+                            '[Service]\nLimitNOFILE=500000')
+                        WOShellExec.cmd_exec(self, 'systemctl daemon-reload')
                 # set innodb_buffer_pool_instances depending
                 # on the amount of RAM
+
                 WOService.stop_service(self, 'mysql')
                 WOFileUtils.mvfile(self, '/var/lib/mysql/ib_logfile0',
                                    '/var/lib/mysql/ib_logfile0.bak')

wo/cli/plugins/update.py

@@ -1,7 +1,7 @@
 import os
 import time
 
-import requests
+
 from cement.core.controller import CementBaseController, expose
 from wo.core.download import WODownload
 from wo.core.logging import Log
@@ -46,36 +46,28 @@ class WOUpdateController(CementBaseController):
         filename = "woupdate" + time.strftime("%Y%m%d-%H%M%S")
 
         wo_current = WOVar.wo_version
-        try:
-            wo_github_latest = requests.get(
-                'https://api.github.com/repos/WordOps/WordOps/releases/latest',
-                timeout=(5, 30)).json()
-        except requests.RequestException:
-            Log.debug(
-                self, "Request to GitHub API failed. "
-                "Switching to Gitea instance")
-            wo_github_latest = requests.get(
-                'https://git.virtubox.net/api/v1/repos/virtubox/WordOps/tags',
-                timeout=(5, 30)).json()
-            wo_latest = wo_github_latest[0]["name"]
-        else:
-            wo_latest = wo_github_latest["tag_name"]
+        wo_latest = WODownload.latest_release(self, "WordOps/WordOps")
 
         install_args = ""
         if pargs.mainline or pargs.beta:
             wo_branch = "mainline"
+            install_args = install_args + "--mainline "
         elif pargs.branch:
             wo_branch = pargs.branch
-        else:
-            wo_branch = "master"
+            install_args = install_args + "-b {0} ".format(wo_branch)
         if pargs.force:
             install_args = install_args + "--force "
-        else:
-            if not pargs.travis:
-                if wo_current == wo_latest:
-                    Log.error(
-                        self, "WordOps {0} is already installed"
-                        .format(wo_latest))
+        if pargs.travis:
+            install_args = install_args + "--travis "
+            wo_branch = "updating-configuration"
+
+        if ((not pargs.force) and (not pargs.travis) and
+            (not pargs.mainline) and (not pargs.beta) and
+                (not pargs.branch)):
+            if wo_current == wo_latest:
+                Log.error(
+                    self, "WordOps {0} is already installed"
+                    .format(wo_latest))
 
         if not os.path.isdir('/var/lib/wo/tmp'):
             os.makedirs('/var/lib/wo/tmp')
@@ -85,21 +77,18 @@ class WOUpdateController(CementBaseController):
                                     "/var/lib/wo/tmp/{0}".format(filename),
                                     "update script"]])
 
-        if pargs.travis:
-            if os.path.isfile('install'):
-                try:
-                    Log.info(self, "updating WordOps, please wait...")
-                    os.system("/bin/bash install --travis "
-                              "--force")
-                except OSError as e:
-                    Log.debug(self, str(e))
-                    Log.error(self, "WordOps update failed !")
+        if os.path.isfile('install'):
+            try:
+                Log.info(self, "updating WordOps, please wait...")
+                os.system("/bin/bash install --travis")
+            except OSError as e:
+                Log.debug(self, str(e))
+                Log.error(self, "WordOps update failed !")
         else:
             try:
                 Log.info(self, "updating WordOps, please wait...")
                 os.system("/bin/bash /var/lib/wo/tmp/{0} "
-                          "-b {1} {2}".format(filename,
-                                              wo_branch, install_args))
+                          "{1}".format(filename, install_args))
             except OSError as e:
                 Log.debug(self, str(e))
                 Log.error(self, "WordOps update failed !")

wo/core/download.py

@@ -1,6 +1,6 @@
 """WordOps download core classes."""
 import os
-from requests import get, RequestException
+import requests
 
 from wo.core.logging import Log
 
@@ -11,7 +11,7 @@ class WODownload():
         pass
 
     def download(self, packages):
-        """Download packages, packges must be list in format of
+        """Download packages, packages must be list in format of
         [url, path, package name]"""
         for package in packages:
             url = package[0]
@@ -23,15 +23,31 @@ class WODownload():
                     os.makedirs(directory)
                 Log.info(self, "Downloading {0:20}".format(pkg_name), end=' ')
                 with open(filename, "wb") as out_file:
-                    req = get(url, timeout=(5, 30))
+                    req = requests.get(url, timeout=(5, 30))
                     if req.encoding is None:
                         req.encoding = 'utf-8'
                     out_file.write(req.content)
                 Log.info(self, "{0}".format("[" + Log.ENDC + "Done"
                                             + Log.OKBLUE + "]"))
-            except RequestException as e:
+            except requests.RequestException as e:
                 Log.debug(self, "[{err}]".format(err=str(e.reason)))
                 Log.error(self, "Unable to download file, {0}"
                           .format(filename))
                 return False
         return 0
+
+    def latest_release(self, repository):
+        """Get the latest release number of a GitHub repository.\n
+        repository format should be: \"user/repo\""""
+        try:
+            req = requests.get(
+                'https://api.github.com/repos/{0}/releases/latest'
+                .format(repository),
+                timeout=(5, 30))
+        except requests.RequestException as e:
+            Log.debug(self, str(e))
+            Log.error(self, "Unable to query GitHub API")
+        else:
+            github_json = req.json()
+            github_release = github_json["tag_name"]
+        return github_release