From 2d21ba82e6c97274261ffeed7ef5661f785885c9 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 28 Oct 2019 14:53:15 +0100 Subject: [PATCH] Fix wo clean issue * add `wo update --mainline` * move mariadb tweaks into wo --- CHANGELOG.md | 4 +- install | 84 ++++++++++++++++-------------------- wo/cli/plugins/clean.py | 3 +- wo/cli/plugins/stack_pref.py | 11 +++++ wo/cli/plugins/update.py | 57 ++++++++++-------------- wo/core/download.py | 24 +++++++++-- 6 files changed, 94 insertions(+), 89 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a5c1d2..e67e8bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), #### Changed - Sysctl tweaks are applied during stack install and removed from install script -- Nginx systemd tweaks during Nginx install/upgrade and removed from install script +- Nginx & MariaDB systemd tweaks are removed from install script and applied during stacks install/upgrade - Initial creation of .gitconfig is displayed the first time you run the command `wo` - Added `/var/lib/php/sessions/` to open_basedir to allow php sessions storage - WordOps now check if a repository already exist before trying to adding it again. @@ -30,6 +30,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), #### Fixed - Incorrect PHP-FPM log path is `wo log` +- force-ssl.conf not removed after removing a site +- `wo clean --opcache` not working with invalid SSL certificate ### v3.9.9.4 - 2019-10-18 diff --git a/install b/install index bef0b18..a65e36c 100755 --- a/install +++ b/install @@ -9,7 +9,7 @@ # ------------------------------------------------------------------------- # wget -qO wo wops.cc && sudo bash wo # ------------------------------------------------------------------------- -# Version 3.10.0 - 2019-10-25 +# Version 3.10.0 - 2019-10-28 # ------------------------------------------------------------------------- # CONTENTS @@ -70,6 +70,9 @@ while [ "$#" -gt 0 ]; do wo_travis="y" wo_force_install="y" ;; + --mainline | --beta) + wo_branch="mainline" + ;; -s | --silent) wo_force_install="y" ;; @@ -102,7 +105,7 @@ export LC_ALL='C.UTF-8' # check if a command exist command_exists() { - command -v "$@" > /dev/null 2>&1 + command -v "$@" >/dev/null 2>&1 } # run functions and exit on failure @@ -110,7 +113,7 @@ _run() { if [ -n "$2" ]; then echo -ne "${TPUT_ECHO}${2}${TPUT_RESET}\t" fi - if ! { "$1" >> "$wo_install_log" 2>&1; }; then + if ! { "$1" >>"$wo_install_log" 2>&1; }; then if [ -n "$2" ]; then echo -e "${TPUT_FAIL}[KO]${TPUT_RESET}" fi @@ -205,13 +208,13 @@ wo_install_dep() { # install dependencies apt-get -option=Dpkg::options::=--force-confmiss --option=Dpkg::options::=--force-confold --assume-yes install \ build-essential curl gzip python3-pip python3-wheel python3-apt python3-setuptools python3-dev sqlite3 git tar software-properties-common pigz \ - gnupg2 cron ccze rsync apt-transport-https tree haveged ufw unattended-upgrades tzdata ntp > /dev/null 2>&1 + gnupg2 cron ccze rsync apt-transport-https tree haveged ufw unattended-upgrades tzdata ntp >/dev/null 2>&1 curl -sL https://download.opensuse.org/repositories/home:/virtubox:/WordOps/xUbuntu_18.04/Release.key | apt-key add - else # install dependencies apt-get -option=Dpkg::options::=--force-confmiss --option=Dpkg::options::=--force-confold --assume-yes install \ build-essential curl gzip dirmngr sudo python3-pip python3-wheel python3-apt python3-setuptools python3-dev ca-certificates sqlite3 git tar \ - software-properties-common pigz apt-transport-https gnupg2 cron ccze rsync tree haveged ufw unattended-upgrades tzdata ntp > /dev/null 2>&1 + software-properties-common pigz apt-transport-https gnupg2 cron ccze rsync tree haveged ufw unattended-upgrades tzdata ntp >/dev/null 2>&1 # add php repository gpg key [ -d /etc/apt/trusted.gpg.d ] && { wget -qO /etc/apt/trusted.gpg.d/php.gpg https://packages.sury.org/php/apt.gpg; } # add nginx repository gpg key @@ -248,17 +251,16 @@ wo_sync_db() { # Switching from EE -> WO ### if [ ! -f /var/lib/wo/dbase.db ]; then - # Create the WordOps folder - mkdir -p /var/lib/wo if [ -f /var/lib/ee/ee.db ]; then + # Make a backup of the EasyEngine database cp /var/lib/ee/ee.db /var/lib/wo/dbase-ee.db # Copy ee database cp /var/lib/ee/ee.db /var/lib/wo/dbase.db else - if [ -d /etc/nginx/sites-available ]; then + if [ -d /etc/nginx/sites-available ] && [ -d /var/www ]; then # Create an empty database for WordOps echo "CREATE TABLE sites ( @@ -423,8 +425,11 @@ wo_install_acme_sh() { # WordOps install wo_install() { - - python3 -m pip install --upgrade wordops + if [ "$wo_branch" = "master" ]; then + python3 -m pip install --upgrade wordops + else + python3 -m pip install -U "git+git://github.com/WordOps/WordOps.git@$wo_branch#egg=wordops" + fi cp -rf /usr/local/lib/python3.*/dist-packages/usr/* /usr/ cp -rn /usr/local/lib/python3.*/dist-packages/etc/* /etc/ cp -f /usr/local/lib/python3.*/dist-packages/etc/bash_completion.d/wo_auto.rc /etc/bash_completion.d/wo_auto.rc @@ -441,7 +446,7 @@ wo_travis_install() { python3 setup.py sdist bdist_wheel python3 -m pip install --upgrade dist/*.whl else - python3 -m pip install -U git+git://github.com/WordOps/WordOps.git@$wo_branch #egg=wordops + python3 -m pip install -U "git+git://github.com/WordOps/WordOps.git@$wo_branch#egg=wordops" fi cp -rf /usr/local/lib/python3.*/dist-packages/usr/* /usr/ cp -rn /usr/local/lib/python3.*/dist-packages/etc/* /etc/ @@ -586,16 +591,7 @@ wo_remove_ee_cron() { } wo_domain_suffix() { - _curl https://raw.githubusercontent.com/publicsuffix/list/master/public_suffix_list.dat | sed '/^\/\//d' | sed '/^$/d' | sed 's/^\s+//g' > /var/lib/wo/public_suffix_list.dat -} - -wo_mariadb_tweak() { - # increase mariadb open_files_limit - if [ -d /etc/systemd/system/mariadb.service.d ] && [ ! -f /etc/systemd/system/mariadb.service.d/limits.conf ]; then - echo -e '[Service]\nLimitNOFILE=500000' > /etc/systemd/system/mariadb.service.d/limits.conf - systemctl daemon-reload - service mysql restart - fi + _curl https://raw.githubusercontent.com/publicsuffix/list/master/public_suffix_list.dat | sed '/^\/\//d' | sed '/^$/d' | sed 's/^\s+//g' >/var/lib/wo/public_suffix_list.dat } wo_clean() { @@ -604,33 +600,36 @@ wo_clean() { } wo_uninstall() { + if { python3 -m pip list | grep -q "wordops" >/dev/null 2>&1; }; then + python3 -m pip uninstall wordops -y + fi rm -rf /usr/local/lib/python3.*/dist-packages/{pystache-*,cement-2.*,wo-*,wordops-*} /usr/local/bin/wo /etc/bash_completion.d/wo_auto.rc /var/lib/wo /etc/wo /usr/lib/wo/templates } wo_cheat_install() { - curl -sL https://cht.sh/:cht.sh > /usr/local/bin/cht.sh + curl -sL https://cht.sh/:cht.sh >/usr/local/bin/cht.sh chmod +x /usr/local/bin/cht.sh [ ! -h /usr/local/bin/cheat ] && { rm -f /usr/local/bin/cheat ln -s /usr/local/bin/cht.sh /usr/local/bin/cheat } - curl -sL https://cheat.sh/:bash_completion > /etc/bash_completion.d/cht.sh + curl -sL https://cheat.sh/:bash_completion >/etc/bash_completion.d/cht.sh } wo_clean_repo() { # remove old EasyEngine Nginx repository if [ -f /etc/apt/sources.list.d/ee-repo.list ]; then cp -f /etc/apt/sources.list.d/ee-repo.list /etc/apt/sources.list.d/ee-repo.list.save - grep -v "/home:/rtCamp:/EasyEngine" /etc/apt/sources.list.d/ee-repo.list.save > /etc/apt/sources.list.d/ee-repo.list + grep -v "/home:/rtCamp:/EasyEngine" /etc/apt/sources.list.d/ee-repo.list.save >/etc/apt/sources.list.d/ee-repo.list fi if [ -f /etc/apt/sources.list.d/wo-repo.list ]; then local wo_linux_distro wo_linux_distro=$(lsb_release -is) cp -f /etc/apt/sources.list.d/wo-repo.list /etc/apt/sources.list.d/wo-repo.list.save if [ "$wo_linux_distro" = "Ubuntu" ]; then - grep -v "opensuse" /etc/apt/sources.list.d/wo-repo.list.save > /etc/apt/sources.list.d/wo-repo.list + grep -v "opensuse" /etc/apt/sources.list.d/wo-repo.list.save >/etc/apt/sources.list.d/wo-repo.list else - grep -v "/home:/rtCamp:/EasyEngine" /etc/apt/sources.list.d/wo-repo.list.save > /etc/apt/sources.list.d/wo-repo.list + grep -v "/home:/rtCamp:/EasyEngine" /etc/apt/sources.list.d/wo-repo.list.save >/etc/apt/sources.list.d/wo-repo.list fi fi } @@ -643,12 +642,12 @@ wo_init() { if [ -z "$wo_travis" ]; then if ! { - apt-get update --allow-releaseinfo-change -qq > /dev/null 2>&1 + apt-get update --allow-releaseinfo-change -qq >/dev/null 2>&1 }; then - apt-get update -qq > /dev/null 2>&1 + apt-get update -qq >/dev/null 2>&1 fi if ! command_exists curl; then - apt-get -y install curl -qq > /dev/null 2>&1 + apt-get -y install curl -qq >/dev/null 2>&1 fi if ! command_exists lsb_release; then apt-get install lsb-release -qq @@ -676,35 +675,26 @@ wo_init() { # 4 - WO MAIN SETUP ### -### -# 1 - Define variables for later use -### - -# inital setup - +# create required directories wo_dir_init +# install lsb_release, curl and display header wo_init +# define main variables wo_init_variables +# remove old repositories _run wo_clean_repo +# check distribution support wo_check_distro # wo uninstall script if [ "$wo_purge" = "y" ]; then - _run wo_backup_wo "Backing-up WO install" _run wo_uninstall "Uninstalling WordOps" wo_lib_echo "The WordOps backup files can be found in $WO_BACKUP_FILE" exit 0 else - # inital checkup - wo_check_distro # 1 - WO already installed if [ -x /usr/local/bin/wo ]; then - if [ -z "$wo_force_install" ]; then - if { wo -v 2>&1 | grep -q "$wo_version_new"; }; then - wo_lib_error "You already have WordOps $wo_version_new" 1 - fi - fi _run wo_clean # 2 - Migration from EEv3 elif [ -x /usr/local/bin/ee ]; then @@ -716,13 +706,13 @@ else fi _run wo_backup_ee "Backing-up EE install" _run wo_remove_ee_cron "Removing EasyEngine cronjob" - _run wo_sync_db "Syncing WO database" fi _run wo_install_dep "Installing wo dependencies" _run wo_timesync # skip steps if travis if [ -z "$wo_travis" ]; then #_run wo_download "Downloading WordOps" + _run wo_sync_db _run wo_install "Installing WordOps" else _run wo_travis_install "Installing WordOps" @@ -732,16 +722,13 @@ else _run wo_clean_ee "Cleaning previous EasyEngine install" fi _run wo_install_acme_sh - if [ -d /etc/systemd/system/mariadb.service.d ]; then - _run wo_mariadb_tweak - fi _run wo_cheat_install "Running post-install steps" _run wo_domain_suffix _run wo_update_wp_cli _run wo_update_latest _run secure_wo_db - wo sync >> $wo_install_log 2>&1 + wo sync >>$wo_install_log 2>&1 if [ "$ee_migration" = "1" ]; then echo @@ -771,6 +758,7 @@ else echo wo_lib_echo "WordOps Documentation : https://docs.wordops.net" wo_lib_echo "WordOps Community Forum : https://community.wordops.net" + wo_lib_echo "WordOps Community Chat : https://chat.wordops.net" echo wo_lib_echo "Give WordOps a GitHub star : https://github.com/WordOps/WordOps/" echo diff --git a/wo/cli/plugins/clean.py b/wo/cli/plugins/clean.py index a7db5f9..3385876 100644 --- a/wo/cli/plugins/clean.py +++ b/wo/cli/plugins/clean.py @@ -75,8 +75,7 @@ class WOCleanController(CementBaseController): try: Log.info(self, "Cleaning opcache") opgui = requests.get( - "https://127.0.0.1:22222/cache/opcache/opgui.php?reset=1", - verify=False) + "https://127.0.0.1:22222/cache/opcache/opgui.php?reset=1") if opgui.status_code != '200': Log.warn(self, 'Cleaning opcache failed') except Exception as e: diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index d5f9c31..1b85165 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -835,8 +835,19 @@ def post_pref(self, apt_packages, packages, upgrade=False): self, '/etc/mysql/my.cnf', 'my.mustache', data) # replacing default values Log.debug(self, "Tuning MySQL configuration") + if os.path.isdir('/etc/systemd/system/mariadb.service.d'): + if not os.path.isfile( + '/etc/systemd/system/' + 'mariadb.service.d/limits.conf'): + WOFileUtils.textwrite( + self, + '/etc/systemd/system/' + 'mariadb.service.d/limits.conf', + '[Service]\nLimitNOFILE=500000') + WOShellExec.cmd_exec(self, 'systemctl daemon-reload') # set innodb_buffer_pool_instances depending # on the amount of RAM + WOService.stop_service(self, 'mysql') WOFileUtils.mvfile(self, '/var/lib/mysql/ib_logfile0', '/var/lib/mysql/ib_logfile0.bak') diff --git a/wo/cli/plugins/update.py b/wo/cli/plugins/update.py index 6fac40c..606da81 100644 --- a/wo/cli/plugins/update.py +++ b/wo/cli/plugins/update.py @@ -1,7 +1,7 @@ import os import time -import requests + from cement.core.controller import CementBaseController, expose from wo.core.download import WODownload from wo.core.logging import Log @@ -46,36 +46,28 @@ class WOUpdateController(CementBaseController): filename = "woupdate" + time.strftime("%Y%m%d-%H%M%S") wo_current = WOVar.wo_version - try: - wo_github_latest = requests.get( - 'https://api.github.com/repos/WordOps/WordOps/releases/latest', - timeout=(5, 30)).json() - except requests.RequestException: - Log.debug( - self, "Request to GitHub API failed. " - "Switching to Gitea instance") - wo_github_latest = requests.get( - 'https://git.virtubox.net/api/v1/repos/virtubox/WordOps/tags', - timeout=(5, 30)).json() - wo_latest = wo_github_latest[0]["name"] - else: - wo_latest = wo_github_latest["tag_name"] + wo_latest = WODownload.latest_release(self, "WordOps/WordOps") install_args = "" if pargs.mainline or pargs.beta: wo_branch = "mainline" + install_args = install_args + "--mainline " elif pargs.branch: wo_branch = pargs.branch - else: - wo_branch = "master" + install_args = install_args + "-b {0} ".format(wo_branch) if pargs.force: install_args = install_args + "--force " - else: - if not pargs.travis: - if wo_current == wo_latest: - Log.error( - self, "WordOps {0} is already installed" - .format(wo_latest)) + if pargs.travis: + install_args = install_args + "--travis " + wo_branch = "updating-configuration" + + if ((not pargs.force) and (not pargs.travis) and + (not pargs.mainline) and (not pargs.beta) and + (not pargs.branch)): + if wo_current == wo_latest: + Log.error( + self, "WordOps {0} is already installed" + .format(wo_latest)) if not os.path.isdir('/var/lib/wo/tmp'): os.makedirs('/var/lib/wo/tmp') @@ -85,21 +77,18 @@ class WOUpdateController(CementBaseController): "/var/lib/wo/tmp/{0}".format(filename), "update script"]]) - if pargs.travis: - if os.path.isfile('install'): - try: - Log.info(self, "updating WordOps, please wait...") - os.system("/bin/bash install --travis " - "--force") - except OSError as e: - Log.debug(self, str(e)) - Log.error(self, "WordOps update failed !") + if os.path.isfile('install'): + try: + Log.info(self, "updating WordOps, please wait...") + os.system("/bin/bash install --travis") + except OSError as e: + Log.debug(self, str(e)) + Log.error(self, "WordOps update failed !") else: try: Log.info(self, "updating WordOps, please wait...") os.system("/bin/bash /var/lib/wo/tmp/{0} " - "-b {1} {2}".format(filename, - wo_branch, install_args)) + "{1}".format(filename, install_args)) except OSError as e: Log.debug(self, str(e)) Log.error(self, "WordOps update failed !") diff --git a/wo/core/download.py b/wo/core/download.py index f81cb42..bd7a725 100644 --- a/wo/core/download.py +++ b/wo/core/download.py @@ -1,6 +1,6 @@ """WordOps download core classes.""" import os -from requests import get, RequestException +import requests from wo.core.logging import Log @@ -11,7 +11,7 @@ class WODownload(): pass def download(self, packages): - """Download packages, packges must be list in format of + """Download packages, packages must be list in format of [url, path, package name]""" for package in packages: url = package[0] @@ -23,15 +23,31 @@ class WODownload(): os.makedirs(directory) Log.info(self, "Downloading {0:20}".format(pkg_name), end=' ') with open(filename, "wb") as out_file: - req = get(url, timeout=(5, 30)) + req = requests.get(url, timeout=(5, 30)) if req.encoding is None: req.encoding = 'utf-8' out_file.write(req.content) Log.info(self, "{0}".format("[" + Log.ENDC + "Done" + Log.OKBLUE + "]")) - except RequestException as e: + except requests.RequestException as e: Log.debug(self, "[{err}]".format(err=str(e.reason))) Log.error(self, "Unable to download file, {0}" .format(filename)) return False return 0 + + def latest_release(self, repository): + """Get the latest release number of a GitHub repository.\n + repository format should be: \"user/repo\"""" + try: + req = requests.get( + 'https://api.github.com/repos/{0}/releases/latest' + .format(repository), + timeout=(5, 30)) + except requests.RequestException as e: + Log.debug(self, str(e)) + Log.error(self, "Unable to query GitHub API") + else: + github_json = req.json() + github_release = github_json["tag_name"] + return github_release