From 6eceb5ed35bbaf08870e399dc2b177a0ee456f5c Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 21 Oct 2019 11:37:19 +0200 Subject: [PATCH 01/79] Update install for pip --- install | 33 ++++++++++++++++++++++----------- setup.py | 2 +- wo/cli/plugins/clean.py | 7 +++++-- 3 files changed, 28 insertions(+), 14 deletions(-) diff --git a/install b/install index 6a07e1d..79c46d3 100755 --- a/install +++ b/install @@ -30,7 +30,7 @@ TPUT_ECHO=$(tput setaf 4) wo_lib_echo() { - echo "${TPUT_ECHO}${*}${TPUT_RESET}" + echo -e "${TPUT_ECHO}${*}${TPUT_RESET}" } wo_lib_echo_info() { @@ -111,10 +111,16 @@ command_exists() { # run functions and exit on failure _run() { if [ -n "$2" ]; then - wo_lib_echo "$2" + wo_lib_echo "$2\r" fi if ! { "$1" >> "$wo_install_log" 2>&1; }; then - exit 1 + if [ -n "$2" ]; then + wo_lib_echo_fail "$2" + fi + else + if [ -n "$2" ]; then + wo_lib_echo "$2 [OK]" + fi fi } @@ -221,6 +227,8 @@ wo_install_dep() { if [ ! -f /etc/apt/apt.conf.d/20auto-upgrades ]; then cp /usr/share/unattended-upgrades/20auto-upgrades /etc/apt/apt.conf.d/20auto-upgrades fi + # upgrade pip + python3 -m pip install --upgrade pip } @@ -447,8 +455,9 @@ wo_download() { # WordOps install wo_install() { - cd /var/lib/wo/tmp/WordOps-install || exit 1 - python3 setup.py install + python3 -m pip install --upgrade wordops + cp -rn /usr/local/lib/python3.*/dist-packages/usr/* /usr/ + cp -rn /usr/local/lib/python3.*/dist-packages/etc/* /etc/ } @@ -458,11 +467,13 @@ wo_travis_install() { if [ "$wo_force_install" = "y" ]; then [ ! -f "$HOME/.gitconfig" ] && { bash -c 'echo -e "[user]\n\tname = $USER\n\temail = root@$HOSTNAME.local" > $HOME/.gitconfig'; } fi - - if [ -f "$HOME/.gitconfig" ]; then - # install and redirect log to not print python package install - python3 setup.py install + if [ -d ./dist ]; then + rm -rf dist fi + python3 setup.py sdist bdist_wheel + python3 -m pip install --upgrade dist/*.whl + cp -rn /usr/local/lib/python3.*/dist-packages/usr/* /usr/ + cp -rn /usr/local/lib/python3.*/dist-packages/etc/* /etc/ } @@ -549,7 +560,7 @@ wo_upgrade_nginx() { systemctl start nginx fi [ -f /var/lib/wo/tmp/nginx-block ] && { mv /var/lib/wo/tmp/nginx-block /etc/apt/preferences.d/nginx-block; } - + return 0 } wo_update_latest() { @@ -665,7 +676,7 @@ wo_nginx_tweak() { } wo_clean() { - rm -rf /usr/local/lib/python3.*/dist-packages/wo-* /usr/local/lib/python3.*/dist-packages/wordops-* + rm -rf /usr/local/lib/python3.*/dist-packages/wo-*.egg /usr/local/lib/python3.*/dist-packages/wordops-*.egg } diff --git a/setup.py b/setup.py index 7c960cf..786954d 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ if os.geteuid() == 0: setup(name='wordops', version='3.9.9.4', - description='WordPress & server administration toolset', + description='An essential toolset that eases server administration', long_description=LONG, long_description_content_type='text/markdown', classifiers=[ diff --git a/wo/cli/plugins/clean.py b/wo/cli/plugins/clean.py index b866fc4..5c949e0 100644 --- a/wo/cli/plugins/clean.py +++ b/wo/cli/plugins/clean.py @@ -2,6 +2,7 @@ import os import urllib.request +import requests from cement.core.controller import CementBaseController, expose @@ -74,8 +75,10 @@ class WOCleanController(CementBaseController): def clean_opcache(self): try: Log.info(self, "Cleaning opcache") - urllib.request.urlopen("https://127.0.0.1:22222/cache" - "/opcache/opgui.php?reset=1").read() + opgui = requests.get( + "https://127.0.0.1:22222/cache/opcache/opgui.php?reset=1") + if opgui.status_code != '200': + Log.warn(self, 'Cleaning opcache failed') except Exception as e: Log.debug(self, "{0}".format(e)) Log.debug(self, "Unable hit url, " From 57130341e61ba6e8e04d97c67b3cbc41b01ea81d Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 21 Oct 2019 12:41:31 +0200 Subject: [PATCH 02/79] Improve install logging --- install | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/install b/install index 79c46d3..1c36c3c 100755 --- a/install +++ b/install @@ -27,10 +27,11 @@ TPUT_RESET=$(tput sgr0) TPUT_FAIL=$(tput setaf 1) TPUT_INFO=$(tput setaf 7) TPUT_ECHO=$(tput setaf 4) +TPUT_OK=$(tput setaf 2) wo_lib_echo() { - echo -e "${TPUT_ECHO}${*}${TPUT_RESET}" + echo "${TPUT_ECHO}${*}${TPUT_RESET}" } wo_lib_echo_info() { @@ -111,15 +112,15 @@ command_exists() { # run functions and exit on failure _run() { if [ -n "$2" ]; then - wo_lib_echo "$2\r" + echo -ne "${TPUT_ECHO}${2}${TPUT_RESET}\t" fi if ! { "$1" >> "$wo_install_log" 2>&1; }; then if [ -n "$2" ]; then - wo_lib_echo_fail "$2" + echo -e "${TPUT_FAIL}[KO]${TPUT_RESET}" fi else if [ -n "$2" ]; then - wo_lib_echo "$2 [OK]" + echo -e "${TPUT_OK}[OK]${TPUT_RESET}" fi fi From 5a0e869c504bd0874117fd29f86bb586095e1f83 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 21 Oct 2019 14:50:42 +0200 Subject: [PATCH 03/79] small color adjustment --- install | 2 +- wo/cli/plugins/clean.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/install b/install index 1c36c3c..1e809f2 100755 --- a/install +++ b/install @@ -120,7 +120,7 @@ _run() { fi else if [ -n "$2" ]; then - echo -e "${TPUT_OK}[OK]${TPUT_RESET}" + echo -e "[${TPUT_OK}OK${TPUT_RESET}]" fi fi diff --git a/wo/cli/plugins/clean.py b/wo/cli/plugins/clean.py index 5c949e0..3385876 100644 --- a/wo/cli/plugins/clean.py +++ b/wo/cli/plugins/clean.py @@ -1,7 +1,6 @@ """Clean Plugin for WordOps.""" import os -import urllib.request import requests from cement.core.controller import CementBaseController, expose From 4d3e6353f74605ca8b95766be6c2e6b81552b585 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Tue, 22 Oct 2019 19:27:21 +0200 Subject: [PATCH 04/79] Update install --- install | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/install b/install index 1e809f2..9b8fdfc 100755 --- a/install +++ b/install @@ -197,11 +197,6 @@ wo_dir_init() { # 2 - Setup the dependencies for installation #### -wo_dist_upgrade() { - # perform server packages upgrade - apt-get dist-upgrade --option=Dpkg::options::=--force-confmiss --option=Dpkg::options::=--force-confold --option=Dpkg::options::=--force-unsafe-io --assume-yes --quiet -} - wo_install_dep() { local wo_linux_distro wo_linux_distro=$(lsb_release -is) @@ -229,7 +224,7 @@ wo_install_dep() { cp /usr/share/unattended-upgrades/20auto-upgrades /etc/apt/apt.conf.d/20auto-upgrades fi # upgrade pip - python3 -m pip install --upgrade pip + python3 -m pip install --upgrade pip setuptools wheel } From 8ae8f0ee7d3c4d6c7ecba075e6066bb698060059 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 23 Oct 2019 00:40:11 +0200 Subject: [PATCH 05/79] Refactor main --- wo/cli/main.py | 79 ++++++++++++++++++++++---------------------------- 1 file changed, 34 insertions(+), 45 deletions(-) diff --git a/wo/cli/main.py b/wo/cli/main.py index dad8f1c..fc80c05 100644 --- a/wo/cli/main.py +++ b/wo/cli/main.py @@ -75,60 +75,49 @@ class WOTestApp(WOApp): class Meta: argv = [] config_files = [] + exit_on_close = True # Define the applicaiton object outside of main, as some libraries might wish # to import it as a global (rather than passing it into another class/func) -app = WOApp() +# app = WOApp() def main(): - try: - global sys - # Default our exit status to 0 (non-error) - code = 0 + with WOApp() as app: + try: + global sys - # if not root...kick out - if not os.geteuid() == 0: - print("\nNon-privileged users cant use WordOps. " - "Switch to root or invoke sudo.\n") - app.close(1) + # if not root...kick out + if not os.geteuid() == 0: + print("\nNon-privileged users cant use WordOps. " + "Switch to root or invoke sudo.\n") + app.close(1) + app.run() + except AssertionError as e: + print("AssertionError => %s" % e.args[0]) + app.exit_code = 1 + except exc.WOError as e: + # Catch our application errors and exit 1 (error) + print(e) + app.exit_code = 1 + except FrameworkError as e: + # Catch framework errors and exit 1 (error) + print('FrameworkError > %s' % e) + app.exit_code = 1 + except CaughtSignal as e: + # Default Cement signals are SIGINT and SIGTERM, exit 0 (non-error) + print('CaughtSignal > %s' % e) + app.exit_code = 0 + finally: + # Print an exception (if it occurred) and --debug was passed + if app.debug: + import sys + import traceback - # Setup the application - app.setup() - - # Dump all arguments into wo log - app.log.debug(sys.argv) - - # Run the application - app.run() - except exc.WOError as e: - # Catch our application errors and exit 1 (error) - code = 1 - print(e) - except CaughtSignal as e: - # Default Cement signals are SIGINT and SIGTERM, exit 0 (non-error) - code = 0 - print(e) - except FrameworkError as e: - # Catch framework errors and exit 1 (error) - code = 1 - print(e) - except Exception as e: - code = 1 - print(e) - finally: - # Print an exception (if it occurred) and --debug was passed - if app.debug: - import sys - import traceback - - exc_type, exc_value, exc_traceback = sys.exc_info() - if exc_traceback is not None: - traceback.print_exc() - - # # Close the application - app.close(code) + exc_type, exc_value, exc_traceback = sys.exc_info() + if exc_traceback is not None: + traceback.print_exc() def get_test_app(**kw): From 2ab250f2ba2f9640d3b844efde331154c4ce537e Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 23 Oct 2019 01:59:10 +0200 Subject: [PATCH 06/79] Linux tweak integration in wo --- install | 45 +------------------------ setup.py | 2 +- wo/cli/plugins/site.py | 3 +- wo/cli/plugins/stack.py | 3 +- wo/cli/plugins/stack_pref.py | 64 ++++++++++++++++++++++++++++++++++++ 5 files changed, 70 insertions(+), 47 deletions(-) diff --git a/install b/install index 9b8fdfc..01380a9 100755 --- a/install +++ b/install @@ -141,9 +141,6 @@ readonly TIME=$(date +"$TIME_FORMAT") readonly NGINX_BACKUP_FILE="/var/lib/wo-backup/nginx-backup.$TIME.tar.gz" readonly EE_BACKUP_FILE="/var/lib/wo-backup/ee-backup.$TIME.tar.gz" readonly WO_BACKUP_FILE="/var/lib/wo-backup/wo-backup.$TIME.tar.gz" -readonly wo_lxc=$(grep "container=lxc" /proc/1/environ) -readonly wo_wsl=$(grep "wsl" /proc/1/environ) -readonly wo_arch="$(uname -m)" if [ -x /usr/local/bin/ee ]; then ee_migration=1 @@ -610,42 +607,6 @@ wo_remove_ee_cron() { } -wo_tweak_kernel() { - local wo_distro_version - wo_distro_version=$(lsb_release -sc) - if [ "$wo_arch" = "x86_64" ] && [ -z "$wo_lxc" ] && [ -z "$wo_wsl" ]; then - rm -f /etc/sysctl.d/60-ubuntu-nginx-web-server.conf - wget -qO /etc/sysctl.d/60-wo-tweaks.conf https://raw.githubusercontent.com/WordOps/WordOps/"$wo_branch"/wo/cli/templates/sysctl.mustache - if [ "$wo_distro_version" = "bionic" ] || [ "$wo_distro_version" = "disco" ] || [ "$wo_distro_version" = "buster" ]; then - modprobe tcp_bbr && echo 'tcp_bbr' >> /etc/modules-load.d/bbr.conf - echo -e '\nnet.ipv4.tcp_congestion_control = bbr\nnet.ipv4.tcp_notsent_lowat = 16384' >> /etc/sysctl.d/60-wo-tweaks.conf - else - modprobe tcp_htcp && echo 'tcp_htcp' >> /etc/modules-load.d/htcp.conf - echo 'net.ipv4.tcp_congestion_control = htcp' >> /etc/sysctl.d/60-wo-tweaks.conf - fi - # apply sysctl tweaks - sysctl -eq -p /etc/sysctl.d/60-wo-tweaks.conf - fi -} - -wo_systemd_tweak() { - - if [ ! -x /opt/wo-kernel.sh ]; then - # download and setup wo-kernel systemd service to apply kernel tweaks for netdata and redis on server startup - wget -qO /opt/wo-kernel.sh https://raw.githubusercontent.com/WordOps/WordOps/updating-configuration/wo/cli/templates/wo-kernel-script.mustache - chmod +x /opt/wo-kernel.sh - wget -qO /lib/systemd/system/wo-kernel.service https://raw.githubusercontent.com/WordOps/WordOps/updating-configuration/wo/cli/templates/wo-kernel-service.mustache - systemctl enable wo-kernel.service - systemctl start wo-kernel.service - fi - - LIMIT_CHECK=$(grep "500000" /etc/security/limits.conf) - if [ -z "$LIMIT_CHECK" ]; then - echo -e "* hard nofile 500000\n* soft nofile 500000\nroot hard nofile 500000\nroot soft nofile 500000\n" >> /etc/security/limits.conf - fi - -} - wo_domain_suffix() { curl -m 10 --retry 3 -sL https://raw.githubusercontent.com/publicsuffix/list/master/public_suffix_list.dat | sed '/^\/\//d' | sed '/^$/d' | sed 's/^\s+//g' > /var/lib/wo/public_suffix_list.dat } @@ -780,7 +741,7 @@ else _run wo_timesync # skip steps if travis if [ -z "$wo_travis" ]; then - _run wo_download "Downloading WordOps" + #_run wo_download "Downloading WordOps" wo_git_config _run wo_install "Installing WordOps" else @@ -791,10 +752,6 @@ else _run wo_clean_ee "Cleaning previous EasyEngine install" fi _run wo_install_acme_sh - _run wo_tweak_kernel "Applying Kernel tweaks" - if [ ! -f /opt/wo-kernel.sh ]; then - _run wo_systemd_tweak "Adding systemd service tweak" - fi if [ -x /usr/sbin/nginx ]; then _run wo_nginx_tweak fi diff --git a/setup.py b/setup.py index 786954d..bc3b692 100644 --- a/setup.py +++ b/setup.py @@ -1,10 +1,10 @@ import glob import os +import sys from setuptools import find_packages, setup - # read the contents of your README file this_directory = os.path.abspath(os.path.dirname(__file__)) with open(os.path.join(this_directory, 'README.md'), encoding='utf-8') as f: diff --git a/wo/cli/plugins/site.py b/wo/cli/plugins/site.py index d293c3b..bbb6b8e 100644 --- a/wo/cli/plugins/site.py +++ b/wo/cli/plugins/site.py @@ -905,8 +905,9 @@ class WOSiteUpdateController(CementBaseController): choices=('on', 'off'), const='on', nargs='?')), (['--ngxblocker'], - dict(help="enable HSTS for site secured with letsencrypt", + dict(help="enable Ultimate Nginx bad bot blocker", action='store' or 'store_const', + choices=('on', 'off'), const='on', nargs='?')), (['--proxy'], dict(help="update to proxy site", nargs='+')), diff --git a/wo/cli/plugins/stack.py b/wo/cli/plugins/stack.py index 8dacd32..de3c606 100644 --- a/wo/cli/plugins/stack.py +++ b/wo/cli/plugins/stack.py @@ -5,7 +5,7 @@ import os from cement.core.controller import CementBaseController, expose from wo.cli.plugins.stack_migrate import WOStackMigrateController -from wo.cli.plugins.stack_pref import post_pref, pre_pref +from wo.cli.plugins.stack_pref import post_pref, pre_pref, pre_stack from wo.cli.plugins.stack_services import WOStackStatusController from wo.cli.plugins.stack_upgrade import WOStackUpgradeController from wo.core.aptget import WOAptGet @@ -485,6 +485,7 @@ class WOStackController(CementBaseController): Log.debug(self, "{0}".format(e)) if (apt_packages) or (packages): + pre_stack(self) if (apt_packages): Log.debug(self, "Calling pre_pref") pre_pref(self, apt_packages) diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index 0127746..18cf67a 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -1362,3 +1362,67 @@ def post_pref(self, apt_packages, packages, upgrade=False): WOShellExec.cmd_exec(self, '/usr/local/sbin/install-ngxblocker -x') WOFileUtils.chmod( self, "/usr/local/sbin/update-ngxblocker", 0o700) + + +def pre_stack(self): + """Inital server configuration and tweak""" + # wo sysctl tweaks + Log.wait(self, 'Applying Linux tweaks') + wo_arch = os.uname()[4] + if os.path.isfile('/proc/1/environ'): + wo_lxc = WOFileUtils.grepcheck( + self, '/proc/1/environ', 'container=lxc') + wo_wsl = WOFileUtils.grepcheck( + self, '/proc/1/environ', 'wsl') + if os.path.isfile('/etc/sysctl.d/60-ubuntu-nginx-web-server.conf'): + WOFileUtils.rm(self, '/etc/sysctl.d/60-ubuntu-nginx-web-server.conf') + if wo_arch == 'x86_64': + if (wo_lxc is not True) and (wo_wsl is not True): + data = dict() + WOTemplate.deploy( + self, '/etc/sysctl.d/60-wo-tweaks.conf', + 'sysctl.mustache', data, True) + if (WOVar.wo_platform_codename == 'bionic' or + WOVar.wo_platform_codename == 'disco' or + WOVar.wo_platform_codename == 'buster'): + if WOShellExec.cmd_exec(self, 'modprobe tcp_bbr'): + with open("/etc/modules-load.d/bbr.conf", + encoding='utf-8', mode='w') as bbr_file: + bbr_file.write('tcp_bbr') + with open("/etc/sysctl.d/60-wo-tweaks.conf", + encoding='utf-8', mode='a') as sysctl_file: + sysctl_file.write( + '\nnet.ipv4.tcp_congestion_control = bbr' + '\nnet.ipv4.tcp_notsent_lowat = 16384') + else: + if WOShellExec.cmd_exec(self, 'modprobe tcp_htcp'): + with open("/etc/modules-load.d/htcp.conf", + encoding='utf-8', mode='w') as bbr_file: + bbr_file.write('tcp_htcp') + with open("/etc/sysctl.d/60-wo-tweaks.conf", + encoding='utf-8', mode='a') as sysctl_file: + sysctl_file.write( + '\nnet.ipv4.tcp_congestion_control = htcp') + WOShellExec.cmd_exec( + self, 'sysctl -eq -p /etc/sysctl.d/60-wo-tweaks.conf') + # sysctl tweak service + if not os.path.isfile('/opt/wo-kernel.sh'): + data = dict() + WOTemplate.deploy(self, '/opt/wo-kernel.sh', + 'wo-kernel-script.mustache', data) + if not os.path.isfile('/lib/systemd/system/wo-kernel.service'): + WOTemplate.deploy( + self, '/lib/systemd/system/wo-kernel.service', + 'wo-kernel-service.mustache') + WOShellExec.cmd_exec(self, 'systemctl enable wo-kernel.service') + WOShellExec.cmd_exec(self, 'systemctl start wo-kernel.service') + # open_files_limit tweak + if not WOFileUtils.grepcheck(self, '/etc/security/limits.conf', '500000'): + with open("/etc/security/limits.conf", + encoding='utf-8', mode='w') as limit_file: + limit_file.write( + '* hard nofile 500000\n' + '* soft nofile 500000\n' + 'root hard nofile 500000\n' + 'root soft nofile 500000\n') + Log.valide(self, 'Applying Linux tweaks') From 3f259ca1858eff504e8eb7713eaa6aa60a5db8e9 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 23 Oct 2019 02:30:23 +0200 Subject: [PATCH 07/79] Fix missing data in stack_pref --- wo/cli/plugins/stack_pref.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index 18cf67a..1f343eb 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -1406,14 +1406,14 @@ def pre_stack(self): WOShellExec.cmd_exec( self, 'sysctl -eq -p /etc/sysctl.d/60-wo-tweaks.conf') # sysctl tweak service + data = dict() if not os.path.isfile('/opt/wo-kernel.sh'): - data = dict() WOTemplate.deploy(self, '/opt/wo-kernel.sh', 'wo-kernel-script.mustache', data) if not os.path.isfile('/lib/systemd/system/wo-kernel.service'): WOTemplate.deploy( self, '/lib/systemd/system/wo-kernel.service', - 'wo-kernel-service.mustache') + 'wo-kernel-service.mustache', data) WOShellExec.cmd_exec(self, 'systemctl enable wo-kernel.service') WOShellExec.cmd_exec(self, 'systemctl start wo-kernel.service') # open_files_limit tweak From 5584d90a2540e5fa5d0ca77a002517f46939cce7 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 23 Oct 2019 12:24:30 +0200 Subject: [PATCH 08/79] Refactor install * remove all tweaking sections * integrate them directly in wo * add pip install from github repo --- .travis.yml | 4 +-- gitconfig.py | 36 ----------------------- install | 57 +++++++----------------------------- wo/cli/plugins/stack_pref.py | 16 ++++++---- wo/core/apt_repo.py | 17 ++++------- wo/core/variables.py | 10 +++++-- 6 files changed, 38 insertions(+), 102 deletions(-) delete mode 100644 gitconfig.py diff --git a/.travis.yml b/.travis.yml index 3337222..b129a9a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -35,6 +35,6 @@ script: - lsb_release -a - sudo bash -c 'echo -e "[user]\n\tname = abc\n\temail = root@localhost.com" > /home/travis/.gitconfig' - sudo echo "Travis Banch = $TRAVIS_BRANCH" - - sudo time bash install --travis -b "$TRAVIS_BRANCH" - - sudo time bash tests/travis.sh + - sudo -E time bash install --travis -b "$TRAVIS_BRANCH" + - sudo -E time bash tests/travis.sh - sudo wo update --travis \ No newline at end of file diff --git a/gitconfig.py b/gitconfig.py deleted file mode 100644 index f335b14..0000000 --- a/gitconfig.py +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env python3 - -import configparser -import os -import re -import shutil - -# WordOps git configuration management -config = configparser.ConfigParser() -config.read(os.path.expanduser("~")+'/.gitconfig') -try: - wo_user = config['user']['name'] - wo_email = config['user']['email'] -except Exception: - print("WordOps (wo) require an username & and an email " - "address to configure Git (used to save server configurations)") - print("Your informations will ONLY be stored locally") - - wo_user = input("Enter your name: ") - while wo_user == "": - print("Unfortunately, this can't be left blank") - wo_user = input("Enter your name: ") - - wo_email = input("Enter your email: ") - - while not re.match(r"^[A-Za-z0-9\.\+_-]+@[A-Za-z0-9\._-]+\.[a-zA-Z]*$", - wo_email): - print("Whoops, seems like you made a typo - " - "the e-mailaddress is invalid...") - wo_email = input("Enter your email: ") - - os.system("git config --global user.name {0}".format(wo_user)) - os.system("git config --global user.email {0}".format(wo_email)) - -if not os.path.isfile('/root/.gitconfig'): - shutil.copy2(os.path.expanduser("~")+'/.gitconfig', '/root/.gitconfig') diff --git a/install b/install index 01380a9..8b86b5e 100755 --- a/install +++ b/install @@ -130,7 +130,11 @@ _run() { # 1 - Define variables for later use ### if [ -z "$wo_branch" ]; then - wo_branch=master + if [ "$wo_travis" = "y" ]; then + wo_branch=updating-configuration + else + wo_branch=master + fi fi readonly wo_log_dir=/var/log/wo/ readonly wo_backup_dir=/var/lib/wo-backup/ @@ -419,32 +423,6 @@ wo_install_acme_sh() { fi } -wo_git_config() { - - if [ "$wo_force_install" = "y" ]; then - [ ! -f "$HOME/.gitconfig" ] && { bash -c 'echo -e "[user]\n\tname = $USER\n\temail = root@$HOSTNAME.local" > $HOME/.gitconfig'; } - fi - - # .gitconfig inital setup - cd /var/lib/wo/tmp/WordOps-install || exit 1 - python3 gitconfig.py - -} - -# Download WordOps -wo_download() { - rm -f /etc/bash_completion.d/wo_auto.rc - rm -rf /var/lib/wo/tmp/WordOps-* - if [ -z "$wo_version" ]; then - curl -sL https://github.com/WordOps/WordOps/archive/${wo_branch}.tar.gz | tar -I pigz -xf - -C /var/lib/wo/tmp - mv "/var/lib/wo/tmp/WordOps-$wo_branch" /var/lib/wo/tmp/WordOps-install - else - curl -sL https://github.com/WordOps/WordOps/archive/v${wo_version}.tar.gz | tar -I pigz -xf - -C /var/lib/wo/tmp - mv "/var/lib/wo/tmp/WordOps-$wo_version" /var/lib/wo/tmp/WordOps-install - fi - return 0 -} - # WordOps install wo_install() { @@ -463,8 +441,12 @@ wo_travis_install() { if [ -d ./dist ]; then rm -rf dist fi - python3 setup.py sdist bdist_wheel - python3 -m pip install --upgrade dist/*.whl + if [ -f ./setup.py ]; then + python3 setup.py sdist bdist_wheel + python3 -m pip install --upgrade dist/*.whl + else + python3 -m pip install git+git://github.com/WordOps/WordOps.git@$wo_branch #egg=wordops -U + fi cp -rn /usr/local/lib/python3.*/dist-packages/usr/* /usr/ cp -rn /usr/local/lib/python3.*/dist-packages/etc/* /etc/ @@ -620,18 +602,6 @@ wo_mariadb_tweak() { fi } -wo_nginx_tweak() { - # increase nginx open_files_limit - if [ ! -d /etc/systemd/system/nginx.service.d ]; then - mkdir -p /etc/systemd/system/nginx.service.d - if [ ! -f /etc/systemd/system/nginx.service.d/limits.conf ]; then - echo -e '[Service]\nLimitNOFILE=500000' > /etc/systemd/system/nginx.service.d/limits.conf - systemctl daemon-reload - nginx -t && service nginx restart - fi - fi -} - wo_clean() { rm -rf /usr/local/lib/python3.*/dist-packages/wo-*.egg /usr/local/lib/python3.*/dist-packages/wordops-*.egg @@ -723,7 +693,6 @@ else wo_lib_error "You already have WordOps $wo_version_new" 1 fi fi - _run wo_backup_wo "Backing-up WO install" _run wo_clean # 2 - Migration from EEv3 elif [ -x /usr/local/bin/ee ]; then @@ -742,7 +711,6 @@ else # skip steps if travis if [ -z "$wo_travis" ]; then #_run wo_download "Downloading WordOps" - wo_git_config _run wo_install "Installing WordOps" else _run wo_travis_install "Installing WordOps" @@ -752,9 +720,6 @@ else _run wo_clean_ee "Cleaning previous EasyEngine install" fi _run wo_install_acme_sh - if [ -x /usr/sbin/nginx ]; then - _run wo_nginx_tweak - fi if [ -d /etc/systemd/system/mariadb.service.d ]; then _run wo_mariadb_tweak fi diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index 1f343eb..cdd2919 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -1,4 +1,3 @@ -import codecs import configparser import os import random @@ -7,11 +6,8 @@ import string import psutil import requests -from wo.cli.plugins.site_functions import * -from wo.cli.plugins.stack_services import WOStackStatusController from wo.core.apt_repo import WORepo from wo.core.aptget import WOAptGet -from wo.core.checkfqdn import check_fqdn_ip from wo.core.cron import WOCron from wo.core.extract import WOExtract from wo.core.fileutils import WOFileUtils @@ -489,6 +485,16 @@ def post_pref(self, apt_packages, packages, upgrade=False): "the cause of this issue", False) else: WOGit.add(self, ["/etc/nginx"], msg="Adding Nginx into Git") + if not os.path.isdir('/etc/systemd/system/nginx.service.d'): + WOFileUtils.mkdir('/etc/systemd/system/nginx.service.d') + if not os.path.isdir( + '/etc/systemd/system/nginx.service.d/limits.conf'): + with open( + '/etc/systemd/system/nginx.service.d/limits.conf', + encoding='utf-8', mode='w') as ngx_limit: + ngx_limit.write('[Service]\nLimitNOFILE=500000') + WOShellExec.cmd_exec(self, 'systemctl daemon-reload') + WOService.restart_service(self, 'nginx') if set(WOVar.wo_php).issubset(set(apt_packages)): WOGit.add(self, ["/etc/php"], msg="Adding PHP into Git") @@ -1415,7 +1421,7 @@ def pre_stack(self): self, '/lib/systemd/system/wo-kernel.service', 'wo-kernel-service.mustache', data) WOShellExec.cmd_exec(self, 'systemctl enable wo-kernel.service') - WOShellExec.cmd_exec(self, 'systemctl start wo-kernel.service') + WOService.start_service(self, 'wo-kernel') # open_files_limit tweak if not WOFileUtils.grepcheck(self, '/etc/security/limits.conf', '500000'): with open("/etc/security/limits.conf", diff --git a/wo/core/apt_repo.py b/wo/core/apt_repo.py index 5192cdd..470ac4e 100644 --- a/wo/core/apt_repo.py +++ b/wo/core/apt_repo.py @@ -49,7 +49,7 @@ class WORepo(): Log.error(self, "Unable to add repo") if ppa is not None: if WOShellExec.cmd_exec( - self, "LC_ALL=C.UTF-8 add-apt-repository -yu '{ppa_name}'" + self, "LC_ALL=C.UTF-8 add-apt-repository -y '{ppa_name}'" .format(ppa_name=ppa)): return True return False @@ -70,7 +70,7 @@ class WORepo(): WOVar().wo_repo_file) try: - repofile = open(repo_file_path, "w+") + repofile = open(repo_file_path, "w+", encoding='utf-8') repofile.write(repofile.read().replace(repo_url, "")) repofile.close() except IOError as e: @@ -96,19 +96,14 @@ class WORepo(): Log.debug(self, "{0}".format(e)) Log.error(self, "Unable to import repo key") - def add_keys(self, keyids, keyserver=None): + def download_key(self, key_url): """ - This function adds imports repository keys from keyserver. - default keyserver is hkp://keyserver.ubuntu.com - user can provide other keyserver with keyserver="hkp://xyz" + This function download gpg keys and add import them with apt-key add" """ - all_keys = ' '.join(keyids) try: WOShellExec.cmd_exec( - self, "apt-key adv --keyserver {serv}" - .format(serv=(keyserver or - "hkp://keyserver.ubuntu.com")) + - " --recv-keys {keys}".format(keys=all_keys)) + self, "curl -sL {0} ".format(key_url) + + "| apt-key add -") except Exception as e: Log.debug(self, "{0}".format(e)) Log.error(self, "Unable to import repo keys") diff --git a/wo/core/variables.py b/wo/core/variables.py index 8654783..4360ca9 100644 --- a/wo/core/variables.py +++ b/wo/core/variables.py @@ -4,6 +4,7 @@ import os from datetime import datetime from re import match from socket import getfqdn +from shutil import copy2 from distro import linux_distribution from sh import git @@ -28,16 +29,18 @@ class WOVar(): wo_date = datetime.now().strftime('%d%b%Y-%H-%M-%S') # WordOps core variables + # linux distribution wo_distro = linux_distribution( full_distribution_name=False)[0].lower() wo_platform_version = linux_distribution( full_distribution_name=False)[1].lower() + # distro codename (bionic, xenial, stretch ...) wo_platform_codename = linux_distribution( full_distribution_name=False)[2].lower() # Get timezone of system if os.path.isfile('/etc/timezone'): - with open("/etc/timezone", "r") as tzfile: + with open("/etc/timezone", mode='r', encoding='utf-8') as tzfile: wo_timezone = tzfile.read().replace('\n', '') if wo_timezone == "Etc/UTC": wo_timezone = "UTC" @@ -59,7 +62,7 @@ class WOVar(): # PHP user wo_php_user = 'www-data' - # Get git user name and EMail + # WordOps git configuration management config = configparser.ConfigParser() config.read(os.path.expanduser("~")+'/.gitconfig') try: @@ -86,6 +89,9 @@ class WOVar(): git.config("--global", "user.name", "{0}".format(wo_user)) git.config("--global", "user.email", "{0}".format(wo_email)) + if not os.path.isfile('/root/.gitconfig'): + copy2(os.path.expanduser("~")+'/.gitconfig', '/root/.gitconfig') + # MySQL hostname wo_mysql_host = "" config = configparser.RawConfigParser() From 44ee18a705c11f47592b1428d44853bc302b269f Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 23 Oct 2019 12:33:48 +0200 Subject: [PATCH 09/79] update changelog and re-order install --- CHANGELOG.md | 7 +++++++ install | 8 ++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 812e66e..255458e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### v3.9.x - [Unreleased] +#### Changed + +- WordOps install is now handled with pip +- Sysctl tweaks are applied during stack install and removed from install script +- Nginx systemd tweaks during Nginx install/upgrade and removed from install script + + ### v3.9.9.4 - 2019-10-18 #### Changed diff --git a/install b/install index 8b86b5e..e5d9aca 100755 --- a/install +++ b/install @@ -164,8 +164,8 @@ wo_check_distro() { if [ -z "$wo_force_install" ]; then if [ "$wo_linux_distro" != "Ubuntu" ] && [ "$wo_linux_distro" != "Debian" ] && [ "$wo_linux_distro" != "Raspbian" ]; then wo_lib_echo_fail "WordOps (wo) only supports Ubuntu, Debian & Raspbian at the moment." - wo_lib_echo_fail "If you are feeling adventurous, you are free to fork WordOps to support" - wo_lib_echo_fail "other Linux distributions and perhaps even Unix deratives." + wo_lib_echo_fail "You can bypass this warning by adding the flag --force to the install command" + wo_lib_echo_fail "Feel free to open a pull-request if you want to add support for another Linux distributions" exit 100 else check_wo_linux_distro=$(lsb_release -sc | grep -E "xenial|bionic|disco|jessie|stretch|buster") @@ -682,10 +682,10 @@ if [ "$wo_purge" = "y" ]; then wo_lib_echo "The WordOps backup files can be found in $WO_BACKUP_FILE" exit 0 else - wo_clean_repo + wo_dir_init + _run wo_clean_repo wo_init wo_check_distro - wo_dir_init # 1 - WO already installed if [ -x /usr/local/bin/wo ]; then if [ -z "$wo_force_install" ]; then From 8f6af1e96597f9502392a05623a2411ae79b7dbc Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 23 Oct 2019 12:37:59 +0200 Subject: [PATCH 10/79] Fix tests --- wo/utils/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wo/utils/test.py b/wo/utils/test.py index 920fd62..6c46790 100644 --- a/wo/utils/test.py +++ b/wo/utils/test.py @@ -1,5 +1,5 @@ """Testing utilities for WordOps""" -from cement.utils.test import * +from cement.utils.tests import CementTestCase from wo.cli.main import WOTestApp From d37c5639e03fd5b0db2d9a110eead4c04dd44887 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 23 Oct 2019 12:40:20 +0200 Subject: [PATCH 11/79] Fix test --- tests/cli/plugins/test_example.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/cli/plugins/test_example.py b/tests/cli/plugins/test_example.py index 0f9d21d..5de3c2d 100644 --- a/tests/cli/plugins/test_example.py +++ b/tests/cli/plugins/test_example.py @@ -1,9 +1,9 @@ """Tests for Example Plugin.""" -from wo.utils import test +from wo.utils import WOTestCase -class ExamplePluginTestCase(test.WOTestCase): +class ExamplePluginTestCase(WOTestCase): def test_load_example_plugin(self): self.app.setup() self.app.plugin.load_plugin('example') From 061744b69f2976448f423b84804d64b809e38808 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 23 Oct 2019 12:43:19 +0200 Subject: [PATCH 12/79] Fix tests --- tests/cli/13_test_stack.py | 9 +++++---- tests/cli/plugins/test_example.py | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/cli/13_test_stack.py b/tests/cli/13_test_stack.py index 63939cb..c6b19b3 100644 --- a/tests/cli/13_test_stack.py +++ b/tests/cli/13_test_stack.py @@ -10,10 +10,11 @@ class CliTestCaseStack(test.WOTestCase): self.app.close() def test_wo_cli_stack_install(self): - self.app = get_test_app(argv=['stack', 'install']) - self.app.setup() - self.app.run() - self.app.close() + argv = ['stack', 'install'] + with self.make_app(argv=argv) as app: + app.run() + self.eq(app.pargs.stack.install) + def test_wo_cli_stack_install_web(self): self.app = get_test_app(argv=['stack', 'install', '--web']) diff --git a/tests/cli/plugins/test_example.py b/tests/cli/plugins/test_example.py index 5de3c2d..dc101f8 100644 --- a/tests/cli/plugins/test_example.py +++ b/tests/cli/plugins/test_example.py @@ -1,6 +1,6 @@ """Tests for Example Plugin.""" -from wo.utils import WOTestCase +from wo.utils.test import WOTestCase class ExamplePluginTestCase(WOTestCase): From e84399540b8399d3cc7deae36d96a8d980a4991f Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 23 Oct 2019 13:47:04 +0200 Subject: [PATCH 13/79] Fix stack pref --- install | 4 ---- wo/cli/plugins/stack_pref.py | 3 ++- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/install b/install index e5d9aca..3acda2d 100755 --- a/install +++ b/install @@ -63,10 +63,6 @@ while [ "$#" -gt 0 ]; do wo_branch="$2" shift ;; - -v | --version) - wo_version="$2" - shift - ;; --force) wo_force_install="y" ;; diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index cdd2919..cce911b 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -486,7 +486,8 @@ def post_pref(self, apt_packages, packages, upgrade=False): else: WOGit.add(self, ["/etc/nginx"], msg="Adding Nginx into Git") if not os.path.isdir('/etc/systemd/system/nginx.service.d'): - WOFileUtils.mkdir('/etc/systemd/system/nginx.service.d') + WOFileUtils.mkdir(self, + '/etc/systemd/system/nginx.service.d') if not os.path.isdir( '/etc/systemd/system/nginx.service.d/limits.conf'): with open( From cadad5b73356d243bb738a9d4a27186b1a755260 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 23 Oct 2019 14:08:54 +0200 Subject: [PATCH 14/79] Refactor wo_download --- wo/cli/plugins/update.py | 2 ++ wo/core/download.py | 25 +++++++------------------ 2 files changed, 9 insertions(+), 18 deletions(-) diff --git a/wo/cli/plugins/update.py b/wo/cli/plugins/update.py index 28f8287..f386c29 100644 --- a/wo/cli/plugins/update.py +++ b/wo/cli/plugins/update.py @@ -50,6 +50,8 @@ class WOUpdateController(CementBaseController): if pargs.preserve: install_args = install_args + "--preserve " + if not os.path.isdir('/var/lib/wo/tmp'): + os.makedirs('/var/lib/wo/tmp') WODownload.download(self, [["https://raw.githubusercontent.com/" "WordOps/WordOps/{0}/install" .format(wo_branch), diff --git a/wo/core/download.py b/wo/core/download.py index 53abbfd..38d0ede 100644 --- a/wo/core/download.py +++ b/wo/core/download.py @@ -1,7 +1,6 @@ """WordOps download core classes.""" import os -import urllib.error -import urllib.request +from requests import get, RequestException from wo.core.logging import Log @@ -23,25 +22,15 @@ class WODownload(): if not os.path.exists(directory): os.makedirs(directory) Log.info(self, "Downloading {0:20}".format(pkg_name), end=' ') - req = urllib.request.Request( - url, headers={'User-Agent': 'Mozilla/5.0'}) - with urllib.request.urlopen(req) as response, open(filename, 'wb') as out_file: - out_file.write(response.read()) + with open(filename, "wb") as out_file: + req = get(url, timeout=15) + if req.encoding is None: + req.encoding = 'utf-8' + out_file.write(req.content) Log.info(self, "{0}".format("[" + Log.ENDC + "Done" + Log.OKBLUE + "]")) - except urllib.error.URLError as e: + except RequestException as e: Log.debug(self, "[{err}]".format(err=str(e.reason))) Log.error(self, "Unable to download file, {0}" .format(filename)) return False - except urllib.HTTPError.error as e: - Log.error(self, "Package download failed. {0}" - .format(pkg_name)) - Log.debug(self, "[{err}]".format(err=str(e.reason))) - return False - except urllib.ContentTooShortError.error as e: - Log.debug(self, "{0}{1}".format(e.errno, e.strerror)) - Log.error(self, "Package download failed. The amount of the" - " downloaded data is less than " - "the expected amount \{0} ".format(pkg_name)) - return False From 85e68b1fc63223d2ccd2b39c1f964ad1bea83d1d Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 24 Oct 2019 11:13:46 +0200 Subject: [PATCH 15/79] Fix templates not updated after upgrade --- .travis.yml | 5 ----- CHANGELOG.md | 10 +++++++++- install | 2 +- wo/cli/plugins/log.py | 38 +++++++++++++++++++------------------- 4 files changed, 29 insertions(+), 26 deletions(-) diff --git a/.travis.yml b/.travis.yml index b129a9a..4398eed 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,10 +7,6 @@ notifications: webhooks: secure: "JiGtzYplTyFg/L6Rsi7ptEQIV29O5qCWU2Zf5pLITsQrBrQO4cIXXp9G4Z+cenXjfIiqbqIgU0US3zXeIAl4g14xdfzmMYeMMwuKBpI8afMYv8MD6ldoP0MTFHQfROE6OXxKLVUvZn1R0oLLU1fzVSI0qGjNkt20cf/Lrt/reH/zS5hAI92kWI3u2zPu7Zn/g/a8MO/Y3Iv7v1PSQaVkVJVqtOK3U2GJqhIv2G1AVcaPb7Nh/V2zm2dDYBVT0UotBnlBUcUXbEMP77D9pjtWXd1/0rWuJIHixMjwUybpZqY75UMee5INynU6OZRsv029LRHAIMkWhfBkdVN/U5jhQJzui14+vRQrb5nfUMG8Cd8INojDlu6dk/ps2GzTCCXBITeMQKAouUoHD2LEbsNp17xi1K4ZlKb3+0lrOAiS4JYFE6wOo4yMlLTYoquYSqk7AuxuUS8A5OD5MYxhk9uafiTSxKFOo39KYWTSaACsPD8q1swaTSjoYm9skyZvIkIFq5bHBCYEGFe6X/NY9l5tz3hSe+TJOerCHsg+dXVuQl+pIp5nw2as9TH9ox5Vgqc9Zh4GbTDQVvdAmUpmlsZ/SKoOMCkmkB1aRNFq/7RnERIJyAEGJbauHWmjtOM4cCxesl0L0b2Eab89zQpSn7pzE8JTiJgpzCUc22p653PTaqM=" -addons: - apt: - update: true - git: quiet: true @@ -23,7 +19,6 @@ before_script: - unset LANG - sudo apt-get update --allow-releaseinfo-change -qq - sudo apt-get -qq purge mysql* graphviz* redis* - - sudo apt-get install -qq git python3-setuptools python3-dev python3-apt ccze tree - sudo apt-get -qq autoremove --purge diff --git a/CHANGELOG.md b/CHANGELOG.md index 255458e..b948351 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,11 +8,19 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### v3.9.x - [Unreleased] +#### Added + +- WordOps install is now installed with pip from PyPi (easier, cleaner and safer) inside a wheel + #### Changed -- WordOps install is now handled with pip - Sysctl tweaks are applied during stack install and removed from install script - Nginx systemd tweaks during Nginx install/upgrade and removed from install script +- Initial creation of .gitconfig is displayed the first time you run the command `wo` + +#### Fixed + +- Incorrect PHP-FPM log path is `wo log` ### v3.9.9.4 - 2019-10-18 diff --git a/install b/install index 3acda2d..8e54d15 100755 --- a/install +++ b/install @@ -423,7 +423,7 @@ wo_install_acme_sh() { wo_install() { python3 -m pip install --upgrade wordops - cp -rn /usr/local/lib/python3.*/dist-packages/usr/* /usr/ + cp -rf /usr/local/lib/python3.*/dist-packages/usr/* /usr/ cp -rn /usr/local/lib/python3.*/dist-packages/etc/* /etc/ } diff --git a/wo/cli/plugins/log.py b/wo/cli/plugins/log.py index 68f9974..54b23de 100644 --- a/wo/cli/plugins/log.py +++ b/wo/cli/plugins/log.py @@ -46,7 +46,7 @@ class WOLogShowController(CementBaseController): (['--php'], dict(help='Show PHP Error logs file', action='store_true')), (['--fpm'], - dict(help='Show PHP5-fpm slow logs file', + dict(help='Show PHP-FPM slow logs file', action='store_true')), (['--mysql'], dict(help='Show MySQL logs file', action='store_true')), @@ -92,10 +92,10 @@ class WOLogShowController(CementBaseController): self.msg = self.msg + ["/var/log/nginx/*access.log"] if self.app.pargs.fpm: - open('/var/log/php5/slow.log', 'a').close() - open('/var/log/php5/fpm.log', 'a').close() - self.msg = self.msg + ['/var/log/php5/slow.log', - '/var/log/php5/fpm.log'] + open('/var/log/php/7.2/slow.log', 'a').close() + open('/var/log/php7.2-fpm.log', 'a').close() + self.msg = self.msg + ['/var/log/php/7.2/slow.log', + '/var/log/php7.2-fpm.log'] if self.app.pargs.mysql: # MySQL debug will not work for remote MySQL if WOVar.wo_mysql_host == "localhost": @@ -171,7 +171,7 @@ class WOLogResetController(CementBaseController): (['--php'], dict(help='Reset PHP Error logs file', action='store_true')), (['--fpm'], - dict(help='Reset PHP5-fpm slow logs file', + dict(help='Reset PHP-FPM slow logs file', action='store_true')), (['--mysql'], dict(help='Reset MySQL logs file', action='store_true')), @@ -231,10 +231,10 @@ class WOLogResetController(CementBaseController): self.msg = self.msg + ["/var/log/nginx/*access.log"] if self.app.pargs.fpm: - open('/var/log/php5/slow.log', 'a').close() - open('/var/log/php5/fpm.log', 'a').close() - self.msg = self.msg + ['/var/log/php5/slow.log', - '/var/log/php5/fpm.log'] + open('/var/log/php/7.2/slow.log', 'a').close() + open('/var/log/php7.2-fpm.log', 'a').close() + self.msg = self.msg + ['/var/log/php/7.2/slow.log', + '/var/log/php7.2-fpm.log'] if self.app.pargs.mysql: # MySQL debug will not work for remote MySQL if WOVar.wo_mysql_host == "localhost": @@ -313,7 +313,7 @@ class WOLogGzipController(CementBaseController): (['--php'], dict(help='GZip PHP Error logs file', action='store_true')), (['--fpm'], - dict(help='GZip PHP5-fpm slow logs file', + dict(help='GZip PHP-FPM slow logs file', action='store_true')), (['--mysql'], dict(help='GZip MySQL logs file', action='store_true')), @@ -359,10 +359,10 @@ class WOLogGzipController(CementBaseController): self.msg = self.msg + ["/var/log/nginx/*access.log"] if self.app.pargs.fpm: - open('/var/log/php5/slow.log', 'a').close() - open('/var/log/php5/fpm.log', 'a').close() - self.msg = self.msg + ['/var/log/php5/slow.log', - '/var/log/php5/fpm.log'] + open('/var/log/php/7.2/slow.log', 'a').close() + open('/var/log/php7.2-fpm.log', 'a').close() + self.msg = self.msg + ['/var/log/php/7.2/slow.log', + '/var/log/php7.2-fpm.log'] if self.app.pargs.mysql: # MySQL debug will not work for remote MySQL if WOVar.wo_mysql_host == "localhost": @@ -497,10 +497,10 @@ class WOLogMailController(CementBaseController): self.msg = self.msg + ["/var/log/nginx/*access.log"] if self.app.pargs.fpm: - open('/var/log/php5/slow.log', 'a').close() - open('/var/log/php5/fpm.log', 'a').close() - self.msg = self.msg + ['/var/log/php5/slow.log', - '/var/log/php5/fpm.log'] + open('/var/log/php/7.2/slow.log', 'a').close() + open('/var/log/php7.2-fpm.log', 'a').close() + self.msg = self.msg + ['/var/log/php/7.2/slow.log', + '/var/log/php7.2-fpm.log'] if self.app.pargs.mysql: # MySQL debug will not work for remote MySQL if WOVar.wo_mysql_host == "localhost": From b3cf60187926a5b0747b0423eb9499515a4b00a0 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Thu, 24 Oct 2019 22:04:52 +0200 Subject: [PATCH 16/79] Improve `wo update` --- CHANGELOG.md | 1 + wo/cli/plugins/update.py | 42 +++++++++++++++++------------- wo/cli/templates/php-pool.mustache | 2 +- 3 files changed, 26 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b948351..7773879 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Sysctl tweaks are applied during stack install and removed from install script - Nginx systemd tweaks during Nginx install/upgrade and removed from install script - Initial creation of .gitconfig is displayed the first time you run the command `wo` +- Added `/var/lib/php/sessions/` to open_basedir to allow php sessions storage #### Fixed diff --git a/wo/cli/plugins/update.py b/wo/cli/plugins/update.py index f386c29..49e0994 100644 --- a/wo/cli/plugins/update.py +++ b/wo/cli/plugins/update.py @@ -22,12 +22,17 @@ class WOUpdateController(CementBaseController): arguments = [ (['--force'], dict(help='Force WordOps update', action='store_true')), - (['--preserve'], - dict(help='Preserve current Nginx configuration', - action='store_true')), (['--beta'], - dict(help='Update WordOps to latest beta release', + dict(help='Update WordOps to latest mainline release ' + '(same than --mainline)', action='store_true')), + (['--mainline'], + dict(help='Update WordOps to latest mainline release', + action='store_true')), + (['--branch'], + dict(help="Update WordOps from a specific repository branch ", + action='store' or 'store_const', + const='develop', nargs='?')), (['--travis'], dict(help='Argument used only for WordOps development', action='store_true')), @@ -38,17 +43,15 @@ class WOUpdateController(CementBaseController): def default(self): pargs = self.app.pargs filename = "woupdate" + time.strftime("%Y%m%d-%H%M%S") - - if pargs.beta: - wo_branch = "beta" - install_args = "" + install_args = "" + if pargs.mainline or pargs.beta: + wo_branch = "mainline" + elif pargs.branch: + wo_branch = pargs.branch else: wo_branch = "master" - install_args = "" if pargs.force: install_args = install_args + "--force " - if pargs.preserve: - install_args = install_args + "--preserve " if not os.path.isdir('/var/lib/wo/tmp'): os.makedirs('/var/lib/wo/tmp') @@ -59,13 +62,14 @@ class WOUpdateController(CementBaseController): "update script"]]) if pargs.travis: - try: - Log.info(self, "updating WordOps, please wait...") - os.system("/bin/bash install --travis " - "-b $TRAVIS_BRANCH --force") - except OSError as e: - Log.debug(self, str(e)) - Log.error(self, "WordOps update failed !") + if os.path.isfile('install'): + try: + Log.info(self, "updating WordOps, please wait...") + os.system("/bin/bash install --travis " + "--force") + except OSError as e: + Log.debug(self, str(e)) + Log.error(self, "WordOps update failed !") else: try: Log.info(self, "updating WordOps, please wait...") @@ -76,6 +80,8 @@ class WOUpdateController(CementBaseController): Log.debug(self, str(e)) Log.error(self, "WordOps update failed !") + os.remove("/var/lib/wo/tmp/{0}".format(filename)) + def load(app): # register the plugin class.. this only happens if the plugin is enabled diff --git a/wo/cli/templates/php-pool.mustache b/wo/cli/templates/php-pool.mustache index 654e944..cbdddb2 100644 --- a/wo/cli/templates/php-pool.mustache +++ b/wo/cli/templates/php-pool.mustache @@ -20,4 +20,4 @@ listen.backlog = 32768 catch_workers_output = yes -{{#openbasedir}}php_admin_value[open_basedir] = "/var/www/:/usr/share/php/:/tmp/:/var/run/nginx-cache/:/dev/urandom:/dev/shm"{{/openbasedir}} +{{#openbasedir}}php_admin_value[open_basedir] = "/var/www/:/usr/share/php/:/tmp/:/var/run/nginx-cache/:/dev/urandom:/dev/shm:/var/lib/php/sessions/"{{/openbasedir}} From 7765b2ce84479ced16acf870c9cd0075b3e823a6 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Fri, 25 Oct 2019 23:58:08 +0200 Subject: [PATCH 17/79] Several improvements * Run `mysql_upgrade` during MySQL upgrade with `wo stack upgrade` to perform migration if needed * WordOps now check if a repository already exist before trying to adding it again. * install script refactored --- CHANGELOG.md | 4 ++ config/bash_completion.d/wo_auto.rc | 2 +- docs/wo.8 | 34 ++++++++------ install | 59 ++++++++++++------------- wo/cli/plugins/site.py | 2 +- wo/cli/plugins/stack_migrate.py | 1 - wo/cli/plugins/stack_pref.py | 66 +++++++++++++++++++++------- wo/cli/plugins/sync.py | 28 +++++++----- wo/cli/templates/nextcloud.mustache | 9 ++-- wo/cli/templates/nginx-core.mustache | 6 +-- wo/core/acme.py | 9 ++++ wo/core/domainvalidate.py | 1 - wo/core/fileutils.py | 14 +++--- 13 files changed, 146 insertions(+), 89 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7773879..f00707f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), #### Added - WordOps install is now installed with pip from PyPi (easier, cleaner and safer) inside a wheel +- Redis 5.0.6 package backported to Debian 8/9/10 +- Custom motd to display a message if a new WordOps release is available +- Run `mysql_upgrade` during MySQL upgrade with `wo stack upgrade` to perform migration if needed #### Changed @@ -18,6 +21,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Nginx systemd tweaks during Nginx install/upgrade and removed from install script - Initial creation of .gitconfig is displayed the first time you run the command `wo` - Added `/var/lib/php/sessions/` to open_basedir to allow php sessions storage +- WordOps now check if a repository already exist before trying to adding it again. #### Fixed diff --git a/config/bash_completion.d/wo_auto.rc b/config/bash_completion.d/wo_auto.rc index d5dacc0..c3146c0 100644 --- a/config/bash_completion.d/wo_auto.rc +++ b/config/bash_completion.d/wo_auto.rc @@ -79,7 +79,7 @@ _wo_complete() ;; "upgrade" ) COMPREPLY=( $(compgen \ - -W "--web --admin --utils --nginx --php --php73 --mysql --all --netdata --composer --phpmyadmin --dashboard --no-prompt --mysqtuner --wpcli --force" \ + -W "--web --admin --utils --nginx --php --php73 --mysql --all --netdata --composer --phpmyadmin --dashboard --mysqtuner --wpcli --force" \ -- $cur) ) ;; "start" | "stop" | "reload" | "restart" | "status") diff --git a/docs/wo.8 b/docs/wo.8 index b6f23b7..357d1ea 100644 --- a/docs/wo.8 +++ b/docs/wo.8 @@ -1,27 +1,27 @@ -.TH wo 8 "WordOps (wo) version: 3.9.6.3" "Jul 26,2019" "WordOps" +.TH wo 8 "WordOps (wo) version: 3.10.0" "Oct 24,2019" "WordOps" .SH NAME .B WordOps (wo) \- Manage Nginx Based Websites. .SH SYNOPSIS wo [ --version | --help | info | stack | site | debug | update | clean | import_slow_log | log | secure | sync | maintenance ] .TP -wo stack [ install | remove | purge | migrate | upgrade] [ --web | --all | --nginx | --php | --php73 | --mysql | --admin | --adminer | --redis | --phpmyadmin | --phpredisadmin | --wpcli | --utils | --dashboard | --netdata | --fail2ban | --proftpd ] +wo stack [ install | remove | purge | migrate | upgrade ] [ --web | --all | --nginx | --php | --php73 | --mysql | --admin | --adminer | --redis | --phpmyadmin | --phpredisadmin | --wpcli | --utils | --dashboard | --netdata | --fail2ban | --proftpd ] .TP wo stack [ status | start | stop | reload | restart ] [--all | --nginx | --php | --php73 |--mysql | --web | --redis | --netdata | --fail2ban | --proftpd] .TP wo site [ list | info | show | enable | disable | edit | cd | show ] [ example.com ] .TP -wo site create example.com [ --html | --php | --php73 | --mysql] [[--wp | --wpsubdir | --wpsubdomain ] [--wpsc | --wpfc | --wpredis | --letsencrypt/-le/--letsencrypt=wildcard][--dns/--dns=dns_cf/dns_do]] +wo site create example.com [ --html | --php | --php73 | --mysql][[--wp | --wpsubdir | --wpsubdomain ] [ --wpsc | --wpfc | --wpredis | --wpce | --wprocket ] [ -le/--letsencrypt=wildcard ][ --dns/--dns=dns_cf/dns_dgon]] .TP -wo site update example.com [ --php | --php73 |--mysql] [[--wp | --wpsubdir | --wpsubdomain ] [--wpsc | --wpfc | --wpredis ] [--password] [-le/--letsencrypt/--letsencrypt=on/off/wildcard/clean/purge] [--dns/--dns=dns_cf/dns_do]] +wo site update example.com [ --php | --php73 |--mysql] [[--wp | --wpsubdir | --wpsubdomain ] [--wpsc | --wpfc | --wpredis | --wpce | --wprocket ] [--password] [-le/--letsencrypt=on/off/wildcard/clean/purge ] [ --dns/--dns=dns_cf/dns_dgon ] .TP -wo site delete example.com [--db | --files | --all | --no-prompt | --force/-f ] +wo site delete example.com [--db | --files | --all | --no-prompt | --force ] .TP wo debug [ -i | --all=on/off |--nginx=on/off | --rewrite=on/off | --php=on/off | --fpm=on/off | --mysql=on/off ] .TP wo debug example.com [ -i | --all=on/off | --nginx=on/off | --rewrite=on/off | --wp=on/off ] .TP -wo secure [ --auth | --port | --ip ] +wo secure [ --auth | --port | --ip | --ssh | --sshport ] .SH DESCRIPTION WordOps aka wo is the opensource project developed with the purpose to automate web-server configuration. .br @@ -48,7 +48,7 @@ Display WordOps (wo) help. .TP .B install [ --all | --web | --nginx | --php | --php73 |--mysql | --redis | --adminer | --phpmyadmin | --phpredismyadmin | --wpcli | --utils | --netdata | --dashboard | --fail2ban | --proftpd ] .br -Install Nginx PHP5 MySQL Postfix stack Packages if not used with +Install Nginx PHP7.2 MariaDB SendMail Netdata Fail2Ban stack Packages if not used with .br any options.Installs specific package if used with option. .TP @@ -129,13 +129,13 @@ Disable site by Destroying softlink with site file in .br Edit NGINX configuration of site. .TP -.B create [ example.com ] [ --html | --php | --php73 |--mysql] [[--wp | --wpsubdir | --wpsubdomain ] [--wpsc | --wpfc | --wpredis ]] +.B create [ example.com ] [ --html | --php | --php73 |--mysql] [[--wp | --wpsubdir | --wpsubdomain ] [--wpsc | --wpfc | --wpredis ] .br Create new site according to given options. If no options provided .br create static site with html only. .TP -.B update [ example.com ] [ --html | --php | --php73 |--mysql] [[--wp | --wpsubdir | --wpsubdomain ] [ --wpsc | --wpfc | --wpredis ] [--password]] +.B update [ example.com ] [ --html | --php | --php73 |--mysql] [[--wp | --wpsubdir | --wpsubdomain ] [ --wpsc | --wpfc | --wpredis ] [--password ] .br Update site configuration according to specified options. .TP @@ -270,17 +270,23 @@ used with wo secure command. Update whitelist IP address .TP .B --wpsc .br -Install and activate Nginx-helper and WP Super Cache plugin. +Install and activate WP Super Cache plugin and serve pages from cache directly with Nginx. .TP .B --wpfc .br -Install and activate Nginx-helper plugin with -.br -Nginx FastCGI cache. +Install and activate Nginx-helper plugin with Nginx FastCGI cache. .TP .B --wpredis .br -Install, activate, configure Nginx-helper and Redis Object Cache Plugin, Configure NGINX for Redis Page Caching. +Install, activate, configure Nginx-helper and Redis Object Cache Plugin, Configure NGINX for Redis Full-Page Caching. +.TP +.B --wpce +.br +Install and activate Cache-enabler plugin and serve pages from cache directly with Nginx. +.TP +.B --wprocket +.br +Configure Nginx for WP-Rocket plugin to serve pages from cache directly with Nginx. .SH FILES .br /etc/wo/wo.conf diff --git a/install b/install index 8e54d15..7e09b22 100755 --- a/install +++ b/install @@ -9,7 +9,7 @@ # ------------------------------------------------------------------------- # wget -qO wo wops.cc && sudo bash wo # ------------------------------------------------------------------------- -# Version 3.9.9.4 - 2019-10-18 +# Version 3.10.0 - 2019-10-25 # ------------------------------------------------------------------------- # CONTENTS @@ -122,6 +122,10 @@ _run() { } +_curl() { + curl -m 10 --retry 3 -sL "$@" +} + ### # 1 - Define variables for later use ### @@ -180,12 +184,13 @@ wo_check_distro() { wo_dir_init() { if [ ! -d "$wo_log_dir" ] || [ ! -d "$wo_backup_dir" ] || [ ! -d "$wo_tmp_dir" ]; then - mkdir -p "$wo_backup_dir" "$wo_log_dir" "$wo_tmp_dir" || wo_lib_error "Whoops - seems we are unable to create the log directory $wo_log_dir, exit status " $? + mkdir -p "$wo_backup_dir" "$wo_log_dir" "$wo_tmp_dir" # create wordops log files touch /var/log/wo/{wordops.log,install.log} - chmod -R 700 "$wo_log_dir" "$wo_backup_dir" "$wo_tmp_dir" || wo_lib_error "Whoops, there was an error setting the permissions on the WordOps log folder, exit status " $? + chmod -R 750 "$wo_log_dir" "$wo_backup_dir" "$wo_tmp_dir" + chown -R root:adm "$wo_log_dir" fi } @@ -203,7 +208,6 @@ wo_install_dep() { build-essential curl gzip python3-pip python3-wheel python3-apt python3-setuptools python3-dev sqlite3 git tar software-properties-common pigz \ gnupg2 cron ccze rsync apt-transport-https tree haveged ufw unattended-upgrades tzdata ntp > /dev/null 2>&1 curl -sL https://download.opensuse.org/repositories/home:/virtubox:/WordOps/xUbuntu_18.04/Release.key | apt-key add - - add-apt-repository ppa:wordops/nginx-wo -yn else # install dependencies apt-get -option=Dpkg::options::=--force-confmiss --option=Dpkg::options::=--force-confold --assume-yes install \ @@ -221,7 +225,8 @@ wo_install_dep() { cp /usr/share/unattended-upgrades/20auto-upgrades /etc/apt/apt.conf.d/20auto-upgrades fi # upgrade pip - python3 -m pip install --upgrade pip setuptools wheel + python3 -m pip install --upgrade pip + python3 -m pip install --upgrade setuptools wheel } @@ -411,12 +416,10 @@ wo_install_acme_sh() { # Let's Encrypt .well-known folder setup if [ ! -d /var/www/html/.well-known/acme-challenge ]; then mkdir -p /var/www/html/.well-known/acme-challenge - chown -R www-data:www-data /var/www/html /var/www/html/.well-known - chmod 750 /var/www/html /var/www/html/.well-known - else - chown -R www-data:www-data /var/www/html /var/www/html/.well-known - chmod 750 /var/www/html /var/www/html/.well-known fi + chown -R www-data:www-data /var/www/html /var/www/html/.well-known + chmod 750 /var/www/html /var/www/html/.well-known + } # WordOps install @@ -537,24 +540,20 @@ wo_upgrade_nginx() { wo_update_latest() { # Move ~/.my.cnf to /etc/mysql/conf.d/my.cnf - if [ ! -f /etc/mysql/conf.d/my.cnf ]; then - # create conf.d folder if not exist - [ ! -d /etc/mysql/conf.d ] && { - mkdir -p /etc/mysql/conf.d - chmod 755 /etc/mysql/conf.d - } - if [ -f "$HOME/.my.cnf" ]; then - cp -f "$HOME/.my.cnf" /etc/mysql/conf.d/my.cnf - chmod 600 /etc/mysql/conf.d/my.cnf - - elif [ -f /root/.my.cnf ]; then - cp -f /root/.my.cnf /etc/mysql/conf.d/my.cnf - chmod 600 /etc/mysql/conf.d/my.cnf - fi - else - if [ ! -f /root/.my.cnf ]; then - cp /etc/mysql/conf.d/my.cnf /root/.my.cnf - chmod 600 /root/.my.cnf + if [ -d /etc/mysql ]; then + if [ ! -f /etc/mysql/conf.d/my.cnf ]; then + # create conf.d folder if not exist + [ ! -d /etc/mysql/conf.d ] && { + mkdir -p /etc/mysql/conf.d + chmod 755 /etc/mysql/conf.d + } + if [ -f /root/.my.cnf ]; then + cp -f /root/.my.cnf /etc/mysql/conf.d/my.cnf + chmod 600 /etc/mysql/conf.d/my.cnf + elif [ -f "$HOME/.my.cnf"]; then + cp -f "$HOME/.my.cnf" /etc/mysql/conf.d/my.cnf + chmod 600 /etc/mysql/conf.d/my.cnf + fi fi fi } @@ -586,7 +585,7 @@ wo_remove_ee_cron() { } wo_domain_suffix() { - curl -m 10 --retry 3 -sL https://raw.githubusercontent.com/publicsuffix/list/master/public_suffix_list.dat | sed '/^\/\//d' | sed '/^$/d' | sed 's/^\s+//g' > /var/lib/wo/public_suffix_list.dat + _curl https://raw.githubusercontent.com/publicsuffix/list/master/public_suffix_list.dat | sed '/^\/\//d' | sed '/^$/d' | sed 's/^\s+//g' > /var/lib/wo/public_suffix_list.dat } wo_mariadb_tweak() { @@ -658,7 +657,7 @@ wo_init() { if [ -f ./setup.py ]; then readonly wo_version_new=$(grep "version='" setup.py | awk -F "'" '{print$2}' 2>&1) else - readonly wo_version_new=$(curl -sL https://wops.cc/setup.py 2>&1 | grep "version='" | awk -F "'" '{print$2}' 2>&1) + readonly wo_version_new=$(curl -m 10 --retry 3 -sI https://github.com/WordOps/WordOps/releases/latest | grep tag | awk -F "/" '{print $8}' 2>&1) fi echo "" diff --git a/wo/cli/plugins/site.py b/wo/cli/plugins/site.py index bbb6b8e..344d0ea 100644 --- a/wo/cli/plugins/site.py +++ b/wo/cli/plugins/site.py @@ -1151,7 +1151,7 @@ class WOSiteUpdateController(CementBaseController): stype = oldsitetype cache = oldcachetype if oldsitetype == 'html' or oldsitetype == 'proxy': - data['static'] = True + data['static'] = False data['wp'] = False data['multisite'] = False data['wpsubdir'] = False diff --git a/wo/cli/plugins/stack_migrate.py b/wo/cli/plugins/stack_migrate.py index 20bf585..205714e 100644 --- a/wo/cli/plugins/stack_migrate.py +++ b/wo/cli/plugins/stack_migrate.py @@ -1,7 +1,6 @@ import configparser import os -from cement.core import handler, hook from cement.core.controller import CementBaseController, expose from wo.core.apt_repo import WORepo diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index cce911b..5cd7987 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -102,41 +102,68 @@ def pre_pref(self, apt_packages): # add nginx repository if set(WOVar.wo_nginx).issubset(set(apt_packages)): - Log.info(self, "Adding repository for NGINX, please wait...") if (WOVar.wo_distro == 'ubuntu'): - WORepo.add(self, ppa=WOVar.wo_nginx_repo) - Log.debug(self, 'Adding ppa for Nginx') + if not os.path.isfile( + 'wordops-ubuntu-nginx-wo-{0}.list' + .format(WOVar.wo_platform_codename)): + Log.info(self, "Adding repository for NGINX, please wait...") + WORepo.add(self, ppa=WOVar.wo_nginx_repo) + Log.debug(self, 'Adding ppa for Nginx') else: - WORepo.add(self, repo_url=WOVar.wo_nginx_repo) - Log.debug(self, 'Adding repository for Nginx') + if not WOFileUtils.grepcheck( + self, '/etc/apt/sources.list/wo-repo.list', + 'download.opensuse.org'): + Log.info(self, "Adding repository for NGINX, please wait...") + Log.debug(self, 'Adding repository for Nginx') + WORepo.add(self, repo_url=WOVar.wo_nginx_repo) WORepo.add_key(self, WOVar.wo_nginx_key) # add php repository if (set(WOVar.wo_php73).issubset(set(apt_packages)) or set(WOVar.wo_php).issubset(set(apt_packages))): - Log.info(self, "Adding repository for PHP, please wait...") if (WOVar.wo_distro == 'ubuntu'): Log.debug(self, 'Adding ppa for PHP') - WORepo.add(self, ppa=WOVar.wo_php_repo) + if not os.path.isfile( + '/etc/apt/sources.list.d/ondrej-ubuntu-php-{0}.list' + .format(WOVar.wo_platform_codename)): + Log.info(self, "Adding repository for PHP, please wait...") + WORepo.add(self, ppa=WOVar.wo_php_repo) else: # Add repository for php if (WOVar.wo_platform_codename == 'buster'): php_pref = ("Package: *\nPin: origin " "packages.sury.org" "\nPin-Priority: 1000\n") - with open('/etc/apt/preferences.d/' - 'PHP.pref', 'w') as php_pref_file: + with open( + '/etc/apt/preferences.d/' + 'PHP.pref', mode='w', + encoding='utf-8') as php_pref_file: php_pref_file.write(php_pref) - Log.debug(self, 'Adding repo_url of php for debian') - WORepo.add(self, repo_url=WOVar.wo_php_repo) + if not WOFileUtils.grepcheck( + self, '/etc/apt/sources.list.d/wo-repo.list', + 'packages.sury.org'): + Log.debug(self, 'Adding repo_url of php for debian') + Log.info(self, "Adding repository for PHP, please wait...") + WORepo.add(self, repo_url=WOVar.wo_php_repo) Log.debug(self, 'Adding deb.sury GPG key') WORepo.add_key(self, WOVar.wo_php_key) # add redis repository if set(WOVar.wo_redis).issubset(set(apt_packages)): - Log.info(self, "Adding repository for Redis, please wait...") if WOVar.wo_distro == 'ubuntu': - Log.debug(self, 'Adding ppa for redis') - WORepo.add(self, ppa=WOVar.wo_redis_repo) + if not os.path.isfile( + '/etc/apt/sources.list.d/' + 'chris-lea-ubuntu-redis-server-{0}.list' + .format(WOVar.wo_platform_codename)): + Log.info(self, "Adding repository for Redis, please wait...") + Log.debug(self, 'Adding ppa for redis') + WORepo.add(self, ppa=WOVar.wo_redis_repo) + else: + if not WOFileUtils.grepcheck( + self, '/etc/apt/sources.list/wo-repo.list', + 'download.opensuse.org'): + Log.info(self, "Adding repository for Redis, please wait...") + WORepo.add(self, repo_url=WOVar.wo_php_repo) + WORepo.add_key(self, WOVar.wo_nginx_key) def post_pref(self, apt_packages, packages, upgrade=False): @@ -1374,21 +1401,29 @@ def post_pref(self, apt_packages, packages, upgrade=False): def pre_stack(self): """Inital server configuration and tweak""" # wo sysctl tweaks - Log.wait(self, 'Applying Linux tweaks') + # check system type wo_arch = os.uname()[4] if os.path.isfile('/proc/1/environ'): + # detect lxc containers wo_lxc = WOFileUtils.grepcheck( self, '/proc/1/environ', 'container=lxc') + # detect wsl wo_wsl = WOFileUtils.grepcheck( self, '/proc/1/environ', 'wsl') + else: + wo_wsl = True + wo_lxc = True + # remove old sysctl tweak if os.path.isfile('/etc/sysctl.d/60-ubuntu-nginx-web-server.conf'): WOFileUtils.rm(self, '/etc/sysctl.d/60-ubuntu-nginx-web-server.conf') + if wo_arch == 'x86_64': if (wo_lxc is not True) and (wo_wsl is not True): data = dict() WOTemplate.deploy( self, '/etc/sysctl.d/60-wo-tweaks.conf', 'sysctl.mustache', data, True) + # use tcp_bbr congestion algorithm only on new kernels if (WOVar.wo_platform_codename == 'bionic' or WOVar.wo_platform_codename == 'disco' or WOVar.wo_platform_codename == 'buster'): @@ -1410,6 +1445,7 @@ def pre_stack(self): encoding='utf-8', mode='a') as sysctl_file: sysctl_file.write( '\nnet.ipv4.tcp_congestion_control = htcp') + # apply sysctl tweaks WOShellExec.cmd_exec( self, 'sysctl -eq -p /etc/sysctl.d/60-wo-tweaks.conf') # sysctl tweak service diff --git a/wo/cli/plugins/sync.py b/wo/cli/plugins/sync.py index 662466d..872f962 100644 --- a/wo/cli/plugins/sync.py +++ b/wo/cli/plugins/sync.py @@ -52,18 +52,22 @@ class WOSyncController(CementBaseController): if configfiles: if WOFileUtils.isexist(self, configfiles[0]): - wo_db_name = (WOFileUtils.grep(self, configfiles[0], - 'DB_NAME').split(',')[1] - .split(')')[0].strip().replace('\'', '')) - wo_db_user = (WOFileUtils.grep(self, configfiles[0], - 'DB_USER').split(',')[1] - .split(')')[0].strip().replace('\'', '')) - wo_db_pass = (WOFileUtils.grep(self, configfiles[0], - 'DB_PASSWORD').split(',')[1] - .split(')')[0].strip().replace('\'', '')) - wo_db_host = (WOFileUtils.grep(self, configfiles[0], - 'DB_HOST').split(',')[1] - .split(')')[0].strip().replace('\'', '')) + wo_db_name = ( + WOFileUtils.grep(self, configfiles[0], + 'DB_NAME').split(',')[1] + .split(')')[0].strip().replace('\'', '')) + wo_db_user = ( + WOFileUtils.grep(self, configfiles[0], + 'DB_USER').split(',')[1] + .split(')')[0].strip().replace('\'', '')) + wo_db_pass = ( + WOFileUtils.grep(self, configfiles[0], + 'DB_PASSWORD').split(',')[1] + .split(')')[0].strip().replace('\'', '')) + wo_db_host = ( + WOFileUtils.grep(self, configfiles[0], + 'DB_HOST').split(',')[1] + .split(')')[0].strip().replace('\'', '')) # Check if database really exist try: diff --git a/wo/cli/templates/nextcloud.mustache b/wo/cli/templates/nextcloud.mustache index 48a7df1..8d2eae0 100644 --- a/wo/cli/templates/nextcloud.mustache +++ b/wo/cli/templates/nextcloud.mustache @@ -8,7 +8,7 @@ location = /robots.txt { access_log off; } location / { - rewrite ^ /index.php$request_uri; + rewrite ^ /index.php; } location ~ ^\/(?:build|tests|config|lib|3rdparty|templates|data)\/ { deny all; @@ -18,13 +18,12 @@ location ~ ^\/(?:\.|autotest|occ|issue|indie|db_|console) { } location ~ ^\/(?:index|remote|public|cron|core\/ajax\/update|status|ocs\/v[12]|updater\/.+|oc[ms]-provider\/.+)\.php(?:$|\/) { fastcgi_split_path_info ^(.+?\.php)(\/.*|)$; + try_files $uri =404; include fastcgi_params; - fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name; fastcgi_param PATH_INFO $fastcgi_path_info; - fastcgi_param HTTPS on; -# Avoid sending the security headers twice + # Avoid sending the security headers twice fastcgi_param modHeadersAvailable true; -# Enable pretty urls + # Enable pretty urls fastcgi_param front_controller_active true; fastcgi_pass {{upstream}}; fastcgi_intercept_errors on; diff --git a/wo/cli/templates/nginx-core.mustache b/wo/cli/templates/nginx-core.mustache index 5de705c..97fc476 100644 --- a/wo/cli/templates/nginx-core.mustache +++ b/wo/cli/templates/nginx-core.mustache @@ -55,12 +55,12 @@ http { ssl_session_cache shared:SSL:50m; ssl_session_tickets off; ssl_prefer_server_ciphers on; - ssl_early_data on; + ssl_early_data on; {{#tls13}}ssl_ciphers 'TLS13+AESGCM+AES256:TLS13+AESGCM+AES128:TLS13+CHACHA20:EECDH+AESGCM:EECDH+CHACHA20'; ssl_protocols TLSv1.2 TLSv1.3;{{/tls13}} ssl_ecdh_curve X25519:P-521:P-384:P-256; - # Previous TLS v1.2 configuration - {{^tls13}}ssl_protocols TLSv1.2; + {{^tls13}}# Previous TLS v1.2 configuration + ssl_protocols TLSv1.2; ssl_ciphers EECDH+CHACHA20:EECDH+AESGCM:EECDH+AES;{{/tls13}} # Common security headers diff --git a/wo/core/acme.py b/wo/core/acme.py index 3d5e04d..9ae31ce 100644 --- a/wo/core/acme.py +++ b/wo/core/acme.py @@ -22,6 +22,7 @@ class WOAcme: self, "{0} ".format(WOAcme.wo_acme_exec) + "--list --listraw > /var/lib/wo/cert.csv"): Log.error(self, "Unable to export certs list") + WOFileUtils.chmod(self, '/var/lib/wo/cert.csv', 0o600) def setupletsencrypt(self, acme_domains, acmedata): """Issue SSL certificates with acme.sh""" @@ -38,6 +39,14 @@ class WOAcme: acme_mode = "-w /var/www/html" validation_mode = "Webroot challenge" Log.debug(self, "Validation : Webroot mode") + if not os.path.isdir('/var/www/html/.well-known/acme-challenge'): + WOFileUtils.mkdir( + self, '/var/www/html/.well-known/acme-challenge') + WOFileUtils.chown( + self, '/var/www/html/.well-known', 'www-data', 'www-data', + recursive=True) + WOFileUtils.chmod(self, '/var/www/html/.well-known', 0o750, + recursive=True) Log.info(self, "Validation mode : {0}".format(validation_mode)) Log.wait(self, "Issuing SSL cert with acme.sh") diff --git a/wo/core/domainvalidate.py b/wo/core/domainvalidate.py index fb4f477..0db491d 100644 --- a/wo/core/domainvalidate.py +++ b/wo/core/domainvalidate.py @@ -25,7 +25,6 @@ class WODomain(): return domain_name - def getlevel(self, domain): """ Returns the domain type : domain, subdomain and the root domain diff --git a/wo/core/fileutils.py b/wo/core/fileutils.py index 6a9b505..94a70a0 100644 --- a/wo/core/fileutils.py +++ b/wo/core/fileutils.py @@ -280,17 +280,19 @@ class WOFileUtils(): """ Searches for string in file and returns True or False. """ - try: - Log.debug(self, "Finding string {0} to file {1}" - .format(sstr, fnm)) + if os.path.isfile(fnm): + try: + Log.debug(self, "Finding string {0} to file {1}" + .format(sstr, fnm)) for line in open(fnm, encoding='utf-8'): if sstr in line: return True return False except OSError as e: - Log.debug(self, "{0}".format(e.strerror)) - Log.error(self, "Unable to Search string {0} in {1}" - .format(sstr, fnm)) + Log.debug(self, "{0}".format(e.strerror)) + Log.error(self, "Unable to Search string {0} in {1}" + .format(sstr, fnm)) + return False def rm(self, path): """ From f3f0573954838752c75ca88a66df466842c9f46a Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Sat, 26 Oct 2019 01:30:52 +0200 Subject: [PATCH 18/79] Fix WOFileUtils --- README.md | 1 + wo/core/fileutils.py | 12 ++++++------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 9dc7748..61df27d 100644 --- a/README.md +++ b/README.md @@ -173,6 +173,7 @@ Apps & Tools shipped with WordOps : - [ClamAV](https://github.com/Cisco-Talos/clamav-devel) - [cheat.sh](https://github.com/chubin/cheat.sh) - [ProFTPd](https://github.com/proftpd/proftpd) +- [nginx-ultimate-bad-bot-blocker](https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/) Cache Plugins supported by WordOps : diff --git a/wo/core/fileutils.py b/wo/core/fileutils.py index 94a70a0..87a3da6 100644 --- a/wo/core/fileutils.py +++ b/wo/core/fileutils.py @@ -280,15 +280,15 @@ class WOFileUtils(): """ Searches for string in file and returns True or False. """ - if os.path.isfile(fnm): + if os.path.isfile('{0}'.format(fnm)): try: Log.debug(self, "Finding string {0} to file {1}" .format(sstr, fnm)) - for line in open(fnm, encoding='utf-8'): - if sstr in line: - return True - return False - except OSError as e: + for line in open(fnm, encoding='utf-8'): + if sstr in line: + return True + return False + except OSError as e: Log.debug(self, "{0}".format(e.strerror)) Log.error(self, "Unable to Search string {0} in {1}" .format(sstr, fnm)) From 179ea7d9cf8618454001100aefa43b9303e4967f Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Sat, 26 Oct 2019 19:28:56 +0200 Subject: [PATCH 19/79] Improve acme process and error log --- CHANGELOG.md | 1 + install | 4 ++-- wo/cli/plugins/stack_upgrade.py | 8 +++++--- wo/core/acme.py | 18 +++++++++--------- 4 files changed, 17 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f00707f..2c5200e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Initial creation of .gitconfig is displayed the first time you run the command `wo` - Added `/var/lib/php/sessions/` to open_basedir to allow php sessions storage - WordOps now check if a repository already exist before trying to adding it again. +- Improved SSL certificate error messages by displaying domain IP and server IP #### Fixed diff --git a/install b/install index 7e09b22..f1a55f1 100755 --- a/install +++ b/install @@ -136,8 +136,8 @@ if [ -z "$wo_branch" ]; then wo_branch=master fi fi -readonly wo_log_dir=/var/log/wo/ -readonly wo_backup_dir=/var/lib/wo-backup/ +readonly wo_log_dir=/var/log/wo +readonly wo_backup_dir=/var/lib/wo-backup readonly wo_tmp_dir=/var/lib/wo/tmp readonly wo_install_log=/var/log/wo/install.log readonly TIME_FORMAT='%d-%b-%Y-%H%M%S' diff --git a/wo/cli/plugins/stack_upgrade.py b/wo/cli/plugins/stack_upgrade.py index e8cc4bf..7b1178d 100644 --- a/wo/cli/plugins/stack_upgrade.py +++ b/wo/cli/plugins/stack_upgrade.py @@ -199,12 +199,12 @@ class WOStackUpgradeController(CementBaseController): Log.wait(self, "Upgrading APT Packages") # additional pre_pref - if ["nginx-custom"] in apt_packages: + if "nginx-custom" in apt_packages: pre_pref(self, WOVar.wo_nginx) - if ["php7.2-fpm"] in apt_packages: + if "php7.2-fpm" in apt_packages: WOAptGet.remove(self, ['php7.2-fpm'], auto=False, purge=True) - if ["php7.3-fpm"] in apt_packages: + if "php7.3-fpm" in apt_packages: WOAptGet.remove(self, ['php7.3-fpm'], auto=False, purge=True) # check if nginx upgrade is blocked @@ -216,6 +216,8 @@ class WOStackUpgradeController(CementBaseController): Log.valide(self, "Upgrading APT Packages") Log.wait(self, "Configuring APT Packages") post_pref(self, apt_packages, [], True) + if "mariadb-server" in apt_packages: + WOShellExec(self, 'mysql_upgrade') Log.valide(self, "Configuring APT Packages") # Post Actions after package updates diff --git a/wo/core/acme.py b/wo/core/acme.py index 9ae31ce..444041e 100644 --- a/wo/core/acme.py +++ b/wo/core/acme.py @@ -72,6 +72,7 @@ class WOAcme: return True def deploycert(self, wo_domain_name): + """Deploy Let's Encrypt certificates with acme.sh""" if not os.path.isfile('/etc/letsencrypt/renewal/{0}_ecc/fullchain.cer' .format(wo_domain_name)): Log.error(self, 'Certificate not found. Deployment canceled') @@ -139,11 +140,13 @@ class WOAcme: .format(domain)).text if(not domain_ip == server_ip): Log.warn( - self, "{0} is not pointing to your server IP" - .format(domain)) + self, "{0}".format(domain) + + " point to the IP {0}".format(domain_ip) + + " but your server IP is {0}.".format(server_ip) + + "\nUse the flag --force to bypass this check.") Log.error( - self, "You have to add the " - "proper DNS record", False) + self, "You have to set the " + "proper DNS record for your domain", False) return False else: Log.debug(self, "DNS record are properly set") @@ -162,9 +165,6 @@ class WOAcme: if wo_domain_name in row[0]: # check if cert expiration exist if not row[3] == '': - cert_exist = True - break - else: - cert_exist = False + return True certfile.close() - return cert_exist + return False From 2ae206fe4a7448b6bd4538ba1cd85429969bc42b Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Sat, 26 Oct 2019 23:29:32 +0200 Subject: [PATCH 20/79] small change in locations-wo --- wo/cli/templates/locations.mustache | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wo/cli/templates/locations.mustache b/wo/cli/templates/locations.mustache index 0d9a88a..d0f1c17 100644 --- a/wo/cli/templates/locations.mustache +++ b/wo/cli/templates/locations.mustache @@ -11,7 +11,7 @@ location @empty_gif { empty_gif; } # Cache static files -location ~* \.(ogg|ogv|svg|svgz|eot|otf|woff|woff2|ttf|m4a|mp4|ttf|rss|atom|jpe?g|gif|cur|heic|png|tiff|ico|webm|mp3|aac|tgz|doc|xls|exe|ppt|tar|mid|midi|wav|bmp|rtf|swf|webp|json|webmanifest)$ { +location ~* \.(ogg|ogv|svg|svgz|eot|otf|woff|woff2|ttf|m4a|mp4|ttf|rss|atom|jpe?g|gif|cur|heic|png|tiff|ico|webm|mp3|aac|tgz|doc|xls|exe|ppt|tar|mid|midi|wav|bmp|rtf|swf|webp|json|webmanifest|cast)$ { more_set_headers 'Access-Control-Allow-Origin : *'; more_set_headers "Cache-Control : public, no-transform"; access_log off; From 2867de7ff14fc65faef1fa25da715b1ad01ea518 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Sun, 27 Oct 2019 17:40:34 +0100 Subject: [PATCH 21/79] Fix mysql_upgrade --- wo/cli/plugins/stack_upgrade.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wo/cli/plugins/stack_upgrade.py b/wo/cli/plugins/stack_upgrade.py index 7b1178d..99ba514 100644 --- a/wo/cli/plugins/stack_upgrade.py +++ b/wo/cli/plugins/stack_upgrade.py @@ -217,7 +217,7 @@ class WOStackUpgradeController(CementBaseController): Log.wait(self, "Configuring APT Packages") post_pref(self, apt_packages, [], True) if "mariadb-server" in apt_packages: - WOShellExec(self, 'mysql_upgrade') + WOShellExec.cmd_exec(self, 'mysql_upgrade') Log.valide(self, "Configuring APT Packages") # Post Actions after package updates From d7cd99f1e8cd4a1bc5e7ea4207a55d91b0f9e038 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Sun, 27 Oct 2019 21:08:50 +0100 Subject: [PATCH 22/79] Add custom motd --- wo/cli/plugins/stack_pref.py | 10 +++++++- wo/cli/templates/wo-update.mustache | 37 +++++++++++++++++++++++++++++ wo/core/fileutils.py | 13 ++++++++++ 3 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 wo/cli/templates/wo-update.mustache diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index 5cd7987..e0de983 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -1468,4 +1468,12 @@ def pre_stack(self): '* soft nofile 500000\n' 'root hard nofile 500000\n' 'root soft nofile 500000\n') - Log.valide(self, 'Applying Linux tweaks') + # custom motd-news + data = dict() + if os.path.isdir('/etc/update-motd.d/'): + if not os.path.isfile('/etc/update-motd.d/98-wo-update'): + WOTemplate.deploy( + self, '/etc/update-motd.d/98-wo-update', + 'wo-update.mustache', data) + WOFileUtils.chmod( + self, "/etc/update-motd.d/98-wo-update", 0o755) diff --git a/wo/cli/templates/wo-update.mustache b/wo/cli/templates/wo-update.mustache new file mode 100644 index 0000000..be5091b --- /dev/null +++ b/wo/cli/templates/wo-update.mustache @@ -0,0 +1,37 @@ +#!/bin/sh +# script to update motd when a new WordOps release is available on Debian/Ubuntu +# the script is added in /etc/update-motd.d + +safe_print() { + cat "$1" | head -n 10 | tr -d '\000-\011\013\014\016-\037' | cut -c -80 +} + +# Ensure sane defaults +[ -n "$URL" ] || URL="https://github.com/WordOps/WordOps/releases/latest" +[ -n "$WAIT" ] || WAIT=5 +[ -n "$CACHE" ] || CACHE="/var/cache/motd-wo" + +# Generate our temp files, clean up when done +NEWS=$(mktemp) || exit 1 +ERR=$(mktemp) || exit 1 +CLOUD=$(mktemp) || exit 1 +trap "rm -f $NEWS $ERR $CLOUD" HUP INT QUIT ILL TRAP BUS TERM + +if [ -n "$(command -v curl > /dev/null 2>&1)" ]; then + LATEST_RELEASE=$(curl -m 5 --retry 3 -sI "$URL" | grep tag | awk -F "/" '{print $8}' 2>&1) +fi +if [ -n "$(command -v wo > /dev/null 2>&1)" ]; then + CURRENT_RELEASE=$(wo -v 2>&1 | grep v | awk -F " " '{print $2}') +fi +if [ -n "$CURRENT_RELEASE" ] && [ -n "$LATEST_RELEASE" ]; then + if [ "$CURRENT_RELEASE" != "$LATEST_RELEASE" ]; then + # display message with motd-news on Ubuntu + echo '*** A new WordOps release is available ***' > "$NEWS" 2> "$ERR" + + echo + # At most, 10 lines of text, remove control characters, print at most 80 characters per line + safe_print "$NEWS" + # Try to update the cache + safe_print "$NEWS" 2> /dev/null > $CACHE || true + fi +fi diff --git a/wo/core/fileutils.py b/wo/core/fileutils.py index 87a3da6..5f9ed35 100644 --- a/wo/core/fileutils.py +++ b/wo/core/fileutils.py @@ -343,3 +343,16 @@ class WOFileUtils(): # If it's not a symlink we're not interested. continue return True + + def writein(self, path, content): + """ + Write content in path + """ + Log.debug(self, "Writing content in {0}".format(path)) + try: + with open("{0}".format(path), + encoding='utf-8', mode='w') as final_file: + final_file.write('{0}'.format(content)) + except IOError as e: + Log.debug(self, "{0}".format(e)) + Log.error(self, "Unable to write content in {0}".format(path)) From 09f0e11cea95a387904e0060deeb50db0919b99d Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Sun, 27 Oct 2019 22:47:28 +0100 Subject: [PATCH 23/79] Fix wo-update motd --- install | 53 ++++++++++++++++------------- wo/cli/plugins/stack_pref.py | 5 +++ wo/cli/templates/wo-update.mustache | 4 +-- 3 files changed, 36 insertions(+), 26 deletions(-) diff --git a/install b/install index f1a55f1..48db4cb 100755 --- a/install +++ b/install @@ -126,31 +126,30 @@ _curl() { curl -m 10 --retry 3 -sL "$@" } -### -# 1 - Define variables for later use -### -if [ -z "$wo_branch" ]; then - if [ "$wo_travis" = "y" ]; then - wo_branch=updating-configuration - else - wo_branch=master +wo_init_variables() { + if [ -z "$wo_branch" ]; then + if [ "$wo_travis" = "y" ]; then + wo_branch=updating-configuration + else + wo_branch=master + fi fi -fi -readonly wo_log_dir=/var/log/wo -readonly wo_backup_dir=/var/lib/wo-backup -readonly wo_tmp_dir=/var/lib/wo/tmp -readonly wo_install_log=/var/log/wo/install.log -readonly TIME_FORMAT='%d-%b-%Y-%H%M%S' -readonly TIME=$(date +"$TIME_FORMAT") -readonly NGINX_BACKUP_FILE="/var/lib/wo-backup/nginx-backup.$TIME.tar.gz" -readonly EE_BACKUP_FILE="/var/lib/wo-backup/ee-backup.$TIME.tar.gz" -readonly WO_BACKUP_FILE="/var/lib/wo-backup/wo-backup.$TIME.tar.gz" + readonly wo_log_dir=/var/log/wo + readonly wo_backup_dir=/var/lib/wo-backup + readonly wo_tmp_dir=/var/lib/wo/tmp + readonly wo_install_log=/var/log/wo/install.log + readonly TIME_FORMAT='%d-%b-%Y-%H%M%S' + readonly TIME=$(date +"$TIME_FORMAT") + readonly NGINX_BACKUP_FILE="/var/lib/wo-backup/nginx-backup.$TIME.tar.gz" + readonly EE_BACKUP_FILE="/var/lib/wo-backup/ee-backup.$TIME.tar.gz" + readonly WO_BACKUP_FILE="/var/lib/wo-backup/wo-backup.$TIME.tar.gz" -if [ -x /usr/local/bin/ee ]; then - ee_migration=1 -elif [ -x /usr/local/bin/wo ]; then - wo_upgrade=1 -fi + if [ -x /usr/local/bin/ee ]; then + ee_migration=1 + elif [ -x /usr/local/bin/wo ]; then + wo_upgrade=1 + fi +} ### # 1 - Checking linux distro @@ -670,6 +669,10 @@ wo_init() { # 4 - WO MAIN SETUP ### +### +# 1 - Define variables for later use +### + # wo uninstall script if [ "$wo_purge" = "y" ]; then _run wo_backup_wo "Backing-up WO install" @@ -677,9 +680,11 @@ if [ "$wo_purge" = "y" ]; then wo_lib_echo "The WordOps backup files can be found in $WO_BACKUP_FILE" exit 0 else + # inital checkup wo_dir_init - _run wo_clean_repo wo_init + wo_init_variables + _run wo_clean_repo wo_check_distro # 1 - WO already installed if [ -x /usr/local/bin/wo ]; then diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index e0de983..d5f9c31 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -1470,10 +1470,15 @@ def pre_stack(self): 'root soft nofile 500000\n') # custom motd-news data = dict() + # check if update-motd.d directory exist if os.path.isdir('/etc/update-motd.d/'): if not os.path.isfile('/etc/update-motd.d/98-wo-update'): + # render custom motd template WOTemplate.deploy( self, '/etc/update-motd.d/98-wo-update', 'wo-update.mustache', data) WOFileUtils.chmod( self, "/etc/update-motd.d/98-wo-update", 0o755) + # restart motd-news service if available + if os.path.isfile('/lib/systemd/system/motd-news.service'): + WOService.restart_service(self, 'motd-news') diff --git a/wo/cli/templates/wo-update.mustache b/wo/cli/templates/wo-update.mustache index be5091b..9ba2e02 100644 --- a/wo/cli/templates/wo-update.mustache +++ b/wo/cli/templates/wo-update.mustache @@ -17,10 +17,10 @@ ERR=$(mktemp) || exit 1 CLOUD=$(mktemp) || exit 1 trap "rm -f $NEWS $ERR $CLOUD" HUP INT QUIT ILL TRAP BUS TERM -if [ -n "$(command -v curl > /dev/null 2>&1)" ]; then +if [ -n "$(command -v curl)" ]; then LATEST_RELEASE=$(curl -m 5 --retry 3 -sI "$URL" | grep tag | awk -F "/" '{print $8}' 2>&1) fi -if [ -n "$(command -v wo > /dev/null 2>&1)" ]; then +if [ -n "$(command -v wo)" ]; then CURRENT_RELEASE=$(wo -v 2>&1 | grep v | awk -F " " '{print $2}') fi if [ -n "$CURRENT_RELEASE" ] && [ -n "$LATEST_RELEASE" ]; then From 3b46f321c9f09ba5c674d5e9a083cfd0ba282c84 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Sun, 27 Oct 2019 22:50:41 +0100 Subject: [PATCH 24/79] Force bash_completion update --- install | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/install b/install index 48db4cb..26ebb1e 100755 --- a/install +++ b/install @@ -427,6 +427,7 @@ wo_install() { python3 -m pip install --upgrade wordops cp -rf /usr/local/lib/python3.*/dist-packages/usr/* /usr/ cp -rn /usr/local/lib/python3.*/dist-packages/etc/* /etc/ + cp -f /usr/local/lib/python3.*/dist-packages/etc/bash_completion.d/wo_auto.rc /etc/bash_completion.d/wo_auto.rc } @@ -445,8 +446,9 @@ wo_travis_install() { else python3 -m pip install git+git://github.com/WordOps/WordOps.git@$wo_branch #egg=wordops -U fi - cp -rn /usr/local/lib/python3.*/dist-packages/usr/* /usr/ + cp -rf /usr/local/lib/python3.*/dist-packages/usr/* /usr/ cp -rn /usr/local/lib/python3.*/dist-packages/etc/* /etc/ + cp -f /usr/local/lib/python3.*/dist-packages/etc/bash_completion.d/wo_auto.rc /etc/bash_completion.d/wo_auto.rc } From 9bf0a78f3d7ac9c5b52a05d5abbd5f598566e074 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 28 Oct 2019 05:14:05 +0100 Subject: [PATCH 25/79] Fix install variables --- install | 57 ++++++++++++++++++++++++++++++++------------------------- 1 file changed, 32 insertions(+), 25 deletions(-) diff --git a/install b/install index 26ebb1e..7124bf2 100755 --- a/install +++ b/install @@ -134,9 +134,6 @@ wo_init_variables() { wo_branch=master fi fi - readonly wo_log_dir=/var/log/wo - readonly wo_backup_dir=/var/lib/wo-backup - readonly wo_tmp_dir=/var/lib/wo/tmp readonly wo_install_log=/var/log/wo/install.log readonly TIME_FORMAT='%d-%b-%Y-%H%M%S' readonly TIME=$(date +"$TIME_FORMAT") @@ -181,6 +178,9 @@ wo_check_distro() { # 1 - To prevent errors or unexpected behaviour, create the log and ACL it ### wo_dir_init() { + local wo_log_dir=/var/log/wo + local wo_backup_dir=/var/lib/wo-backup + local wo_tmp_dir=/var/lib/wo/tmp if [ ! -d "$wo_log_dir" ] || [ ! -d "$wo_backup_dir" ] || [ ! -d "$wo_tmp_dir" ]; then mkdir -p "$wo_backup_dir" "$wo_log_dir" "$wo_tmp_dir" @@ -434,9 +434,6 @@ wo_install() { # Clone Github repository if it doesn't exist wo_travis_install() { - if [ "$wo_force_install" = "y" ]; then - [ ! -f "$HOME/.gitconfig" ] && { bash -c 'echo -e "[user]\n\tname = $USER\n\temail = root@$HOSTNAME.local" > $HOME/.gitconfig'; } - fi if [ -d ./dist ]; then rm -rf dist fi @@ -444,7 +441,7 @@ wo_travis_install() { python3 setup.py sdist bdist_wheel python3 -m pip install --upgrade dist/*.whl else - python3 -m pip install git+git://github.com/WordOps/WordOps.git@$wo_branch #egg=wordops -U + python3 -m pip install -U git+git://github.com/WordOps/WordOps.git@$wo_branch #egg=wordops fi cp -rf /usr/local/lib/python3.*/dist-packages/usr/* /usr/ cp -rn /usr/local/lib/python3.*/dist-packages/etc/* /etc/ @@ -482,9 +479,10 @@ wo_upgrade_nginx() { fi # install new nginx package - if [ -n "$CHECK_NGINX_EE" ]; then - if [ -x /usr/local/bin/wo ]; then - [ -f /etc/apt/preferences.d/nginx-block ] && { mv /etc/apt/preferences.d/nginx-block /var/lib/wo/tmp/nginx-block; } + if { + if [ -n "$CHECK_NGINX_EE" ]; then + if [ -x /usr/local/bin/wo ]; then + [ -f /etc/apt/preferences.d/nginx-block ] && { mv /etc/apt/preferences.d/nginx-block /var/lib/wo/tmp/nginx-block; } # stop nginx service nginx stop # remove previous package @@ -501,18 +499,20 @@ wo_upgrade_nginx() { /usr/local/bin/wo stack install --nginx --php rm -f /etc/nginx/common/acl.conf /etc/nginx/htpasswd-wo /usr/bin/rsync -au --noatime /var/lib/wo-backup/nginx/ /etc/nginx/ - /usr/local/bin/wo stack upgrade --nginx --force + /usr/local/bin/wo stack upgrade --nginx --force + fi fi - fi - # restore sites and configuration - [ -f /etc/nginx/htpasswd-ee ] && { cp -f /etc/nginx/htpasswd-ee /etc/nginx/htpasswd-wo; } - sed -i "s/locations.conf/locations-wo.conf/" /etc/nginx/sites-available/* + }; then + # restore sites and configuration + [ -f /etc/nginx/htpasswd-ee ] && { cp -f /etc/nginx/htpasswd-ee /etc/nginx/htpasswd-wo; } + sed -i "s/locations.conf/locations-wo.conf/" /etc/nginx/sites-available/* sed -i "s/locations-php7.conf/locations-wo.conf/" /etc/nginx/sites-available/* sed -i "s/locations-php71.conf/locations-wo.conf/" /etc/nginx/sites-available/* sed -i "s/locations-php72.conf/locations-wo.conf/" /etc/nginx/sites-available/* - sed -i "s/locations-php73.conf/locations-wo.conf/" /etc/nginx/sites-available/* - sed -i "s/htpasswd-ee/htpasswd-wo/" /etc/nginx/common/acl.conf - sed -i 's/ssl on;/#ssl on;/' /var/www/*/conf/nginx/ssl.conf + sed -i "s/locations-php73.conf/locations-wo.conf/" /etc/nginx/sites-available/* + sed -i "s/htpasswd-ee/htpasswd-wo/" /etc/nginx/common/acl.conf + sed -i 's/ssl on;/#ssl on;/' /var/www/*/conf/nginx/ssl.conf + fi # update redis.conf headers if [ -f /etc/nginx/common/redis.conf ]; then @@ -551,7 +551,7 @@ wo_update_latest() { if [ -f /root/.my.cnf ]; then cp -f /root/.my.cnf /etc/mysql/conf.d/my.cnf chmod 600 /etc/mysql/conf.d/my.cnf - elif [ -f "$HOME/.my.cnf"]; then + elif [ -f "$HOME/.my.cnf" ]; then cp -f "$HOME/.my.cnf" /etc/mysql/conf.d/my.cnf chmod 600 /etc/mysql/conf.d/my.cnf fi @@ -654,11 +654,13 @@ wo_init() { apt-get install lsb-release -qq fi fi - + if [ "$wo_force_install" = "y" ]; then + [ ! -f "$HOME/.gitconfig" ] && { bash -c 'echo -e "[user]\n\tname = $USER\n\temail = root@$HOSTNAME.local" > $HOME/.gitconfig'; } + fi if [ -f ./setup.py ]; then readonly wo_version_new=$(grep "version='" setup.py | awk -F "'" '{print$2}' 2>&1) else - readonly wo_version_new=$(curl -m 10 --retry 3 -sI https://github.com/WordOps/WordOps/releases/latest | grep tag | awk -F "/" '{print $8}' 2>&1) + readonly wo_version_new=$(curl -m 5 --retry 3 -sI https://github.com/WordOps/WordOps/releases/latest | grep tag | awk -F "/" '{print $8}' 2>&1) fi echo "" @@ -675,18 +677,23 @@ wo_init() { # 1 - Define variables for later use ### +# inital setup + +wo_dir_init +wo_init +wo_init_variables +_run wo_clean_repo +wo_check_distro + # wo uninstall script if [ "$wo_purge" = "y" ]; then + _run wo_backup_wo "Backing-up WO install" _run wo_uninstall "Uninstalling WordOps" wo_lib_echo "The WordOps backup files can be found in $WO_BACKUP_FILE" exit 0 else # inital checkup - wo_dir_init - wo_init - wo_init_variables - _run wo_clean_repo wo_check_distro # 1 - WO already installed if [ -x /usr/local/bin/wo ]; then From 2c2b9c28fb5369a765d2b23f66dd2df1fe87c3ef Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 28 Oct 2019 06:52:23 +0100 Subject: [PATCH 26/79] Refactor `wo update` --- CHANGELOG.md | 2 ++ install | 45 +++++++++++++++++++++------------------- setup.py | 2 +- wo/cli/plugins/update.py | 26 ++++++++++++++++++++++- wo/core/download.py | 3 ++- wo/core/variables.py | 2 +- 6 files changed, 55 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c5200e..a2a8028 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Added `/var/lib/php/sessions/` to open_basedir to allow php sessions storage - WordOps now check if a repository already exist before trying to adding it again. - Improved SSL certificate error messages by displaying domain IP and server IP +- Version check before updating WordOps with `wo update` is now directly handled by `wo` +- Refactored WordOps download function with python3-requests #### Fixed diff --git a/install b/install index 7124bf2..bef0b18 100755 --- a/install +++ b/install @@ -483,22 +483,22 @@ wo_upgrade_nginx() { if [ -n "$CHECK_NGINX_EE" ]; then if [ -x /usr/local/bin/wo ]; then [ -f /etc/apt/preferences.d/nginx-block ] && { mv /etc/apt/preferences.d/nginx-block /var/lib/wo/tmp/nginx-block; } - # stop nginx - service nginx stop - # remove previous package - apt-mark unhold nginx-ee nginx-common nginx-custom - apt-get autoremove nginx-ee nginx-common nginx-custom --allow-change-held-packages --purge -qq - # remove previous php-fpm pool configuration - if [ -n "$CHECK_PHP72" ]; then - apt-get purge php7.2-fpm -y -qq - rm -f /etc/php/7.2/fpm/pool.d/{www.conf,www-two.conf,debug.conf} - fi - if [ -d /etc/nginx ]; then - rm -rf /etc/nginx - fi - /usr/local/bin/wo stack install --nginx --php - rm -f /etc/nginx/common/acl.conf /etc/nginx/htpasswd-wo - /usr/bin/rsync -au --noatime /var/lib/wo-backup/nginx/ /etc/nginx/ + # stop nginx + service nginx stop + # remove previous package + apt-mark unhold nginx-ee nginx-common nginx-custom + apt-get autoremove nginx-ee nginx-common nginx-custom --allow-change-held-packages --purge -qq + # remove previous php-fpm pool configuration + if [ -n "$CHECK_PHP72" ]; then + apt-get purge php7.2-fpm -y -qq + rm -f /etc/php/7.2/fpm/pool.d/{www.conf,www-two.conf,debug.conf} + fi + if [ -d /etc/nginx ]; then + rm -rf /etc/nginx + fi + /usr/local/bin/wo stack install --nginx --php + rm -f /etc/nginx/common/acl.conf /etc/nginx/htpasswd-wo + /usr/bin/rsync -au --noatime /var/lib/wo-backup/nginx/ /etc/nginx/ /usr/local/bin/wo stack upgrade --nginx --force fi fi @@ -506,9 +506,9 @@ wo_upgrade_nginx() { # restore sites and configuration [ -f /etc/nginx/htpasswd-ee ] && { cp -f /etc/nginx/htpasswd-ee /etc/nginx/htpasswd-wo; } sed -i "s/locations.conf/locations-wo.conf/" /etc/nginx/sites-available/* - sed -i "s/locations-php7.conf/locations-wo.conf/" /etc/nginx/sites-available/* - sed -i "s/locations-php71.conf/locations-wo.conf/" /etc/nginx/sites-available/* - sed -i "s/locations-php72.conf/locations-wo.conf/" /etc/nginx/sites-available/* + sed -i "s/locations-php7.conf/locations-wo.conf/" /etc/nginx/sites-available/* + sed -i "s/locations-php71.conf/locations-wo.conf/" /etc/nginx/sites-available/* + sed -i "s/locations-php72.conf/locations-wo.conf/" /etc/nginx/sites-available/* sed -i "s/locations-php73.conf/locations-wo.conf/" /etc/nginx/sites-available/* sed -i "s/htpasswd-ee/htpasswd-wo/" /etc/nginx/common/acl.conf sed -i 's/ssl on;/#ssl on;/' /var/www/*/conf/nginx/ssl.conf @@ -653,14 +653,17 @@ wo_init() { if ! command_exists lsb_release; then apt-get install lsb-release -qq fi + if ! command_exists jq; then + apt-get install jq -qq + fi fi if [ "$wo_force_install" = "y" ]; then [ ! -f "$HOME/.gitconfig" ] && { bash -c 'echo -e "[user]\n\tname = $USER\n\temail = root@$HOSTNAME.local" > $HOME/.gitconfig'; } fi if [ -f ./setup.py ]; then - readonly wo_version_new=$(grep "version='" setup.py | awk -F "'" '{print$2}' 2>&1) + readonly wo_version_new=$(grep "version='" setup.py | awk -F "'" '{print $2}' 2>&1) else - readonly wo_version_new=$(curl -m 5 --retry 3 -sI https://github.com/WordOps/WordOps/releases/latest | grep tag | awk -F "/" '{print $8}' 2>&1) + readonly wo_version_new=$(curl -m 5 --retry 3 -sL https://api.github.com/repos/WordOps/WordOps/releases/latest | jq -r '.tag_name' 2>&1) fi echo "" diff --git a/setup.py b/setup.py index bc3b692..21a9b89 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ if os.geteuid() == 0: os.makedirs('/var/lib/wo/tmp/') setup(name='wordops', - version='3.9.9.4', + version='3.10.0', description='An essential toolset that eases server administration', long_description=LONG, long_description_content_type='text/markdown', diff --git a/wo/cli/plugins/update.py b/wo/cli/plugins/update.py index 49e0994..61989c3 100644 --- a/wo/cli/plugins/update.py +++ b/wo/cli/plugins/update.py @@ -2,9 +2,10 @@ import os import time from cement.core.controller import CementBaseController, expose - +from requests import RequestException, get, json from wo.core.download import WODownload from wo.core.logging import Log +from wo.core.variables import WOVar def wo_update_hook(app): @@ -43,6 +44,23 @@ class WOUpdateController(CementBaseController): def default(self): pargs = self.app.pargs filename = "woupdate" + time.strftime("%Y%m%d-%H%M%S") + + wo_current = WOVar.wo_version + try: + wo_github_latest = get( + 'https://api.github.com/repos/WordOps/WordOps/releases/latest', + timeout=(5, 30)).json() + except RequestException: + Log.debug( + self, "Request to GitHub API failed. " + "Switching to Gitea instance") + wo_github_latest = get( + 'https://git.virtubox.net/api/v1/repos/virtubox/WordOps/tags', + timeout=(5, 30)).json() + wo_latest = wo_github_latest[0]["name"] + else: + wo_latest = wo_github_latest["tag_name"] + install_args = "" if pargs.mainline or pargs.beta: wo_branch = "mainline" @@ -52,6 +70,12 @@ class WOUpdateController(CementBaseController): wo_branch = "master" if pargs.force: install_args = install_args + "--force " + else: + if not pargs.travis: + if wo_current == wo_latest: + Log.error( + self, "WordOps {0} is already installed" + .format(wo_latest)) if not os.path.isdir('/var/lib/wo/tmp'): os.makedirs('/var/lib/wo/tmp') diff --git a/wo/core/download.py b/wo/core/download.py index 38d0ede..f81cb42 100644 --- a/wo/core/download.py +++ b/wo/core/download.py @@ -23,7 +23,7 @@ class WODownload(): os.makedirs(directory) Log.info(self, "Downloading {0:20}".format(pkg_name), end=' ') with open(filename, "wb") as out_file: - req = get(url, timeout=15) + req = get(url, timeout=(5, 30)) if req.encoding is None: req.encoding = 'utf-8' out_file.write(req.content) @@ -34,3 +34,4 @@ class WODownload(): Log.error(self, "Unable to download file, {0}" .format(filename)) return False + return 0 diff --git a/wo/core/variables.py b/wo/core/variables.py index 4360ca9..0bafd50 100644 --- a/wo/core/variables.py +++ b/wo/core/variables.py @@ -14,7 +14,7 @@ class WOVar(): """Intialization of core variables""" # WordOps version - wo_version = "3.9.9.4" + wo_version = "3.10.0" # WordOps packages versions wo_wp_cli = "2.3.0" wo_adminer = "4.7.3" From 26ac09677e1d3ad43e76389e6d8c9b45fa231887 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 28 Oct 2019 07:10:31 +0100 Subject: [PATCH 27/79] Fix requests import --- wo/cli/plugins/update.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wo/cli/plugins/update.py b/wo/cli/plugins/update.py index 61989c3..ce901ab 100644 --- a/wo/cli/plugins/update.py +++ b/wo/cli/plugins/update.py @@ -1,8 +1,8 @@ import os import time +import requests from cement.core.controller import CementBaseController, expose -from requests import RequestException, get, json from wo.core.download import WODownload from wo.core.logging import Log from wo.core.variables import WOVar From e3558537b8d7ee71a33c52ff256f78b84f107343 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 28 Oct 2019 07:11:04 +0100 Subject: [PATCH 28/79] Additional fix for requests --- wo/cli/plugins/update.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/wo/cli/plugins/update.py b/wo/cli/plugins/update.py index ce901ab..6fac40c 100644 --- a/wo/cli/plugins/update.py +++ b/wo/cli/plugins/update.py @@ -47,14 +47,14 @@ class WOUpdateController(CementBaseController): wo_current = WOVar.wo_version try: - wo_github_latest = get( + wo_github_latest = requests.get( 'https://api.github.com/repos/WordOps/WordOps/releases/latest', timeout=(5, 30)).json() - except RequestException: + except requests.RequestException: Log.debug( self, "Request to GitHub API failed. " "Switching to Gitea instance") - wo_github_latest = get( + wo_github_latest = requests.get( 'https://git.virtubox.net/api/v1/repos/virtubox/WordOps/tags', timeout=(5, 30)).json() wo_latest = wo_github_latest[0]["name"] From 45252fdd1bc91299e83b28366ca39cb10a3fbbe9 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 28 Oct 2019 09:09:28 +0100 Subject: [PATCH 29/79] Change MySQL backup path --- CHANGELOG.md | 1 + wo/cli/plugins/clean.py | 3 ++- wo/cli/plugins/stack_upgrade.py | 19 ++++++++++--------- wo/core/fileutils.py | 17 +++++++++++++++-- wo/core/mysql.py | 12 ++++++------ 5 files changed, 34 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a2a8028..6a5c1d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Improved SSL certificate error messages by displaying domain IP and server IP - Version check before updating WordOps with `wo update` is now directly handled by `wo` - Refactored WordOps download function with python3-requests +- MySQL backup path changed to `/var/lib/wo-backup/mysql` #### Fixed diff --git a/wo/cli/plugins/clean.py b/wo/cli/plugins/clean.py index 3385876..a7db5f9 100644 --- a/wo/cli/plugins/clean.py +++ b/wo/cli/plugins/clean.py @@ -75,7 +75,8 @@ class WOCleanController(CementBaseController): try: Log.info(self, "Cleaning opcache") opgui = requests.get( - "https://127.0.0.1:22222/cache/opcache/opgui.php?reset=1") + "https://127.0.0.1:22222/cache/opcache/opgui.php?reset=1", + verify=False) if opgui.status_code != '200': Log.warn(self, 'Cleaning opcache failed') except Exception as e: diff --git a/wo/cli/plugins/stack_upgrade.py b/wo/cli/plugins/stack_upgrade.py index 99ba514..976aa75 100644 --- a/wo/cli/plugins/stack_upgrade.py +++ b/wo/cli/plugins/stack_upgrade.py @@ -126,12 +126,12 @@ class WOStackUpgradeController(CementBaseController): if pargs.wpcli: if os.path.isfile('/usr/local/bin/wp'): - packages = packages + [["https://github.com/wp-cli/wp-cli/" - "releases/download/v{0}/" - "wp-cli-{0}.phar" - "".format(WOVar.wo_wp_cli), - "/usr/local/bin/wp", - "WP-CLI"]] + packages = packages + [[ + "https://github.com/wp-cli/wp-cli/" + "releases/download/v{0}/" + "wp-cli-{0}.phar".format(WOVar.wo_wp_cli), + "/usr/local/bin/wp", + "WP-CLI"]] else: Log.info(self, "WPCLI is not installed with WordOps") @@ -168,9 +168,10 @@ class WOStackUpgradeController(CementBaseController): if pargs.composer: if os.path.isfile('/usr/local/bin/composer'): - packages = packages + [["https://getcomposer.org/installer", - "/var/lib/wo/tmp/composer-install", - "Composer"]] + packages = packages + [[ + "https://getcomposer.org/installer", + "/var/lib/wo/tmp/composer-install", + "Composer"]] else: Log.info(self, "Composer isn't installed") diff --git a/wo/core/fileutils.py b/wo/core/fileutils.py index 5f9ed35..7b70f5c 100644 --- a/wo/core/fileutils.py +++ b/wo/core/fileutils.py @@ -344,9 +344,9 @@ class WOFileUtils(): continue return True - def writein(self, path, content): + def textwrite(self, path, content): """ - Write content in path + Write content into a file """ Log.debug(self, "Writing content in {0}".format(path)) try: @@ -356,3 +356,16 @@ class WOFileUtils(): except IOError as e: Log.debug(self, "{0}".format(e)) Log.error(self, "Unable to write content in {0}".format(path)) + + def textappend(self, path, content): + """ + Append content to a file + """ + Log.debug(self, "Writing content in {0}".format(path)) + try: + with open("{0}".format(path), + encoding='utf-8', mode='a') as final_file: + final_file.write('{0}'.format(content)) + except IOError as e: + Log.debug(self, "{0}".format(e)) + Log.error(self, "Unable to write content in {0}".format(path)) diff --git a/wo/core/mysql.py b/wo/core/mysql.py index cd9de53..907a608 100644 --- a/wo/core/mysql.py +++ b/wo/core/mysql.py @@ -70,7 +70,7 @@ class WOMysql(): # Get login details from /etc/mysql/conf.d/my.cnf # & Execute MySQL query connection = WOMysql.connect(self) - log and Log.debug(self, "Exceuting MySQL Statement : {0}" + log and Log.debug(self, "Executing MySQL Statement : {0}" .format(statement)) try: cursor = connection.cursor() @@ -93,12 +93,12 @@ class WOMysql(): import subprocess try: Log.info(self, "Backing up database at location: " - "/var/wo-mysqlbackup") + "/var/lib/wo-backup/mysql") # Setup Nginx common directory - if not os.path.exists('/var/wo-mysqlbackup'): + if not os.path.exists('/var/lib/wo-backup/mysql'): Log.debug(self, 'Creating directory' - '/var/wo-mysqlbackup') - os.makedirs('/var/wo-mysqlbackup') + '/var/lib/wo-backup/mysql') + os.makedirs('/var/lib/wo-backup/mysql') db = subprocess.check_output(["/usr/bin/mysql " "-Bse \'show databases\'"], @@ -114,7 +114,7 @@ class WOMysql(): stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) p2 = subprocess.Popen("/usr/bin/pigz -c > " - "/var/wo-mysqlbackup/{0}{1}.sql.gz" + "/var/lib/wo-backup/mysql/{0}{1}.sql.gz" .format(dbs, WOVar.wo_date), stdin=p1.stdout, shell=True) From c2d27f1480be211657f9d2f3c9b71633ac0ff321 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 28 Oct 2019 09:22:04 +0100 Subject: [PATCH 30/79] Fix small issues detected by codacy --- wo/cli/plugins/debug.py | 5 ++--- wo/cli/plugins/site.py | 3 +-- wo/core/acme.py | 5 ++--- wo/core/domainvalidate.py | 3 +-- 4 files changed, 6 insertions(+), 10 deletions(-) diff --git a/wo/cli/plugins/debug.py b/wo/cli/plugins/debug.py index 1cb171e..ed38c5d 100644 --- a/wo/cli/plugins/debug.py +++ b/wo/cli/plugins/debug.py @@ -14,7 +14,7 @@ from wo.core.fileutils import WOFileUtils from wo.core.logging import Log from wo.core.mysql import WOMysql from wo.core.services import WOService -from wo.core.shellexec import WOShellExec +from wo.core.shellexec import WOShellExec, CommandExecutionError from wo.core.variables import WOVar @@ -703,8 +703,7 @@ class WODebugController(CementBaseController): "-l | sed '/WordOps " "start MySQL slow " "log/,+2d'" - "| crontab -\"" - .format(cron_time)): + "| crontab -\""): Log.error(self, "failed to remove crontab entry") except CommandExecutionError as e: Log.debug(self, str(e)) diff --git a/wo/cli/plugins/site.py b/wo/cli/plugins/site.py index 344d0ea..0319a6d 100644 --- a/wo/cli/plugins/site.py +++ b/wo/cli/plugins/site.py @@ -227,8 +227,7 @@ class WOSiteController(CementBaseController): Log.info(self, Log.ENDC + text) f.close() else: - Log.error(self, "nginx configuration file does not exists" - .format(wo_domain)) + Log.error(self, "nginx configuration file does not exists") @expose(help="Change directory to site webroot") def cd(self): diff --git a/wo/core/acme.py b/wo/core/acme.py index 444041e..eccaaf5 100644 --- a/wo/core/acme.py +++ b/wo/core/acme.py @@ -148,9 +148,8 @@ class WOAcme: self, "You have to set the " "proper DNS record for your domain", False) return False - else: - Log.debug(self, "DNS record are properly set") - return True + Log.debug(self, "DNS record are properly set") + return True def cert_check(self, wo_domain_name): """Check certificate existance with acme.sh and return Boolean""" diff --git a/wo/core/domainvalidate.py b/wo/core/domainvalidate.py index 0db491d..e84a26b 100644 --- a/wo/core/domainvalidate.py +++ b/wo/core/domainvalidate.py @@ -2,7 +2,7 @@ import os -class WODomain(): +class WODomain: """WordOps domain validation utilities""" def validate(self, url): @@ -24,7 +24,6 @@ class WODomain(): return final_domain return domain_name - def getlevel(self, domain): """ Returns the domain type : domain, subdomain and the root domain From 72d6f33b7bf66814317d8256d85ad16f6e61179d Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 28 Oct 2019 10:35:26 +0100 Subject: [PATCH 31/79] Move removeconf into acme --- wo/cli/plugins/site.py | 2 +- wo/cli/plugins/site_functions.py | 58 +------------------------------- wo/core/acme.py | 47 +++++++++++++++++++++++++- 3 files changed, 48 insertions(+), 59 deletions(-) diff --git a/wo/cli/plugins/site.py b/wo/cli/plugins/site.py index 0319a6d..b0f870c 100644 --- a/wo/cli/plugins/site.py +++ b/wo/cli/plugins/site.py @@ -1521,7 +1521,7 @@ class WOSiteUpdateController(CementBaseController): elif (pargs.letsencrypt == "clean" or pargs.letsencrypt == "purge"): - removeAcmeConf(self, wo_domain) + WOAcme.removeconf(self, wo_domain) # find all broken symlinks sympath = "/var/www" WOFileUtils.findBrokenSymlink(self, sympath) diff --git a/wo/cli/plugins/site_functions.py b/wo/cli/plugins/site_functions.py index 4251c7a..d0924e0 100644 --- a/wo/cli/plugins/site_functions.py +++ b/wo/cli/plugins/site_functions.py @@ -1274,62 +1274,6 @@ def removeNginxConf(self, domain): .format(domain)) -def removeAcmeConf(self, domain): - sslconf = ("/var/www/{0}/conf/nginx/ssl.conf" - .format(domain)) - sslforce = ("/etc/nginx/conf.d/force-ssl-{0}.conf" - .format(domain)) - if os.path.isdir('/etc/letsencrypt/renewal/{0}_ecc' - .format(domain)): - Log.info(self, "Removing Acme configuration") - Log.debug(self, "Removing Acme configuration") - try: - WOShellExec.cmd_exec(self, "/etc/letsencrypt/acme.sh " - "--config-home " - "'/etc/letsencrypt/config' " - "--remove " - "-d {0} --ecc" - .format(domain)) - except CommandExecutionError as e: - Log.debug(self, "{0}".format(e)) - Log.error(self, "Cert removal failed") - - WOFileUtils.rm(self, '{0}/{1}_ecc' - .format(WOVar.wo_ssl_archive, domain)) - WOFileUtils.rm(self, '{0}/{1}' - .format(WOVar.wo_ssl_live, domain)) - WOFileUtils.rm(self, '{0}'.format(sslconf)) - WOFileUtils.rm(self, '{0}.disabled'.format(sslconf)) - WOFileUtils.rm(self, '{0}'.format(sslforce)) - WOFileUtils.rm(self, '{0}.disabled' - .format(sslforce)) - WOFileUtils.rm(self, '/etc/letsencrypt/shared/{0}.conf' - .format(domain)) - - # find all broken symlinks - sympath = "/var/www" - WOFileUtils.findBrokenSymlink(self, sympath) - - else: - if os.path.islink("{0}".format(sslconf)): - WOFileUtils.remove_symlink(self, "{0}".format(sslconf)) - WOFileUtils.rm(self, '{0}'.format(sslforce)) - - if WOFileUtils.grepcheck(self, '/var/www/22222/conf/nginx/ssl.conf', - '{0}'.format(domain)): - Log.info(self, "Setting back default certificate for WordOps backend") - with open("/var/www/22222/conf/nginx/" - "ssl.conf", "w") as ssl_conf_file: - ssl_conf_file.write("ssl_certificate " - "/var/www/22222/cert/22222.crt;\n" - "ssl_certificate_key " - "/var/www/22222/cert/22222.key;\n") - WOGit.add(self, ["/etc/letsencrypt"], - msg="Deleted {0} " - .format(domain)) - WOService.restart_service(self, "nginx") - - def doCleanupAction(self, domain='', webroot='', dbname='', dbuser='', dbhost=''): """ @@ -1341,7 +1285,7 @@ def doCleanupAction(self, domain='', webroot='', dbname='', dbuser='', if os.path.isfile('/etc/nginx/sites-available/{0}' .format(domain)): removeNginxConf(self, domain) - removeAcmeConf(self, domain) + WOAcme.removeconf(self, domain) if webroot: deleteWebRoot(self, webroot) diff --git a/wo/core/acme.py b/wo/core/acme.py index eccaaf5..63b0bfa 100644 --- a/wo/core/acme.py +++ b/wo/core/acme.py @@ -6,7 +6,7 @@ import requests from wo.core.fileutils import WOFileUtils from wo.core.git import WOGit from wo.core.logging import Log -from wo.core.shellexec import WOShellExec +from wo.core.shellexec import WOShellExec, CommandExecutionError from wo.core.variables import WOVar @@ -167,3 +167,48 @@ class WOAcme: return True certfile.close() return False + + def removeconf(self, domain): + sslconf = ("/var/www/{0}/conf/nginx/ssl.conf" + .format(domain)) + sslforce = ("/etc/nginx/conf.d/force-ssl-{0}.conf" + .format(domain)) + wo_domain = domain + if WOAcme.cert_check(self, wo_domain): + Log.info(self, "Removing Acme configuration") + Log.debug(self, "Removing Acme configuration") + try: + WOShellExec.cmd_exec( + self, "{0} ".format(WOAcme.wo_acme_exec) + + "--remove -d {0} --ecc".format(domain)) + except CommandExecutionError as e: + Log.debug(self, "{0}".format(e)) + Log.error(self, "Cert removal failed") + WOFileUtils.rm(self, '{0}/{1}_ecc' + .format(WOVar.wo_ssl_archive, domain)) + WOFileUtils.rm(self, '{0}/{1}' + .format(WOVar.wo_ssl_live, domain)) + WOFileUtils.rm(self, '{0}'.format(sslconf)) + WOFileUtils.rm(self, '{0}.disabled'.format(sslconf)) + WOFileUtils.rm(self, '{0}'.format(sslforce)) + WOFileUtils.rm(self, '{0}.disabled' + .format(sslforce)) + WOFileUtils.rm(self, '/etc/letsencrypt/shared/{0}.conf' + .format(domain)) + # find all broken symlinks + WOFileUtils.findBrokenSymlink(self, "/var/www") + else: + if os.path.islink("{0}".format(sslconf)): + WOFileUtils.remove_symlink(self, "{0}".format(sslconf)) + WOFileUtils.rm(self, '{0}'.format(sslforce)) + + if WOFileUtils.grepcheck(self, '/var/www/22222/conf/nginx/ssl.conf', + '{0}'.format(domain)): + Log.info( + self, "Setting back default certificate for WordOps backend") + with open("/var/www/22222/conf/nginx/" + "ssl.conf", "w") as ssl_conf_file: + ssl_conf_file.write("ssl_certificate " + "/var/www/22222/cert/22222.crt;\n" + "ssl_certificate_key " + "/var/www/22222/cert/22222.key;\n") From 1e5e07e2771d408f0748396b4db631507d25a3ad Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 28 Oct 2019 11:15:53 +0100 Subject: [PATCH 32/79] Simplify acme --- wo/core/acme.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/wo/core/acme.py b/wo/core/acme.py index 63b0bfa..5b76eb5 100644 --- a/wo/core/acme.py +++ b/wo/core/acme.py @@ -173,6 +173,13 @@ class WOAcme: .format(domain)) sslforce = ("/etc/nginx/conf.d/force-ssl-{0}.conf" .format(domain)) + acmedir = [ + '{0}'.format(sslforce), '{0}'.format(sslconf), + '{0}/{1}_ecc'.format(WOVar.wo_ssl_archive, domain), + '{0}.disabled'.format(sslconf), '{0}.disabled' + .format(sslforce), '{0}/{1}' + .format(WOVar.wo_ssl_live, domain), + '/etc/letsencrypt/shared/{0}.conf'.format(domain)] wo_domain = domain if WOAcme.cert_check(self, wo_domain): Log.info(self, "Removing Acme configuration") @@ -184,17 +191,10 @@ class WOAcme: except CommandExecutionError as e: Log.debug(self, "{0}".format(e)) Log.error(self, "Cert removal failed") - WOFileUtils.rm(self, '{0}/{1}_ecc' - .format(WOVar.wo_ssl_archive, domain)) - WOFileUtils.rm(self, '{0}/{1}' - .format(WOVar.wo_ssl_live, domain)) - WOFileUtils.rm(self, '{0}'.format(sslconf)) - WOFileUtils.rm(self, '{0}.disabled'.format(sslconf)) - WOFileUtils.rm(self, '{0}'.format(sslforce)) - WOFileUtils.rm(self, '{0}.disabled' - .format(sslforce)) - WOFileUtils.rm(self, '/etc/letsencrypt/shared/{0}.conf' - .format(domain)) + # remove all files and directories + for dir in acmedir: + if os.path.exists('{0}'.format(dir)): + WOFileUtils.rm(self, '{0}'.format(dir)) # find all broken symlinks WOFileUtils.findBrokenSymlink(self, "/var/www") else: From 4a7bcfbf4b5af8b1813468e6cd14020040932f3a Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 28 Oct 2019 11:52:32 +0100 Subject: [PATCH 33/79] Update command to get latest WordOps version with curl --- README.md | 2 +- wo/cli/templates/wo-update.mustache | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 61df27d..f7d0980 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ MIT Commits GitHub release -
WordOps install +
PyPI - Downloads codacy Badge Twitter Badge Rocket.chat diff --git a/wo/cli/templates/wo-update.mustache b/wo/cli/templates/wo-update.mustache index 9ba2e02..6ad2c01 100644 --- a/wo/cli/templates/wo-update.mustache +++ b/wo/cli/templates/wo-update.mustache @@ -7,7 +7,7 @@ safe_print() { } # Ensure sane defaults -[ -n "$URL" ] || URL="https://github.com/WordOps/WordOps/releases/latest" +[ -n "$URL" ] || URL="https://api.github.com/repos/WordOps/WordOps/releases/latest" [ -n "$WAIT" ] || WAIT=5 [ -n "$CACHE" ] || CACHE="/var/cache/motd-wo" @@ -18,7 +18,7 @@ CLOUD=$(mktemp) || exit 1 trap "rm -f $NEWS $ERR $CLOUD" HUP INT QUIT ILL TRAP BUS TERM if [ -n "$(command -v curl)" ]; then - LATEST_RELEASE=$(curl -m 5 --retry 3 -sI "$URL" | grep tag | awk -F "/" '{print $8}' 2>&1) + LATEST_RELEASE=$(curl -m 5 --retry 3 -sL "$URL" | jq -r '.tag_name' 2>&1) fi if [ -n "$(command -v wo)" ]; then CURRENT_RELEASE=$(wo -v 2>&1 | grep v | awk -F " " '{print $2}') From 2d21ba82e6c97274261ffeed7ef5661f785885c9 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 28 Oct 2019 14:53:15 +0100 Subject: [PATCH 34/79] Fix wo clean issue * add `wo update --mainline` * move mariadb tweaks into wo --- CHANGELOG.md | 4 +- install | 84 ++++++++++++++++-------------------- wo/cli/plugins/clean.py | 3 +- wo/cli/plugins/stack_pref.py | 11 +++++ wo/cli/plugins/update.py | 57 ++++++++++-------------- wo/core/download.py | 24 +++++++++-- 6 files changed, 94 insertions(+), 89 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a5c1d2..e67e8bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), #### Changed - Sysctl tweaks are applied during stack install and removed from install script -- Nginx systemd tweaks during Nginx install/upgrade and removed from install script +- Nginx & MariaDB systemd tweaks are removed from install script and applied during stacks install/upgrade - Initial creation of .gitconfig is displayed the first time you run the command `wo` - Added `/var/lib/php/sessions/` to open_basedir to allow php sessions storage - WordOps now check if a repository already exist before trying to adding it again. @@ -30,6 +30,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), #### Fixed - Incorrect PHP-FPM log path is `wo log` +- force-ssl.conf not removed after removing a site +- `wo clean --opcache` not working with invalid SSL certificate ### v3.9.9.4 - 2019-10-18 diff --git a/install b/install index bef0b18..a65e36c 100755 --- a/install +++ b/install @@ -9,7 +9,7 @@ # ------------------------------------------------------------------------- # wget -qO wo wops.cc && sudo bash wo # ------------------------------------------------------------------------- -# Version 3.10.0 - 2019-10-25 +# Version 3.10.0 - 2019-10-28 # ------------------------------------------------------------------------- # CONTENTS @@ -70,6 +70,9 @@ while [ "$#" -gt 0 ]; do wo_travis="y" wo_force_install="y" ;; + --mainline | --beta) + wo_branch="mainline" + ;; -s | --silent) wo_force_install="y" ;; @@ -102,7 +105,7 @@ export LC_ALL='C.UTF-8' # check if a command exist command_exists() { - command -v "$@" > /dev/null 2>&1 + command -v "$@" >/dev/null 2>&1 } # run functions and exit on failure @@ -110,7 +113,7 @@ _run() { if [ -n "$2" ]; then echo -ne "${TPUT_ECHO}${2}${TPUT_RESET}\t" fi - if ! { "$1" >> "$wo_install_log" 2>&1; }; then + if ! { "$1" >>"$wo_install_log" 2>&1; }; then if [ -n "$2" ]; then echo -e "${TPUT_FAIL}[KO]${TPUT_RESET}" fi @@ -205,13 +208,13 @@ wo_install_dep() { # install dependencies apt-get -option=Dpkg::options::=--force-confmiss --option=Dpkg::options::=--force-confold --assume-yes install \ build-essential curl gzip python3-pip python3-wheel python3-apt python3-setuptools python3-dev sqlite3 git tar software-properties-common pigz \ - gnupg2 cron ccze rsync apt-transport-https tree haveged ufw unattended-upgrades tzdata ntp > /dev/null 2>&1 + gnupg2 cron ccze rsync apt-transport-https tree haveged ufw unattended-upgrades tzdata ntp >/dev/null 2>&1 curl -sL https://download.opensuse.org/repositories/home:/virtubox:/WordOps/xUbuntu_18.04/Release.key | apt-key add - else # install dependencies apt-get -option=Dpkg::options::=--force-confmiss --option=Dpkg::options::=--force-confold --assume-yes install \ build-essential curl gzip dirmngr sudo python3-pip python3-wheel python3-apt python3-setuptools python3-dev ca-certificates sqlite3 git tar \ - software-properties-common pigz apt-transport-https gnupg2 cron ccze rsync tree haveged ufw unattended-upgrades tzdata ntp > /dev/null 2>&1 + software-properties-common pigz apt-transport-https gnupg2 cron ccze rsync tree haveged ufw unattended-upgrades tzdata ntp >/dev/null 2>&1 # add php repository gpg key [ -d /etc/apt/trusted.gpg.d ] && { wget -qO /etc/apt/trusted.gpg.d/php.gpg https://packages.sury.org/php/apt.gpg; } # add nginx repository gpg key @@ -248,17 +251,16 @@ wo_sync_db() { # Switching from EE -> WO ### if [ ! -f /var/lib/wo/dbase.db ]; then - # Create the WordOps folder - mkdir -p /var/lib/wo if [ -f /var/lib/ee/ee.db ]; then + # Make a backup of the EasyEngine database cp /var/lib/ee/ee.db /var/lib/wo/dbase-ee.db # Copy ee database cp /var/lib/ee/ee.db /var/lib/wo/dbase.db else - if [ -d /etc/nginx/sites-available ]; then + if [ -d /etc/nginx/sites-available ] && [ -d /var/www ]; then # Create an empty database for WordOps echo "CREATE TABLE sites ( @@ -423,8 +425,11 @@ wo_install_acme_sh() { # WordOps install wo_install() { - - python3 -m pip install --upgrade wordops + if [ "$wo_branch" = "master" ]; then + python3 -m pip install --upgrade wordops + else + python3 -m pip install -U "git+git://github.com/WordOps/WordOps.git@$wo_branch#egg=wordops" + fi cp -rf /usr/local/lib/python3.*/dist-packages/usr/* /usr/ cp -rn /usr/local/lib/python3.*/dist-packages/etc/* /etc/ cp -f /usr/local/lib/python3.*/dist-packages/etc/bash_completion.d/wo_auto.rc /etc/bash_completion.d/wo_auto.rc @@ -441,7 +446,7 @@ wo_travis_install() { python3 setup.py sdist bdist_wheel python3 -m pip install --upgrade dist/*.whl else - python3 -m pip install -U git+git://github.com/WordOps/WordOps.git@$wo_branch #egg=wordops + python3 -m pip install -U "git+git://github.com/WordOps/WordOps.git@$wo_branch#egg=wordops" fi cp -rf /usr/local/lib/python3.*/dist-packages/usr/* /usr/ cp -rn /usr/local/lib/python3.*/dist-packages/etc/* /etc/ @@ -586,16 +591,7 @@ wo_remove_ee_cron() { } wo_domain_suffix() { - _curl https://raw.githubusercontent.com/publicsuffix/list/master/public_suffix_list.dat | sed '/^\/\//d' | sed '/^$/d' | sed 's/^\s+//g' > /var/lib/wo/public_suffix_list.dat -} - -wo_mariadb_tweak() { - # increase mariadb open_files_limit - if [ -d /etc/systemd/system/mariadb.service.d ] && [ ! -f /etc/systemd/system/mariadb.service.d/limits.conf ]; then - echo -e '[Service]\nLimitNOFILE=500000' > /etc/systemd/system/mariadb.service.d/limits.conf - systemctl daemon-reload - service mysql restart - fi + _curl https://raw.githubusercontent.com/publicsuffix/list/master/public_suffix_list.dat | sed '/^\/\//d' | sed '/^$/d' | sed 's/^\s+//g' >/var/lib/wo/public_suffix_list.dat } wo_clean() { @@ -604,33 +600,36 @@ wo_clean() { } wo_uninstall() { + if { python3 -m pip list | grep -q "wordops" >/dev/null 2>&1; }; then + python3 -m pip uninstall wordops -y + fi rm -rf /usr/local/lib/python3.*/dist-packages/{pystache-*,cement-2.*,wo-*,wordops-*} /usr/local/bin/wo /etc/bash_completion.d/wo_auto.rc /var/lib/wo /etc/wo /usr/lib/wo/templates } wo_cheat_install() { - curl -sL https://cht.sh/:cht.sh > /usr/local/bin/cht.sh + curl -sL https://cht.sh/:cht.sh >/usr/local/bin/cht.sh chmod +x /usr/local/bin/cht.sh [ ! -h /usr/local/bin/cheat ] && { rm -f /usr/local/bin/cheat ln -s /usr/local/bin/cht.sh /usr/local/bin/cheat } - curl -sL https://cheat.sh/:bash_completion > /etc/bash_completion.d/cht.sh + curl -sL https://cheat.sh/:bash_completion >/etc/bash_completion.d/cht.sh } wo_clean_repo() { # remove old EasyEngine Nginx repository if [ -f /etc/apt/sources.list.d/ee-repo.list ]; then cp -f /etc/apt/sources.list.d/ee-repo.list /etc/apt/sources.list.d/ee-repo.list.save - grep -v "/home:/rtCamp:/EasyEngine" /etc/apt/sources.list.d/ee-repo.list.save > /etc/apt/sources.list.d/ee-repo.list + grep -v "/home:/rtCamp:/EasyEngine" /etc/apt/sources.list.d/ee-repo.list.save >/etc/apt/sources.list.d/ee-repo.list fi if [ -f /etc/apt/sources.list.d/wo-repo.list ]; then local wo_linux_distro wo_linux_distro=$(lsb_release -is) cp -f /etc/apt/sources.list.d/wo-repo.list /etc/apt/sources.list.d/wo-repo.list.save if [ "$wo_linux_distro" = "Ubuntu" ]; then - grep -v "opensuse" /etc/apt/sources.list.d/wo-repo.list.save > /etc/apt/sources.list.d/wo-repo.list + grep -v "opensuse" /etc/apt/sources.list.d/wo-repo.list.save >/etc/apt/sources.list.d/wo-repo.list else - grep -v "/home:/rtCamp:/EasyEngine" /etc/apt/sources.list.d/wo-repo.list.save > /etc/apt/sources.list.d/wo-repo.list + grep -v "/home:/rtCamp:/EasyEngine" /etc/apt/sources.list.d/wo-repo.list.save >/etc/apt/sources.list.d/wo-repo.list fi fi } @@ -643,12 +642,12 @@ wo_init() { if [ -z "$wo_travis" ]; then if ! { - apt-get update --allow-releaseinfo-change -qq > /dev/null 2>&1 + apt-get update --allow-releaseinfo-change -qq >/dev/null 2>&1 }; then - apt-get update -qq > /dev/null 2>&1 + apt-get update -qq >/dev/null 2>&1 fi if ! command_exists curl; then - apt-get -y install curl -qq > /dev/null 2>&1 + apt-get -y install curl -qq >/dev/null 2>&1 fi if ! command_exists lsb_release; then apt-get install lsb-release -qq @@ -676,35 +675,26 @@ wo_init() { # 4 - WO MAIN SETUP ### -### -# 1 - Define variables for later use -### - -# inital setup - +# create required directories wo_dir_init +# install lsb_release, curl and display header wo_init +# define main variables wo_init_variables +# remove old repositories _run wo_clean_repo +# check distribution support wo_check_distro # wo uninstall script if [ "$wo_purge" = "y" ]; then - _run wo_backup_wo "Backing-up WO install" _run wo_uninstall "Uninstalling WordOps" wo_lib_echo "The WordOps backup files can be found in $WO_BACKUP_FILE" exit 0 else - # inital checkup - wo_check_distro # 1 - WO already installed if [ -x /usr/local/bin/wo ]; then - if [ -z "$wo_force_install" ]; then - if { wo -v 2>&1 | grep -q "$wo_version_new"; }; then - wo_lib_error "You already have WordOps $wo_version_new" 1 - fi - fi _run wo_clean # 2 - Migration from EEv3 elif [ -x /usr/local/bin/ee ]; then @@ -716,13 +706,13 @@ else fi _run wo_backup_ee "Backing-up EE install" _run wo_remove_ee_cron "Removing EasyEngine cronjob" - _run wo_sync_db "Syncing WO database" fi _run wo_install_dep "Installing wo dependencies" _run wo_timesync # skip steps if travis if [ -z "$wo_travis" ]; then #_run wo_download "Downloading WordOps" + _run wo_sync_db _run wo_install "Installing WordOps" else _run wo_travis_install "Installing WordOps" @@ -732,16 +722,13 @@ else _run wo_clean_ee "Cleaning previous EasyEngine install" fi _run wo_install_acme_sh - if [ -d /etc/systemd/system/mariadb.service.d ]; then - _run wo_mariadb_tweak - fi _run wo_cheat_install "Running post-install steps" _run wo_domain_suffix _run wo_update_wp_cli _run wo_update_latest _run secure_wo_db - wo sync >> $wo_install_log 2>&1 + wo sync >>$wo_install_log 2>&1 if [ "$ee_migration" = "1" ]; then echo @@ -771,6 +758,7 @@ else echo wo_lib_echo "WordOps Documentation : https://docs.wordops.net" wo_lib_echo "WordOps Community Forum : https://community.wordops.net" + wo_lib_echo "WordOps Community Chat : https://chat.wordops.net" echo wo_lib_echo "Give WordOps a GitHub star : https://github.com/WordOps/WordOps/" echo diff --git a/wo/cli/plugins/clean.py b/wo/cli/plugins/clean.py index a7db5f9..3385876 100644 --- a/wo/cli/plugins/clean.py +++ b/wo/cli/plugins/clean.py @@ -75,8 +75,7 @@ class WOCleanController(CementBaseController): try: Log.info(self, "Cleaning opcache") opgui = requests.get( - "https://127.0.0.1:22222/cache/opcache/opgui.php?reset=1", - verify=False) + "https://127.0.0.1:22222/cache/opcache/opgui.php?reset=1") if opgui.status_code != '200': Log.warn(self, 'Cleaning opcache failed') except Exception as e: diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index d5f9c31..1b85165 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -835,8 +835,19 @@ def post_pref(self, apt_packages, packages, upgrade=False): self, '/etc/mysql/my.cnf', 'my.mustache', data) # replacing default values Log.debug(self, "Tuning MySQL configuration") + if os.path.isdir('/etc/systemd/system/mariadb.service.d'): + if not os.path.isfile( + '/etc/systemd/system/' + 'mariadb.service.d/limits.conf'): + WOFileUtils.textwrite( + self, + '/etc/systemd/system/' + 'mariadb.service.d/limits.conf', + '[Service]\nLimitNOFILE=500000') + WOShellExec.cmd_exec(self, 'systemctl daemon-reload') # set innodb_buffer_pool_instances depending # on the amount of RAM + WOService.stop_service(self, 'mysql') WOFileUtils.mvfile(self, '/var/lib/mysql/ib_logfile0', '/var/lib/mysql/ib_logfile0.bak') diff --git a/wo/cli/plugins/update.py b/wo/cli/plugins/update.py index 6fac40c..606da81 100644 --- a/wo/cli/plugins/update.py +++ b/wo/cli/plugins/update.py @@ -1,7 +1,7 @@ import os import time -import requests + from cement.core.controller import CementBaseController, expose from wo.core.download import WODownload from wo.core.logging import Log @@ -46,36 +46,28 @@ class WOUpdateController(CementBaseController): filename = "woupdate" + time.strftime("%Y%m%d-%H%M%S") wo_current = WOVar.wo_version - try: - wo_github_latest = requests.get( - 'https://api.github.com/repos/WordOps/WordOps/releases/latest', - timeout=(5, 30)).json() - except requests.RequestException: - Log.debug( - self, "Request to GitHub API failed. " - "Switching to Gitea instance") - wo_github_latest = requests.get( - 'https://git.virtubox.net/api/v1/repos/virtubox/WordOps/tags', - timeout=(5, 30)).json() - wo_latest = wo_github_latest[0]["name"] - else: - wo_latest = wo_github_latest["tag_name"] + wo_latest = WODownload.latest_release(self, "WordOps/WordOps") install_args = "" if pargs.mainline or pargs.beta: wo_branch = "mainline" + install_args = install_args + "--mainline " elif pargs.branch: wo_branch = pargs.branch - else: - wo_branch = "master" + install_args = install_args + "-b {0} ".format(wo_branch) if pargs.force: install_args = install_args + "--force " - else: - if not pargs.travis: - if wo_current == wo_latest: - Log.error( - self, "WordOps {0} is already installed" - .format(wo_latest)) + if pargs.travis: + install_args = install_args + "--travis " + wo_branch = "updating-configuration" + + if ((not pargs.force) and (not pargs.travis) and + (not pargs.mainline) and (not pargs.beta) and + (not pargs.branch)): + if wo_current == wo_latest: + Log.error( + self, "WordOps {0} is already installed" + .format(wo_latest)) if not os.path.isdir('/var/lib/wo/tmp'): os.makedirs('/var/lib/wo/tmp') @@ -85,21 +77,18 @@ class WOUpdateController(CementBaseController): "/var/lib/wo/tmp/{0}".format(filename), "update script"]]) - if pargs.travis: - if os.path.isfile('install'): - try: - Log.info(self, "updating WordOps, please wait...") - os.system("/bin/bash install --travis " - "--force") - except OSError as e: - Log.debug(self, str(e)) - Log.error(self, "WordOps update failed !") + if os.path.isfile('install'): + try: + Log.info(self, "updating WordOps, please wait...") + os.system("/bin/bash install --travis") + except OSError as e: + Log.debug(self, str(e)) + Log.error(self, "WordOps update failed !") else: try: Log.info(self, "updating WordOps, please wait...") os.system("/bin/bash /var/lib/wo/tmp/{0} " - "-b {1} {2}".format(filename, - wo_branch, install_args)) + "{1}".format(filename, install_args)) except OSError as e: Log.debug(self, str(e)) Log.error(self, "WordOps update failed !") diff --git a/wo/core/download.py b/wo/core/download.py index f81cb42..bd7a725 100644 --- a/wo/core/download.py +++ b/wo/core/download.py @@ -1,6 +1,6 @@ """WordOps download core classes.""" import os -from requests import get, RequestException +import requests from wo.core.logging import Log @@ -11,7 +11,7 @@ class WODownload(): pass def download(self, packages): - """Download packages, packges must be list in format of + """Download packages, packages must be list in format of [url, path, package name]""" for package in packages: url = package[0] @@ -23,15 +23,31 @@ class WODownload(): os.makedirs(directory) Log.info(self, "Downloading {0:20}".format(pkg_name), end=' ') with open(filename, "wb") as out_file: - req = get(url, timeout=(5, 30)) + req = requests.get(url, timeout=(5, 30)) if req.encoding is None: req.encoding = 'utf-8' out_file.write(req.content) Log.info(self, "{0}".format("[" + Log.ENDC + "Done" + Log.OKBLUE + "]")) - except RequestException as e: + except requests.RequestException as e: Log.debug(self, "[{err}]".format(err=str(e.reason))) Log.error(self, "Unable to download file, {0}" .format(filename)) return False return 0 + + def latest_release(self, repository): + """Get the latest release number of a GitHub repository.\n + repository format should be: \"user/repo\"""" + try: + req = requests.get( + 'https://api.github.com/repos/{0}/releases/latest' + .format(repository), + timeout=(5, 30)) + except requests.RequestException as e: + Log.debug(self, str(e)) + Log.error(self, "Unable to query GitHub API") + else: + github_json = req.json() + github_release = github_json["tag_name"] + return github_release From a4c7d615ab99b73d0def8aa05e2b2a20516b1c01 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 28 Oct 2019 17:56:19 +0100 Subject: [PATCH 35/79] Fix small errors in install script --- install | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/install b/install index a65e36c..a0222bf 100755 --- a/install +++ b/install @@ -650,10 +650,10 @@ wo_init() { apt-get -y install curl -qq >/dev/null 2>&1 fi if ! command_exists lsb_release; then - apt-get install lsb-release -qq + apt-get install lsb-release -qq > /dev/null 2>&1 fi if ! command_exists jq; then - apt-get install jq -qq + apt-get install jq -qq > /dev/null 2>&1 fi fi if [ "$wo_force_install" = "y" ]; then @@ -662,11 +662,11 @@ wo_init() { if [ -f ./setup.py ]; then readonly wo_version_new=$(grep "version='" setup.py | awk -F "'" '{print $2}' 2>&1) else - readonly wo_version_new=$(curl -m 5 --retry 3 -sL https://api.github.com/repos/WordOps/WordOps/releases/latest | jq -r '.tag_name' 2>&1) + readonly wo_version_new=$(curl -m 5 --retry 3 -sL https://api.github.com/repos/WordOps/WordOps/releases/latest 2>&1 | jq -r '.tag_name' ) fi echo "" - wo_lib_echo "Welcome to WordOps install/update script v${wo_version_new}" + wo_lib_echo "Welcome to WordOps install/update script ${wo_version_new}" echo "" } From 5b9a17c1168aad3980f49f03c84335629b952582 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 28 Oct 2019 20:07:38 +0100 Subject: [PATCH 36/79] Fix `wo stack install --cheat` --- CHANGELOG.md | 1 + install | 19 ++++------------- wo/cli/plugins/stack.py | 40 ++++++++++++++++++++++++++++++++++++ wo/cli/plugins/stack_pref.py | 25 +++++++++++++++++----- wo/utils/test.py | 2 +- 5 files changed, 66 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e67e8bd..9a87c2e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Incorrect PHP-FPM log path is `wo log` - force-ssl.conf not removed after removing a site - `wo clean --opcache` not working with invalid SSL certificate +- `wo stack install --cheat` wasn't working properly previously ### v3.9.9.4 - 2019-10-18 diff --git a/install b/install index a0222bf..2dd3a85 100755 --- a/install +++ b/install @@ -606,16 +606,6 @@ wo_uninstall() { rm -rf /usr/local/lib/python3.*/dist-packages/{pystache-*,cement-2.*,wo-*,wordops-*} /usr/local/bin/wo /etc/bash_completion.d/wo_auto.rc /var/lib/wo /etc/wo /usr/lib/wo/templates } -wo_cheat_install() { - curl -sL https://cht.sh/:cht.sh >/usr/local/bin/cht.sh - chmod +x /usr/local/bin/cht.sh - [ ! -h /usr/local/bin/cheat ] && { - rm -f /usr/local/bin/cheat - ln -s /usr/local/bin/cht.sh /usr/local/bin/cheat - } - curl -sL https://cheat.sh/:bash_completion >/etc/bash_completion.d/cht.sh -} - wo_clean_repo() { # remove old EasyEngine Nginx repository if [ -f /etc/apt/sources.list.d/ee-repo.list ]; then @@ -721,8 +711,7 @@ else _run wo_upgrade_nginx "Upgrading Nginx" _run wo_clean_ee "Cleaning previous EasyEngine install" fi - _run wo_install_acme_sh - _run wo_cheat_install "Running post-install steps" + _run wo_install_acme_sh "Running post-install steps" _run wo_domain_suffix _run wo_update_wp_cli _run wo_update_latest @@ -738,10 +727,10 @@ else elif [ "$wo_upgrade" = "1" ]; then wo_lib_echo "WordOps (wo) upgrade to $wo_version_new was succesfull!" echo - wo_lib_echo "To upgrade WordOps stacks use the command:" - wo_lib_echo_info "wo stack upgrade --all" + wo_lib_echo "To upgrade WordOps web stacks use the command:" + wo_lib_echo_info "wo stack upgrade" echo - wo_lib_echo "To update all other server packages use the command:" + wo_lib_echo "To update all other packages use the command:" wo_lib_echo_info "wo maintenance" else wo_lib_echo "WordOps (wo) installed successfully" diff --git a/wo/cli/plugins/stack.py b/wo/cli/plugins/stack.py index de3c606..f152104 100644 --- a/wo/cli/plugins/stack.py +++ b/wo/cli/plugins/stack.py @@ -85,6 +85,8 @@ class WOStackController(CementBaseController): (['--ngxblocker'], dict(help='Install Nginx Ultimate Bad Bot Blocker', action='store_true')), + (['--cheat'], + dict(help='Install cheat.sh', action='store_true')), (['--force'], dict(help='Force install/remove/purge without prompt', action='store_true')), @@ -116,6 +118,7 @@ class WOStackController(CementBaseController): (not pargs.adminer) and (not pargs.utils) and (not pargs.redis) and (not pargs.proftpd) and (not pargs.extplorer) and (not pargs.clamav) and + (not pargs.cheat) and (not pargs.ufw) and (not pargs.ngxblocker) and (not pargs.phpredisadmin) and (not pargs.sendmail) and (not pargs.php73)): @@ -147,6 +150,7 @@ class WOStackController(CementBaseController): pargs.dashboard = True pargs.phpredisadmin = True pargs.extplorer = True + pargs.cheat = True if pargs.security: pargs.fail2ban = True @@ -426,6 +430,7 @@ class WOStackController(CementBaseController): Log.debug(self, "eXtplorer is already installed") Log.info(self, "eXtplorer is already installed") + # ultimate ngx_blocker if pargs.ngxblocker: if not os.path.isdir('/etc/nginx/bots.d'): Log.debug(self, "Setting packages variable for ngxblocker") @@ -439,6 +444,21 @@ class WOStackController(CementBaseController): Log.debug(self, "ngxblocker is already installed") Log.info(self, "ngxblocker is already installed") + # cheat.sh + if pargs.cheat: + if ((not os.path.exists('/usr/local/bin/cht.sh')) and + (not os.path.exists('/usr/bin/cht.sh'))): + Log.debug(self, 'Setting packages variable for cheat.sh') + packages = packages + [[ + "https://raw.githubusercontent.com/chubin/cheat.sh" + "/master/share/cht.sh.txt", + "/usr/local/bin/cht.sh", + "cheat.sh"], + ["https://raw.githubusercontent.com/chubin/cheat.sh" + "/master/share/bash_completion.txt", + "/etc/bash_completion.d/cht.sh", + "cheat.sh bash_completion"]] + # UTILS if pargs.utils: Log.debug(self, "Setting packages variable for utils") @@ -536,6 +556,7 @@ class WOStackController(CementBaseController): (not pargs.adminer) and (not pargs.utils) and (not pargs.redis) and (not pargs.proftpd) and (not pargs.extplorer) and (not pargs.clamav) and + (not pargs.cheat) and (not pargs.ufw) and (not pargs.ngxblocker) and (not pargs.phpredisadmin) and (not pargs.sendmail) and (not pargs.php73)): @@ -565,6 +586,7 @@ class WOStackController(CementBaseController): pargs.utils = True pargs.netdata = True pargs.mysqltuner = True + pargs.cheat = True if pargs.security: pargs.fail2ban = True @@ -674,6 +696,14 @@ class WOStackController(CementBaseController): Log.debug(self, "Removing packages for MySQLTuner ") packages = packages + ['/usr/bin/mysqltuner'] + # cheat.sh + if pargs.cheat: + if os.path.isfile('/usr/local/bin/cht.sh'): + Log.debug(self, "Removing packages for cheat.sh ") + packages = packages + [ + '/usr/local/bin/cht.sh', '/usr/local/bin/cheat', + '/etc/bash_completion.d/cht.sh'] + # PHPREDISADMIN if pargs.phpredisadmin: Log.debug(self, "Removing package variable of phpRedisAdmin ") @@ -794,6 +824,7 @@ class WOStackController(CementBaseController): (not pargs.adminer) and (not pargs.utils) and (not pargs.redis) and (not pargs.proftpd) and (not pargs.extplorer) and (not pargs.clamav) and + (not pargs.cheat) and (not pargs.ufw) and (not pargs.ngxblocker) and (not pargs.phpredisadmin) and (not pargs.sendmail) and (not pargs.php73)): @@ -823,6 +854,7 @@ class WOStackController(CementBaseController): pargs.composer = True pargs.netdata = True pargs.mysqltuner = True + pargs.cheat = True if pargs.security: pargs.fail2ban = True @@ -939,6 +971,14 @@ class WOStackController(CementBaseController): Log.debug(self, "Removing packages for MySQLTuner ") packages = packages + ['/usr/bin/mysqltuner'] + # cheat.sh + if pargs.cheat: + if os.path.isfile('/usr/local/bin/cht.sh'): + Log.debug(self, "Removing packages for cheat.sh ") + packages = packages + [ + '/usr/local/bin/cht.sh', '/usr/local/bin/cheat', + '/etc/bash_completion.d/cht.sh'] + # PHPREDISADMIN if pargs.phpredisadmin: Log.debug(self, "Removing package variable of phpRedisAdmin ") diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index 1b85165..d55f1d2 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -1192,6 +1192,21 @@ def post_pref(self, apt_packages, packages, upgrade=False): Log.debug(self, "CHMOD MySQLTuner in /usr/bin/mysqltuner") WOFileUtils.chmod(self, "/usr/bin/mysqltuner", 0o775) + # cheat.sh + if any('/usr/local/bin/cht.sh' == x[1] + for x in packages): + Log.debug(self, "CHMOD cht.sh in /usr/local/bin/cht.sh") + WOFileUtils.chmod(self, "/usr/local/bin/cht.sh", 0o775) + if WOFileUtils.grepcheck(self, '/etc/bash_completion.d/cht.sh', + 'cht_complete cht.sh'): + WOFileUtils.searchreplace( + self, '/etc/bash_completion.d/cht.sh', + '_cht_complete cht.sh', + '_cht_complete cheat') + if not os.path.islink('/usr/local/bin/cheat'): + WOFileUtils.create_symlink( + self, ['/usr/local/bin/cht.sh', '/usr/local/bin/cheat']) + # netdata install if any('/var/lib/wo/tmp/kickstart.sh' == x[1] for x in packages): @@ -1488,8 +1503,8 @@ def pre_stack(self): WOTemplate.deploy( self, '/etc/update-motd.d/98-wo-update', 'wo-update.mustache', data) - WOFileUtils.chmod( - self, "/etc/update-motd.d/98-wo-update", 0o755) - # restart motd-news service if available - if os.path.isfile('/lib/systemd/system/motd-news.service'): - WOService.restart_service(self, 'motd-news') + WOFileUtils.chmod( + self, "/etc/update-motd.d/98-wo-update", 0o755) + # restart motd-news service if available + if os.path.isfile('/lib/systemd/system/motd-news.service'): + WOService.restart_service(self, 'motd-news') diff --git a/wo/utils/test.py b/wo/utils/test.py index 6c46790..d737fdd 100644 --- a/wo/utils/test.py +++ b/wo/utils/test.py @@ -1,5 +1,5 @@ """Testing utilities for WordOps""" -from cement.utils.tests import CementTestCase +from cement.utils.test import CementTestCase from wo.cli.main import WOTestApp From 123c4955e42b9afb32b74f8e9748f59bd637ff3a Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 28 Oct 2019 22:07:51 +0100 Subject: [PATCH 37/79] add cheat to travis --- tests/travis.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/travis.sh b/tests/travis.sh index 3391d08..e5e7bd9 100644 --- a/tests/travis.sh +++ b/tests/travis.sh @@ -28,7 +28,7 @@ exit_script() { echo -e "${CGREEN}#############################################${CEND}" echo -e ' stack install ' echo -e "${CGREEN}#############################################${CEND}" -stack_list='nginx php php73 mysql redis fail2ban clamav proftpd netdata phpmyadmin composer dashboard extplorer adminer redis phpredisadmin mysqltuner utils ufw ngxblocker' +stack_list='nginx php php73 mysql redis fail2ban clamav proftpd netdata phpmyadmin composer dashboard extplorer adminer redis phpredisadmin mysqltuner utils ufw ngxblocker cheat' for stack in $stack_list; do echo -ne " Installing $stack [..]\r" if { From bda69cb552e5ecec92bc4d145ac01e9ea8956465 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Mon, 28 Oct 2019 22:08:23 +0100 Subject: [PATCH 38/79] Add cheat to travis --- tests/travis.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/travis.sh b/tests/travis.sh index e5e7bd9..1b386d2 100644 --- a/tests/travis.sh +++ b/tests/travis.sh @@ -229,7 +229,7 @@ wo site info wp.net echo -e "${CGREEN}#############################################${CEND}" echo -e ' wo stack purge ' echo -e "${CGREEN}#############################################${CEND}" -stack_purge='nginx php php73 mysql redis fail2ban clamav proftpd netdata phpmyadmin composer dashboard extplorer adminer redis ufw ngxblocker' +stack_purge='nginx php php73 mysql redis fail2ban clamav proftpd netdata phpmyadmin composer dashboard extplorer adminer redis ufw ngxblocker cheat' for stack in $stack_purge; do echo -ne " purging $stack [..]\r" if { From fd1105f4594c3871841e183311eb8c9669eec460 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Tue, 29 Oct 2019 01:10:37 +0100 Subject: [PATCH 39/79] Improve `wo stack services` --- CHANGELOG.md | 1 + MANIFEST.in | 4 + snapcraft.yaml | 23 ------ wo/cli/plugins/stack.py | 2 +- wo/cli/plugins/stack_services.py | 129 +++++++++++++------------------ 5 files changed, 61 insertions(+), 98 deletions(-) create mode 100644 MANIFEST.in delete mode 100644 snapcraft.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a87c2e..c51559c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Version check before updating WordOps with `wo update` is now directly handled by `wo` - Refactored WordOps download function with python3-requests - MySQL backup path changed to `/var/lib/wo-backup/mysql` +- Do not check anymore if stack are installed with apt in `wo service` but only if there is a systemd service #### Fixed diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..60115fe --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,4 @@ +recursive-include *.py +include setup.cfg +include README.md CHANGELOG.md LICENSE +include *.txt diff --git a/snapcraft.yaml b/snapcraft.yaml deleted file mode 100644 index eb46364..0000000 --- a/snapcraft.yaml +++ /dev/null @@ -1,23 +0,0 @@ -name: test-wordops -version: git -summary: WordOps -description: | - WordOps is an essential toolset that eases WordPress - site and server administration. It provide the ability - to install a high performance WordPress stack - with a few keystrokes. -confinement: devmode -base: core18 - -parts: - test-wordops: - plugin: python - python-version: python3 - source: . - stage-packages: - - cement - - python-apt - -apps: - test-wordops: - command: wo diff --git a/wo/cli/plugins/stack.py b/wo/cli/plugins/stack.py index f152104..46681d7 100644 --- a/wo/cli/plugins/stack.py +++ b/wo/cli/plugins/stack.py @@ -457,7 +457,7 @@ class WOStackController(CementBaseController): ["https://raw.githubusercontent.com/chubin/cheat.sh" "/master/share/bash_completion.txt", "/etc/bash_completion.d/cht.sh", - "cheat.sh bash_completion"]] + "bash_completion"]] # UTILS if pargs.utils: diff --git a/wo/cli/plugins/stack_services.py b/wo/cli/plugins/stack_services.py index c9c4f43..6b40649 100644 --- a/wo/cli/plugins/stack_services.py +++ b/wo/cli/plugins/stack_services.py @@ -1,8 +1,6 @@ import os from cement.core.controller import CementBaseController, expose - -from wo.core.aptget import WOAptGet from wo.core.logging import Log from wo.core.services import WOService from wo.core.variables import WOVar @@ -19,6 +17,7 @@ class WOStackStatusController(CementBaseController): def start(self): """Start services""" services = [] + wo_system = "/lib/systemd/system/" pargs = self.app.pargs if not (pargs.nginx or pargs.php or pargs.php73 or @@ -34,23 +33,23 @@ class WOStackStatusController(CementBaseController): pargs.netdata = True if pargs.nginx: - if (WOAptGet.is_installed(self, 'nginx-custom')): + if os.path.exists('{0}'.format(wo_system) + 'nginx.service'): services = services + ['nginx'] else: Log.info(self, "Nginx is not installed") if pargs.php: - if WOAptGet.is_installed(self, 'php7.2-fpm'): + if os.path.exists('{0}'.format(wo_system) + 'php7.2-fpm.service'): services = services + ['php7.2-fpm'] else: Log.info(self, "PHP7.2-FPM is not installed") - if WOAptGet.is_installed(self, 'php7.3-fpm'): + if os.path.exists('{0}'.format(wo_system) + 'php7.3-fpm.service'): services = services + ['php7.3-fpm'] else: Log.info(self, "PHP7.3-FPM is not installed") if pargs.php73: - if WOAptGet.is_installed(self, 'php7.3-fpm'): + if os.path.exists('{0}'.format(wo_system) + 'php7.3-fpm.service'): services = services + ['php7.3-fpm'] else: Log.info(self, "PHP7.3-FPM is not installed") @@ -58,9 +57,7 @@ class WOStackStatusController(CementBaseController): if pargs.mysql: if ((WOVar.wo_mysql_host == "localhost") or (WOVar.wo_mysql_host == "127.0.0.1")): - if (WOAptGet.is_installed(self, 'mysql-server') or - WOAptGet.is_installed(self, 'percona-server-server-5.6') or - WOAptGet.is_installed(self, 'mariadb-server')): + if os.path.exists('/etc/systemd/system/mysql.service'): services = services + ['mysql'] else: Log.info(self, "MySQL is not installed") @@ -69,28 +66,28 @@ class WOStackStatusController(CementBaseController): "Unable to check MySQL service status") if pargs.redis: - if WOAptGet.is_installed(self, 'redis-server'): + if os.path.exists('{0}'.format(wo_system) + + 'redis-server.service'): services = services + ['redis-server'] else: Log.info(self, "Redis server is not installed") if pargs.fail2ban: - if WOAptGet.is_installed(self, 'fail2ban'): + if os.path.exists('{0}'.format(wo_system) + 'fail2ban.service'): services = services + ['fail2ban'] else: Log.info(self, "fail2ban is not installed") # proftpd if pargs.proftpd: - if WOAptGet.is_installed(self, 'proftpd-basic'): + if os.path.exists('/etc/init.d/proftpd'): services = services + ['proftpd'] else: Log.info(self, "ProFTPd is not installed") # netdata if pargs.netdata: - if (os.path.isdir("/opt/netdata") or - os.path.isdir("/etc/netdata")): + if os.path.exists('{0}'.format(wo_system) + 'netdata.service'): services = services + ['netdata'] else: Log.info(self, "Netdata is not installed") @@ -103,6 +100,7 @@ class WOStackStatusController(CementBaseController): def stop(self): """Stop services""" services = [] + wo_system = "/lib/systemd/system/" pargs = self.app.pargs if not (pargs.nginx or pargs.php or pargs.php73 or @@ -115,39 +113,32 @@ class WOStackStatusController(CementBaseController): pargs.php = True pargs.mysql = True - # nginx if pargs.nginx: - if (WOAptGet.is_installed(self, 'nginx-custom')): + if os.path.exists('{0}'.format(wo_system) + 'nginx.service'): services = services + ['nginx'] else: Log.info(self, "Nginx is not installed") - # php7.2 if pargs.php: - if WOAptGet.is_installed(self, 'php7.2-fpm'): + if os.path.exists('{0}'.format(wo_system) + 'php7.2-fpm.service'): services = services + ['php7.2-fpm'] else: Log.info(self, "PHP7.2-FPM is not installed") - - if WOAptGet.is_installed(self, 'php7.3-fpm'): + if os.path.exists('{0}'.format(wo_system) + 'php7.3-fpm.service'): services = services + ['php7.3-fpm'] else: Log.info(self, "PHP7.3-FPM is not installed") - # php7.3 if pargs.php73: - if WOAptGet.is_installed(self, 'php7.3-fpm'): + if os.path.exists('{0}'.format(wo_system) + 'php7.3-fpm.service'): services = services + ['php7.3-fpm'] else: Log.info(self, "PHP7.3-FPM is not installed") - # mysql if pargs.mysql: if ((WOVar.wo_mysql_host == "localhost") or (WOVar.wo_mysql_host == "127.0.0.1")): - if (WOAptGet.is_installed(self, 'mysql-server') or - WOAptGet.is_installed(self, 'percona-server-server-5.6') or - WOAptGet.is_installed(self, 'mariadb-server')): + if os.path.exists('/etc/systemd/system/mysql.service'): services = services + ['mysql'] else: Log.info(self, "MySQL is not installed") @@ -155,31 +146,29 @@ class WOStackStatusController(CementBaseController): Log.warn(self, "Remote MySQL found, " "Unable to check MySQL service status") - # redis if pargs.redis: - if WOAptGet.is_installed(self, 'redis-server'): + if os.path.exists('{0}'.format(wo_system) + + 'redis-server.service'): services = services + ['redis-server'] else: Log.info(self, "Redis server is not installed") - # fail2ban if pargs.fail2ban: - if WOAptGet.is_installed(self, 'fail2ban'): + if os.path.exists('{0}'.format(wo_system) + 'fail2ban.service'): services = services + ['fail2ban'] else: Log.info(self, "fail2ban is not installed") # proftpd if pargs.proftpd: - if WOAptGet.is_installed(self, 'proftpd-basic'): + if os.path.exists('/etc/init.d/proftpd'): services = services + ['proftpd'] else: Log.info(self, "ProFTPd is not installed") # netdata if pargs.netdata: - if (os.path.isdir("/opt/netdata") or - os.path.isdir("/etc/netdata")): + if os.path.exists('{0}'.format(wo_system) + 'netdata.service'): services = services + ['netdata'] else: Log.info(self, "Netdata is not installed") @@ -192,6 +181,7 @@ class WOStackStatusController(CementBaseController): def restart(self): """Restart services""" services = [] + wo_system = "/lib/systemd/system/" pargs = self.app.pargs if not (pargs.nginx or pargs.php or pargs.php73 or @@ -206,24 +196,23 @@ class WOStackStatusController(CementBaseController): pargs.netdata = True if pargs.nginx: - if (WOAptGet.is_installed(self, 'nginx-custom')): + if os.path.exists('{0}'.format(wo_system) + 'nginx.service'): services = services + ['nginx'] else: Log.info(self, "Nginx is not installed") if pargs.php: - if WOAptGet.is_installed(self, 'php7.2-fpm'): + if os.path.exists('{0}'.format(wo_system) + 'php7.2-fpm.service'): services = services + ['php7.2-fpm'] else: Log.info(self, "PHP7.2-FPM is not installed") - - if WOAptGet.is_installed(self, 'php7.3-fpm'): + if os.path.exists('{0}'.format(wo_system) + 'php7.3-fpm.service'): services = services + ['php7.3-fpm'] else: Log.info(self, "PHP7.3-FPM is not installed") if pargs.php73: - if WOAptGet.is_installed(self, 'php7.3-fpm'): + if os.path.exists('{0}'.format(wo_system) + 'php7.3-fpm.service'): services = services + ['php7.3-fpm'] else: Log.info(self, "PHP7.3-FPM is not installed") @@ -231,10 +220,7 @@ class WOStackStatusController(CementBaseController): if pargs.mysql: if ((WOVar.wo_mysql_host == "localhost") or (WOVar.wo_mysql_host == "127.0.0.1")): - if ((WOAptGet.is_installed(self, 'mysql-server') or - WOAptGet.is_installed(self, - 'percona-server-server-5.6') or - WOAptGet.is_installed(self, 'mariadb-server'))): + if os.path.exists('/etc/systemd/system/mysql.service'): services = services + ['mysql'] else: Log.info(self, "MySQL is not installed") @@ -243,28 +229,28 @@ class WOStackStatusController(CementBaseController): "Unable to check MySQL service status") if pargs.redis: - if WOAptGet.is_installed(self, 'redis-server'): + if os.path.exists('{0}'.format(wo_system) + + 'redis-server.service'): services = services + ['redis-server'] else: Log.info(self, "Redis server is not installed") if pargs.fail2ban: - if WOAptGet.is_installed(self, 'fail2ban'): + if os.path.exists('{0}'.format(wo_system) + 'fail2ban.service'): services = services + ['fail2ban'] else: Log.info(self, "fail2ban is not installed") # proftpd if pargs.proftpd: - if WOAptGet.is_installed(self, 'proftpd-basic'): + if os.path.exists('/etc/init.d/proftpd'): services = services + ['proftpd'] else: Log.info(self, "ProFTPd is not installed") # netdata if pargs.netdata: - if (os.path.isdir("/opt/netdata") or - os.path.isdir("/etc/netdata")): + if os.path.exists('{0}'.format(wo_system) + 'netdata.service'): services = services + ['netdata'] else: Log.info(self, "Netdata is not installed") @@ -277,6 +263,7 @@ class WOStackStatusController(CementBaseController): def status(self): """Status of services""" services = [] + wo_system = "/lib/systemd/system/" pargs = self.app.pargs if not (pargs.nginx or pargs.php or pargs.php73 or @@ -292,24 +279,23 @@ class WOStackStatusController(CementBaseController): pargs.netdata = True if pargs.nginx: - if (WOAptGet.is_installed(self, 'nginx-custom')): + if os.path.exists('{0}'.format(wo_system) + 'nginx.service'): services = services + ['nginx'] else: Log.info(self, "Nginx is not installed") if pargs.php: - if WOAptGet.is_installed(self, 'php7.2-fpm'): + if os.path.exists('{0}'.format(wo_system) + 'php7.2-fpm.service'): services = services + ['php7.2-fpm'] else: Log.info(self, "PHP7.2-FPM is not installed") - - if WOAptGet.is_installed(self, 'php7.3-fpm'): + if os.path.exists('{0}'.format(wo_system) + 'php7.3-fpm.service'): services = services + ['php7.3-fpm'] else: Log.info(self, "PHP7.3-FPM is not installed") if pargs.php73: - if WOAptGet.is_installed(self, 'php7.3-fpm'): + if os.path.exists('{0}'.format(wo_system) + 'php7.3-fpm.service'): services = services + ['php7.3-fpm'] else: Log.info(self, "PHP7.3-FPM is not installed") @@ -317,9 +303,7 @@ class WOStackStatusController(CementBaseController): if pargs.mysql: if ((WOVar.wo_mysql_host == "localhost") or (WOVar.wo_mysql_host == "127.0.0.1")): - if (WOAptGet.is_installed(self, 'mysql-server') or - WOAptGet.is_installed(self, 'percona-server-server-5.6') or - WOAptGet.is_installed(self, 'mariadb-server')): + if os.path.exists('/etc/systemd/system/mysql.service'): services = services + ['mysql'] else: Log.info(self, "MySQL is not installed") @@ -328,28 +312,28 @@ class WOStackStatusController(CementBaseController): "Unable to check MySQL service status") if pargs.redis: - if WOAptGet.is_installed(self, 'redis-server'): + if os.path.exists('{0}'.format(wo_system) + + 'redis-server.service'): services = services + ['redis-server'] else: Log.info(self, "Redis server is not installed") if pargs.fail2ban: - if WOAptGet.is_installed(self, 'fail2ban'): + if os.path.exists('{0}'.format(wo_system) + 'fail2ban.service'): services = services + ['fail2ban'] else: Log.info(self, "fail2ban is not installed") # proftpd if pargs.proftpd: - if WOAptGet.is_installed(self, 'proftpd-basic'): + if os.path.exists('/etc/init.d/proftpd'): services = services + ['proftpd'] else: Log.info(self, "ProFTPd is not installed") # netdata if pargs.netdata: - if (os.path.isdir("/opt/netdata") or - os.path.isdir("/etc/netdata")): + if os.path.exists('{0}'.format(wo_system) + 'netdata.service'): services = services + ['netdata'] else: Log.info(self, "Netdata is not installed") @@ -362,6 +346,7 @@ class WOStackStatusController(CementBaseController): def reload(self): """Reload service""" services = [] + wo_system = "/lib/systemd/system/" pargs = self.app.pargs if not (pargs.nginx or pargs.php or pargs.php73 or @@ -376,25 +361,23 @@ class WOStackStatusController(CementBaseController): pargs.fail2ban = True if pargs.nginx: - if (WOAptGet.is_installed(self, 'nginx-custom') or - WOAptGet.is_installed(self, 'nginx-mainline')): + if os.path.exists('{0}'.format(wo_system) + 'nginx.service'): services = services + ['nginx'] else: Log.info(self, "Nginx is not installed") if pargs.php: - if WOAptGet.is_installed(self, 'php7.2-fpm'): + if os.path.exists('{0}'.format(wo_system) + 'php7.2-fpm.service'): services = services + ['php7.2-fpm'] else: Log.info(self, "PHP7.2-FPM is not installed") - - if WOAptGet.is_installed(self, 'php7.3-fpm'): + if os.path.exists('{0}'.format(wo_system) + 'php7.3-fpm.service'): services = services + ['php7.3-fpm'] else: Log.info(self, "PHP7.3-FPM is not installed") if pargs.php73: - if WOAptGet.is_installed(self, 'php7.3-fpm'): + if os.path.exists('{0}'.format(wo_system) + 'php7.3-fpm.service'): services = services + ['php7.3-fpm'] else: Log.info(self, "PHP7.3-FPM is not installed") @@ -402,9 +385,7 @@ class WOStackStatusController(CementBaseController): if pargs.mysql: if ((WOVar.wo_mysql_host == "localhost") or (WOVar.wo_mysql_host == "127.0.0.1")): - if (WOAptGet.is_installed(self, 'mysql-server') or - WOAptGet.is_installed(self, 'percona-server-server-5.6') or - WOAptGet.is_installed(self, 'mariadb-server')): + if os.path.exists('/etc/systemd/system/mysql.service'): services = services + ['mysql'] else: Log.info(self, "MySQL is not installed") @@ -413,28 +394,28 @@ class WOStackStatusController(CementBaseController): "Unable to check MySQL service status") if pargs.redis: - if WOAptGet.is_installed(self, 'redis-server'): + if os.path.exists('{0}'.format(wo_system) + + 'redis-server.service'): services = services + ['redis-server'] else: Log.info(self, "Redis server is not installed") if pargs.fail2ban: - if WOAptGet.is_installed(self, 'fail2ban'): + if os.path.exists('{0}'.format(wo_system) + 'fail2ban.service'): services = services + ['fail2ban'] else: Log.info(self, "fail2ban is not installed") # proftpd if pargs.proftpd: - if WOAptGet.is_installed(self, 'proftpd-basic'): + if os.path.exists('/etc/init.d/proftpd'): services = services + ['proftpd'] else: Log.info(self, "ProFTPd is not installed") # netdata if pargs.netdata: - if (os.path.isdir("/opt/netdata") or - os.path.isdir("/etc/netdata")): + if os.path.exists('{0}'.format(wo_system) + 'netdata.service'): services = services + ['netdata'] else: Log.info(self, "Netdata is not installed") From 8b32910c07d110c5797533d3ae7753949cd0e0ab Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Tue, 29 Oct 2019 03:25:44 +0100 Subject: [PATCH 40/79] Remove useless variable --- wo/cli/plugins/site.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wo/cli/plugins/site.py b/wo/cli/plugins/site.py index b0f870c..b5ab0a2 100644 --- a/wo/cli/plugins/site.py +++ b/wo/cli/plugins/site.py @@ -724,7 +724,7 @@ class WOSiteCreateController(CementBaseController): Log.info(self, "Successfully created site" " http://{0}".format(wo_domain)) - except SiteError as e: + except SiteError: Log.error(self, "Check the log for details: " "`tail /var/log/wo/wordops.log` and please try again") From 4c1569781dbfa151381ca4e3697f9ec3318e9be0 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Tue, 29 Oct 2019 13:58:17 +0100 Subject: [PATCH 41/79] Refactor tests --- tests/cli/13_test_stack.py | 73 +++++++------------------------------- 1 file changed, 12 insertions(+), 61 deletions(-) diff --git a/tests/cli/13_test_stack.py b/tests/cli/13_test_stack.py index c6b19b3..b16f56f 100644 --- a/tests/cli/13_test_stack.py +++ b/tests/cli/13_test_stack.py @@ -5,67 +5,18 @@ from wo.cli.main import get_test_app class CliTestCaseStack(test.WOTestCase): def test_wo_cli(self): - self.app.setup() - self.app.run() - self.app.close() - - def test_wo_cli_stack_install(self): - argv = ['stack', 'install'] + argv = [] with self.make_app(argv=argv) as app: app.run() - self.eq(app.pargs.stack.install) + self.eq(app.pargs.stack) - - def test_wo_cli_stack_install_web(self): - self.app = get_test_app(argv=['stack', 'install', '--web']) - self.app.setup() - self.app.run() - self.app.close() - - def test_wo_cli_stack_install_admin(self): - self.app = get_test_app(argv=['stack', 'install', '--admin']) - self.app.setup() - self.app.run() - self.app.close() - - def test_wo_cli_stack_install_nginx(self): - self.app = get_test_app(argv=['stack', 'install', '--nginx']) - self.app.setup() - self.app.run() - self.app.close() - - def test_wo_cli_stack_install_php(self): - self.app = get_test_app(argv=['stack', 'install', '--php']) - self.app.setup() - self.app.run() - self.app.close() - - def test_wo_cli_stack_install_mysql(self): - self.app = get_test_app(argv=['stack', 'install', '--mysql']) - self.app.setup() - self.app.run() - self.app.close() - - def test_wo_cli_stack_install_wpcli(self): - self.app = get_test_app(argv=['stack', 'install', '--wpcli']) - self.app.setup() - self.app.run() - self.app.close() - - def test_wo_cli_stack_install_phpmyadmin(self): - self.app = get_test_app(argv=['stack', 'install', '--phpmyadmin']) - self.app.setup() - self.app.run() - self.app.close() - - def test_wo_cli_stack_install_adminer(self): - self.app = get_test_app(argv=['stack', 'install', '--adminer']) - self.app.setup() - self.app.run() - self.app.close() - - def test_wo_cli_stack_install_utils(self): - self.app = get_test_app(argv=['stack', 'install', '--utils']) - self.app.setup() - self.app.run() - self.app.close() + def test_wo_cli_stacks_install(self): + wo_stacks = ['nginx', 'php', 'php73', 'mysql', 'redis', 'fail2ban', + 'clamav', 'proftpd', 'netdata', + 'phpmyadmin', 'composer', 'dashboard', 'extplorer', + 'adminer', 'redis', 'ufw', 'ngxblocker', 'cheat'] + for wo_stack in wo_stacks: + argv = ['stack', 'install', '--{0}'.format(wo_stack)] + with self.make_app(argv=argv) as app: + app.run() + self.eq(app.pargs.stack.install.wo_stack) \ No newline at end of file From dd45835ffacfac7d6295b6c178f903c6fb4c965a Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Tue, 29 Oct 2019 14:08:48 +0100 Subject: [PATCH 42/79] refactor tests --- tests/cli/13_test_stack.py | 78 ++++++++++++++++++++++++++++++++------ 1 file changed, 66 insertions(+), 12 deletions(-) diff --git a/tests/cli/13_test_stack.py b/tests/cli/13_test_stack.py index b16f56f..c9f4544 100644 --- a/tests/cli/13_test_stack.py +++ b/tests/cli/13_test_stack.py @@ -5,18 +5,72 @@ from wo.cli.main import get_test_app class CliTestCaseStack(test.WOTestCase): def test_wo_cli(self): - argv = [] + self.app.setup() + self.app.run() + self.app.close() + + def test_wo_cli_stack_install_nginx(self): + argv = ['stack', 'install', '--nginx'] with self.make_app(argv=argv) as app: app.run() - self.eq(app.pargs.stack) + self.eq(app.pargs.stack.install.nginx) - def test_wo_cli_stacks_install(self): - wo_stacks = ['nginx', 'php', 'php73', 'mysql', 'redis', 'fail2ban', - 'clamav', 'proftpd', 'netdata', - 'phpmyadmin', 'composer', 'dashboard', 'extplorer', - 'adminer', 'redis', 'ufw', 'ngxblocker', 'cheat'] - for wo_stack in wo_stacks: - argv = ['stack', 'install', '--{0}'.format(wo_stack)] - with self.make_app(argv=argv) as app: - app.run() - self.eq(app.pargs.stack.install.wo_stack) \ No newline at end of file + def test_wo_cli_stack_install_php(self): + argv = ['stack', 'install', '--php'] + with self.make_app(argv=argv) as app: + app.run() + self.eq(app.pargs.stack.install.php) + + def test_wo_cli_stack_install_mysql(self): + argv = ['stack', 'install', '--mysql'] + with self.make_app(argv=argv) as app: + app.run() + self.eq(app.pargs.stack.install.mysql) + + def test_wo_cli_stack_install_admin(self): + self.app = get_test_app(argv=['stack', 'install', '--admin']) + self.app.setup() + self.app.run() + self.app.close() + + def test_wo_cli_stack_install_nginx(self): + self.app = get_test_app(argv=['stack', 'install', '--nginx']) + self.app.setup() + self.app.run() + self.app.close() + + def test_wo_cli_stack_install_php(self): + self.app = get_test_app(argv=['stack', 'install', '--php']) + self.app.setup() + self.app.run() + self.app.close() + + def test_wo_cli_stack_install_mysql(self): + self.app = get_test_app(argv=['stack', 'install', '--mysql']) + self.app.setup() + self.app.run() + self.app.close() + + def test_wo_cli_stack_install_wpcli(self): + self.app = get_test_app(argv=['stack', 'install', '--wpcli']) + self.app.setup() + self.app.run() + self.app.close() + + def test_wo_cli_stack_install_phpmyadmin(self): + self.app = get_test_app(argv=['stack', 'install', '--phpmyadmin']) + self.app.setup() + self.app.run() + self.app.close() + + def test_wo_cli_stack_install_adminer(self): + self.app = get_test_app(argv=['stack', 'install', '--adminer']) + self.app.setup() + self.app.run() + self.app.close() + + def test_wo_cli_stack_install_utils(self): + self.app = get_test_app(argv=['stack', 'install', '--utils']) + self.app.setup() + self.app.run() + self.app.close() From 94b64c962e2d44e0291eb23fd8e86166bd210020 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Tue, 29 Oct 2019 14:11:42 +0100 Subject: [PATCH 43/79] Fix tests --- tests/cli/13_test_stack.py | 30 ++++++------------------------ 1 file changed, 6 insertions(+), 24 deletions(-) diff --git a/tests/cli/13_test_stack.py b/tests/cli/13_test_stack.py index c9f4544..7e752b4 100644 --- a/tests/cli/13_test_stack.py +++ b/tests/cli/13_test_stack.py @@ -9,30 +9,6 @@ class CliTestCaseStack(test.WOTestCase): self.app.run() self.app.close() - def test_wo_cli_stack_install_nginx(self): - argv = ['stack', 'install', '--nginx'] - with self.make_app(argv=argv) as app: - app.run() - self.eq(app.pargs.stack.install.nginx) - - def test_wo_cli_stack_install_php(self): - argv = ['stack', 'install', '--php'] - with self.make_app(argv=argv) as app: - app.run() - self.eq(app.pargs.stack.install.php) - - def test_wo_cli_stack_install_mysql(self): - argv = ['stack', 'install', '--mysql'] - with self.make_app(argv=argv) as app: - app.run() - self.eq(app.pargs.stack.install.mysql) - - def test_wo_cli_stack_install_admin(self): - self.app = get_test_app(argv=['stack', 'install', '--admin']) - self.app.setup() - self.app.run() - self.app.close() - def test_wo_cli_stack_install_nginx(self): self.app = get_test_app(argv=['stack', 'install', '--nginx']) self.app.setup() @@ -45,6 +21,12 @@ class CliTestCaseStack(test.WOTestCase): self.app.run() self.app.close() + def test_wo_cli_stack_install_php73(self): + self.app = get_test_app(argv=['stack', 'install', '--php73']) + self.app.setup() + self.app.run() + self.app.close() + def test_wo_cli_stack_install_mysql(self): self.app = get_test_app(argv=['stack', 'install', '--mysql']) self.app.setup() From 6648764836a7c8378ba9f4ffeb6ed2fcb51ee97e Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Tue, 29 Oct 2019 15:20:25 +0100 Subject: [PATCH 44/79] Update tests --- wo/cli/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wo/cli/main.py b/wo/cli/main.py index fc80c05..84b85af 100644 --- a/wo/cli/main.py +++ b/wo/cli/main.py @@ -121,8 +121,8 @@ def main(): def get_test_app(**kw): - app = WOApp(**kw) - return app + with WOApp(**kw) as app: + return app if __name__ == '__main__': From 521daa3b32485db992a5f3151caa2f2148c8e1c9 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Tue, 29 Oct 2019 15:22:06 +0100 Subject: [PATCH 45/79] Fix tests --- tests/cli/13_test_stack.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/tests/cli/13_test_stack.py b/tests/cli/13_test_stack.py index 7e752b4..26acf95 100644 --- a/tests/cli/13_test_stack.py +++ b/tests/cli/13_test_stack.py @@ -7,52 +7,51 @@ class CliTestCaseStack(test.WOTestCase): def test_wo_cli(self): self.app.setup() self.app.run() - self.app.close() + def test_wo_cli_stack_install_nginx(self): self.app = get_test_app(argv=['stack', 'install', '--nginx']) self.app.setup() self.app.run() - self.app.close() + def test_wo_cli_stack_install_php(self): self.app = get_test_app(argv=['stack', 'install', '--php']) self.app.setup() self.app.run() - self.app.close() + def test_wo_cli_stack_install_php73(self): self.app = get_test_app(argv=['stack', 'install', '--php73']) self.app.setup() self.app.run() - self.app.close() + def test_wo_cli_stack_install_mysql(self): self.app = get_test_app(argv=['stack', 'install', '--mysql']) self.app.setup() self.app.run() - self.app.close() + def test_wo_cli_stack_install_wpcli(self): self.app = get_test_app(argv=['stack', 'install', '--wpcli']) self.app.setup() self.app.run() - self.app.close() + def test_wo_cli_stack_install_phpmyadmin(self): self.app = get_test_app(argv=['stack', 'install', '--phpmyadmin']) self.app.setup() self.app.run() - self.app.close() + def test_wo_cli_stack_install_adminer(self): self.app = get_test_app(argv=['stack', 'install', '--adminer']) self.app.setup() self.app.run() - self.app.close() + def test_wo_cli_stack_install_utils(self): self.app = get_test_app(argv=['stack', 'install', '--utils']) self.app.setup() self.app.run() - self.app.close() From 290e9edeb9c4851361e9b947c05d81b9ac515e10 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Tue, 29 Oct 2019 15:22:22 +0100 Subject: [PATCH 46/79] Fix tests --- tests/cli/13_test_stack.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tests/cli/13_test_stack.py b/tests/cli/13_test_stack.py index 26acf95..fbecf58 100644 --- a/tests/cli/13_test_stack.py +++ b/tests/cli/13_test_stack.py @@ -8,49 +8,41 @@ class CliTestCaseStack(test.WOTestCase): self.app.setup() self.app.run() - def test_wo_cli_stack_install_nginx(self): self.app = get_test_app(argv=['stack', 'install', '--nginx']) self.app.setup() self.app.run() - def test_wo_cli_stack_install_php(self): self.app = get_test_app(argv=['stack', 'install', '--php']) self.app.setup() self.app.run() - def test_wo_cli_stack_install_php73(self): self.app = get_test_app(argv=['stack', 'install', '--php73']) self.app.setup() self.app.run() - def test_wo_cli_stack_install_mysql(self): self.app = get_test_app(argv=['stack', 'install', '--mysql']) self.app.setup() self.app.run() - def test_wo_cli_stack_install_wpcli(self): self.app = get_test_app(argv=['stack', 'install', '--wpcli']) self.app.setup() self.app.run() - def test_wo_cli_stack_install_phpmyadmin(self): self.app = get_test_app(argv=['stack', 'install', '--phpmyadmin']) self.app.setup() self.app.run() - def test_wo_cli_stack_install_adminer(self): self.app = get_test_app(argv=['stack', 'install', '--adminer']) self.app.setup() self.app.run() - def test_wo_cli_stack_install_utils(self): self.app = get_test_app(argv=['stack', 'install', '--utils']) self.app.setup() From c18975cca0d39218c47109a58642fb18948c49d8 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Tue, 29 Oct 2019 15:24:28 +0100 Subject: [PATCH 47/79] Switch back main --- wo/cli/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wo/cli/main.py b/wo/cli/main.py index 84b85af..fc80c05 100644 --- a/wo/cli/main.py +++ b/wo/cli/main.py @@ -121,8 +121,8 @@ def main(): def get_test_app(**kw): - with WOApp(**kw) as app: - return app + app = WOApp(**kw) + return app if __name__ == '__main__': From 97103704eb4216c76eff69ca185294f624c1c9be Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Tue, 29 Oct 2019 15:35:08 +0100 Subject: [PATCH 48/79] Add function name into log_debug --- wo/core/logging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wo/core/logging.py b/wo/core/logging.py index bb6cbda..8e034f7 100644 --- a/wo/core/logging.py +++ b/wo/core/logging.py @@ -44,7 +44,7 @@ class Log: """ Logs debug messages into log file """ - self.app.log.debug(Log.HEADER + msg + Log.ENDC) + self.app.log.debug(Log.HEADER + msg + Log.ENDC + __name__) def wait(self, msg, end='\r', log=True): """ From aa5f2eeb496e2746e875088c07df2ee1b77b2e65 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Tue, 29 Oct 2019 18:47:52 +0100 Subject: [PATCH 49/79] Refactor `-le=renew` --- wo/cli/ext/wo_outputhandler.py | 1 + wo/cli/plugins/site.py | 42 ++++++++++++++++++++++---------- wo/cli/plugins/site_functions.py | 38 +++++++++++++++++++---------- wo/core/acme.py | 11 +++++++++ wo/core/logging.py | 2 +- 5 files changed, 67 insertions(+), 27 deletions(-) diff --git a/wo/cli/ext/wo_outputhandler.py b/wo/cli/ext/wo_outputhandler.py index c472420..47a92f8 100644 --- a/wo/cli/ext/wo_outputhandler.py +++ b/wo/cli/ext/wo_outputhandler.py @@ -19,3 +19,4 @@ class WOOutputHandler(MustacheOutputHandler): return open(full_path, encoding='utf-8', mode='r').read() else: continue + return 1 diff --git a/wo/cli/plugins/site.py b/wo/cli/plugins/site.py index b5ab0a2..7a2be55 100644 --- a/wo/cli/plugins/site.py +++ b/wo/cli/plugins/site.py @@ -1069,23 +1069,39 @@ class WOSiteUpdateController(CementBaseController): else: Log.error(self, 'ngxblocker stack is not installed') elif pargs.ngxblocker == "off": - if os.path.isfile( - '/var/www/{0}/conf/nginx/ngxblocker.conf' - .format(wo_domain)): - WOFileUtils.mvfile(self, '/var/www/{0}/conf/' - 'nginx/ngxblocker.conf' - .format(wo_domain), - '/var/www/{0}/conf/' - 'nginx/ngxblocker.conf.disabled' - .format(wo_domain)) - else: - Log.error(self, "ngxblocker isn't enabled") + try: + setupngxblocker(self, wo_domain, False) + except SiteError as e: + Log.debug(self, str(e)) + Log.info(self, "\nngxblocker not enabled.") # Service Nginx Reload if not WOService.reload_service(self, 'nginx'): Log.error(self, "service nginx reload failed. " "check issues with `nginx -t` command") return 0 + # + if (pargs.letsencrypt == 'renew' and + not (pargs.html or + pargs.php or pargs.php73 or pargs.mysql or + pargs.wp or pargs.wpfc or pargs.wpsc or + pargs.wprocket or pargs.wpce or + pargs.wpsubdir or pargs.wpsubdomain or + pargs.ngxblocker or pargs.hsts)): + + if WOAcme.cert_check(self, wo_domain): + if not pargs.force: + if (SSL.getexpirationdays(self, wo_domain) > 45): + Log.error( + self, + 'Your certificate expire in more than 45 days.\n' + 'Add \'--force\' to force to renew') + Log.wait(self, "Renewing SSL certificate") + if WOAcme.renew(self, wo_domain): + Log.valide(self, "Renewing SSL certificate") + else: + Log.error(self, "Certificate doesn't exist") + return 0 if ((stype == 'php' and oldsitetype not in ['html', 'proxy', 'php73']) or @@ -1281,10 +1297,10 @@ class WOSiteUpdateController(CementBaseController): if not (acme_subdomain is True): if letsencrypt is check_ssl: if letsencrypt is False: - Log.error(self, "SSl is not configured for given " + Log.error(self, "SSL is not configured for given " "site") elif letsencrypt is True: - Log.error(self, "SSl is already configured for given " + Log.error(self, "SSL is already configured for given " "site") pargs.letsencrypt = False diff --git a/wo/cli/plugins/site_functions.py b/wo/cli/plugins/site_functions.py index d0924e0..d40bf6e 100644 --- a/wo/cli/plugins/site_functions.py +++ b/wo/cli/plugins/site_functions.py @@ -1554,19 +1554,31 @@ def setuprocketchat(self): def setupngxblocker(self, domain, block=True): - if os.path.isdir('/var/www/{0}/conf/nginx'.format(domain)): - if not os.path.isfile('/var/www/{0}/conf/nginx/ngxblocker.disabled' - .format(domain)): - ngxconf = open("/var/www/{0}/conf/nginx/ngxblocker.conf" - .format(domain), - encoding='utf-8', mode='w') - ngxconf.write("# Bad Bot Blocker\n" - "include /etc/nginx/bots.d/ddos.conf;\n" - "include /etc/nginx/bots.d/blockbots.conf;\n") - ngxconf.close() - else: + if block: + if os.path.isdir('/var/www/{0}/conf/nginx'.format(domain)): + if not os.path.isfile( + '/var/www/{0}/conf/nginx/ngxblocker.conf.disabled' + .format(domain)): + ngxconf = open( + "/var/www/{0}/conf/nginx/ngxblocker.conf" + .format(domain), + encoding='utf-8', mode='w') + ngxconf.write( + "# Bad Bot Blocker\n" + "include /etc/nginx/bots.d/ddos.conf;\n" + "include /etc/nginx/bots.d/blockbots.conf;\n") + ngxconf.close() + else: + WOFileUtils.mvfile( + self, '/var/www/{0}/conf/nginx/ngxblocker.conf.disabled' + .format(domain), '/var/www/{0}/conf/nginx/ngxblocker.conf' + .format(domain)) + else: + if os.path.isfile('/var/www/{0}/conf/nginx/ngxblocker.conf' + .format(domain)): WOFileUtils.mvfile( - self, '/var/www/{0}/conf/nginx/ngxblocker.disabled' - .format(domain), '/var/www/{0}/conf/nginx/ngxblocker' + self, '/var/www/{0}/conf/nginx/ngxblocker.conf' + .format(domain), + '/var/www/{0}/conf/nginx/ngxblocker.conf.disabled' .format(domain)) return 0 diff --git a/wo/core/acme.py b/wo/core/acme.py index 5b76eb5..8196238 100644 --- a/wo/core/acme.py +++ b/wo/core/acme.py @@ -132,6 +132,17 @@ class WOAcme: "ssl.conf") return 0 + def renew(self, domain): + """Renew letsencrypt certificate with acme.sh""" + try: + WOShellExec.cmd_exec( + self, "{0} ".format(WOAcme.wo_acme_exec) + + "--renew -d {0} --ecc --force".format(domain)) + except CommandExecutionError as e: + Log.debug(self, str(e)) + Log.error(self, 'Unable to renew certificate') + return True + def check_dns(self, acme_domains): """Check if a list of domains point to the server IP""" server_ip = requests.get('http://v4.wordops.eu/').text diff --git a/wo/core/logging.py b/wo/core/logging.py index 8e034f7..bb6cbda 100644 --- a/wo/core/logging.py +++ b/wo/core/logging.py @@ -44,7 +44,7 @@ class Log: """ Logs debug messages into log file """ - self.app.log.debug(Log.HEADER + msg + Log.ENDC + __name__) + self.app.log.debug(Log.HEADER + msg + Log.ENDC) def wait(self, msg, end='\r', log=True): """ From b874616343fbfa9873e64061e24d38628771774b Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Tue, 29 Oct 2019 19:26:51 +0100 Subject: [PATCH 50/79] Refactor wp-config.php setup --- wo/cli/plugins/site_functions.py | 73 +++++++++++++++----------------- 1 file changed, 34 insertions(+), 39 deletions(-) diff --git a/wo/cli/plugins/site_functions.py b/wo/cli/plugins/site_functions.py index d40bf6e..8d643f8 100644 --- a/wo/cli/plugins/site_functions.py +++ b/wo/cli/plugins/site_functions.py @@ -23,6 +23,7 @@ from wo.core.services import WOService from wo.core.shellexec import CommandExecutionError, WOShellExec from wo.core.sslutils import SSL from wo.core.variables import WOVar +from wo.core.wpcli import WOWp class SiteError(Exception): @@ -366,45 +367,33 @@ def setupwordpress(self, data, vhostonly=False): except CommandExecutionError: raise SiteError("generate wp-config failed for wp multi site") - try: + # set all wp-config.php variables + wp_conf_variables = [ + ['WP_CACHE_KEY_SALT', '{0}:'.format(wo_domain_name)], + ['WP_MEMORY_LIMIT', '128M'], + ['WP_MAX_MEMORY_LIMIT', '256M'], + ['CONCATENATE_SCRIPTS', 'false'], + ['WP_POST_REVISIONS', '10'], + ['MEDIA_TRASH', 'true'], + ['EMPTY_TRASH_DAYS', '15'], + ['WP_AUTO_UPDATE_CORE', 'minor']] - WOShellExec.cmd_exec(self, "/bin/bash -c \"{0} --allow-root " - .format(WOVar.wo_wpcli_path) + - "config set WP_CACHE_KEY_SALT " - "\'{0}:\'\"".format(wo_domain_name)) - - WOShellExec.cmd_exec(self, "/bin/bash -c \"{0} --allow-root " - .format(WOVar.wo_wpcli_path) + - "config set WP_MEMORY_LIMIT " - "\'128M\'\"") - WOShellExec.cmd_exec(self, "/bin/bash -c \"{0} --allow-root " - .format(WOVar.wo_wpcli_path) + - "config set WP_MAX_MEMORY_LIMIT " - "\'256M\'\"") - WOShellExec.cmd_exec(self, "/bin/bash -c \"{0} --allow-root " - .format(WOVar.wo_wpcli_path) + - "config set CONCATENATE_SCRIPTS " - "false --raw\"") - WOShellExec.cmd_exec(self, "/bin/bash -c \"{0} --allow-root " - .format(WOVar.wo_wpcli_path) + - "config set WP_POST_REVISIONS " - "\'10\'\"") - WOShellExec.cmd_exec(self, "/bin/bash -c \"{0} --allow-root " - .format(WOVar.wo_wpcli_path) + - "config set MEDIA_TRASH " - "true --raw\"") - WOShellExec.cmd_exec(self, "/bin/bash -c \"{0} --allow-root " - .format(WOVar.wo_wpcli_path) + - "config set EMPTY_TRASH_DAYS " - "\'15\'\"") - WOShellExec.cmd_exec(self, "/bin/bash -c \"{0} --allow-root " - .format(WOVar.wo_wpcli_path) + - "config set WP_AUTO_UPDATE_CORE " - "minor\"") - - except CommandExecutionError as e: - Log.debug(self, str(e)) - Log.error(self, "Unable to define extra variable in wp-config.php") + for wp_conf in wp_conf_variables: + wp_var = wp_conf[0] + wp_val = wp_conf[1] + if wp_val == 'true' or wp_val == 'false': + var_raw = True + try: + WOShellExec.cmd_exec( + self, "/bin/bash -c \"{0} --allow-root " + .format(WOVar.wo_wpcli_path) + + "config set {0} " + "\'{1}\' {wp_raw}\"" + .format(wp_var, wp_val, + wp_raw='--raw' if var_raw is True else '')) + except CommandExecutionError as e: + Log.debug(self, str(e)) + Log.error(self, 'Unable to define wp-config.php variables') # WOFileUtils.mvfile(self, os.getcwd()+'/wp-config.php', # os.path.abspath(os.path.join(os.getcwd(), os.pardir))) @@ -458,7 +447,8 @@ def setupwordpress(self, data, vhostonly=False): if not data['multisite']: Log.debug(self, "Creating tables for WordPress Single site") - Log.debug(self, "{0} --allow-root core install " + Log.debug( + self, "{0} --allow-root core install " .format(WOVar.wo_wpcli_path) + "--url=\'{0}\' --title=\'{0}\' --admin_name=\'{1}\' " .format(data['site_name'], wo_wp_user) + @@ -605,6 +595,11 @@ def setupwordpress(self, data, vhostonly=False): plugin_data = json.dumps(plugin_data_object) setupwp_plugin(self, 'cache-enabler', 'cache-enabler', plugin_data, data) + WOShellExec.cmd_exec( + self, "/bin/bash -c \"{0} --allow-root " + .format(WOVar.wo_wpcli_path) + + "config set WP_CACHE " + "true --raw\"") if vhostonly: try: From c859e3d4b76376a14e2240cb54c65cbb8fa799ac Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Tue, 29 Oct 2019 19:28:55 +0100 Subject: [PATCH 51/79] Fix incorrect import --- wo/cli/plugins/site_functions.py | 1 - 1 file changed, 1 deletion(-) diff --git a/wo/cli/plugins/site_functions.py b/wo/cli/plugins/site_functions.py index 8d643f8..871fe2c 100644 --- a/wo/cli/plugins/site_functions.py +++ b/wo/cli/plugins/site_functions.py @@ -23,7 +23,6 @@ from wo.core.services import WOService from wo.core.shellexec import CommandExecutionError, WOShellExec from wo.core.sslutils import SSL from wo.core.variables import WOVar -from wo.core.wpcli import WOWp class SiteError(Exception): From 56a01c3f8d1a5f9f3f31e04d48bc70d496bd6a54 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Tue, 29 Oct 2019 19:32:26 +0100 Subject: [PATCH 52/79] Fix wp_raw --- wo/cli/plugins/site_functions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/wo/cli/plugins/site_functions.py b/wo/cli/plugins/site_functions.py index 871fe2c..81e6798 100644 --- a/wo/cli/plugins/site_functions.py +++ b/wo/cli/plugins/site_functions.py @@ -389,7 +389,8 @@ def setupwordpress(self, data, vhostonly=False): "config set {0} " "\'{1}\' {wp_raw}\"" .format(wp_var, wp_val, - wp_raw='--raw' if var_raw is True else '')) + wp_raw='--raw' + if var_raw is True else '')) except CommandExecutionError as e: Log.debug(self, str(e)) Log.error(self, 'Unable to define wp-config.php variables') From 61a2201e173e4f175dc1337e680db8598c5afdae Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Tue, 29 Oct 2019 19:33:57 +0100 Subject: [PATCH 53/79] fix var_raw --- wo/cli/plugins/site_functions.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/wo/cli/plugins/site_functions.py b/wo/cli/plugins/site_functions.py index 81e6798..38912b6 100644 --- a/wo/cli/plugins/site_functions.py +++ b/wo/cli/plugins/site_functions.py @@ -382,6 +382,8 @@ def setupwordpress(self, data, vhostonly=False): wp_val = wp_conf[1] if wp_val == 'true' or wp_val == 'false': var_raw = True + else: + var_raw = False try: WOShellExec.cmd_exec( self, "/bin/bash -c \"{0} --allow-root " From 5cb8ab6268e946887085c47a452c4bd7b3c3a704 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Tue, 29 Oct 2019 22:20:36 +0100 Subject: [PATCH 54/79] Set `--force` required to renew > 30 days --- CHANGELOG.md | 1 + wo/cli/plugins/site.py | 16 ++++++++++------ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c51559c..c48c4ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Refactored WordOps download function with python3-requests - MySQL backup path changed to `/var/lib/wo-backup/mysql` - Do not check anymore if stack are installed with apt in `wo service` but only if there is a systemd service +- Refactored `--letsencrypt=renew`. Require the flag `--force` if certificate expiration is more than 45 days #### Fixed diff --git a/wo/cli/plugins/site.py b/wo/cli/plugins/site.py index 7a2be55..41ee7f3 100644 --- a/wo/cli/plugins/site.py +++ b/wo/cli/plugins/site.py @@ -1091,11 +1091,12 @@ class WOSiteUpdateController(CementBaseController): if WOAcme.cert_check(self, wo_domain): if not pargs.force: - if (SSL.getexpirationdays(self, wo_domain) > 45): + if (SSL.getexpirationdays(self, wo_domain) > 30): Log.error( - self, - 'Your certificate expire in more than 45 days.\n' - 'Add \'--force\' to force to renew') + self, "Your cert will expire in more " + "than 30 days ( " + + str(SSL.getexpirationdays(self, wo_domain)) + + " days).\nAdd \'--force\' to force to renew") Log.wait(self, "Renewing SSL certificate") if WOAcme.renew(self, wo_domain): Log.valide(self, "Renewing SSL certificate") @@ -1943,7 +1944,7 @@ class WOSiteDeleteController(CementBaseController): dict(help="forcefully delete site and configuration", action='store_true')), (['--all'], - dict(help="delete all", action='store_true')), + dict(help="delete files & db", action='store_true')), (['--db'], dict(help="delete db only", action='store_true')), (['--files'], @@ -1954,7 +1955,7 @@ class WOSiteDeleteController(CementBaseController): @expose(hide=True) def default(self): pargs = self.app.pargs - if not pargs.site_name: + if not pargs.site_name and not pargs.all: try: while not pargs.site_name: pargs.site_name = (input('Enter site name : ') @@ -1980,6 +1981,9 @@ class WOSiteDeleteController(CementBaseController): (not pargs.all)): pargs.all = True + if pargs.force: + pargs.no_prompt = True + # Gather information from wo-db for wo_domain check_site = getSiteInfo(self, wo_domain) wo_site_type = check_site.site_type From f2d4bcb353589ec76445501bad27172783c00f94 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 30 Oct 2019 04:23:20 +0100 Subject: [PATCH 55/79] Fix `wo info` and add tweaks to `wo stack upgrade` --- CHANGELOG.md | 3 ++ setup.cfg | 15 +++++++++- tests/travis.sh | 4 +-- wo/cli/plugins/info.py | 52 ++++++++++++++++++--------------- wo/cli/plugins/site.py | 26 ++++++++++------- wo/cli/plugins/stack.py | 1 + wo/cli/plugins/stack_pref.py | 38 ++++++++++++------------ wo/cli/plugins/stack_upgrade.py | 3 +- wo/core/download.py | 4 +-- wo/core/git.py | 10 ++++--- wo/core/services.py | 5 ++-- wo/core/variables.py | 6 ++-- 12 files changed, 98 insertions(+), 69 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c48c4ea..e432c45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### v3.9.x - [Unreleased] +### v3.10.0 - 2019-10-30 + #### Added - WordOps install is now installed with pip from PyPi (easier, cleaner and safer) inside a wheel @@ -35,6 +37,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - force-ssl.conf not removed after removing a site - `wo clean --opcache` not working with invalid SSL certificate - `wo stack install --cheat` wasn't working properly previously +- `wo info` failure depending on php-fpm pool name. ConfigParser will now detect the section name. ### v3.9.9.4 - 2019-10-18 diff --git a/setup.cfg b/setup.cfg index 4ee5196..514f780 100644 --- a/setup.cfg +++ b/setup.cfg @@ -12,4 +12,17 @@ cover-html-dir=coverage_report/ where=tests/ [metadata] -license-file = LICENSE \ No newline at end of file +license-file = LICENSE + +[flake8] +ignore = F405,W504,S322,S404,S603,s607,s602 +exclude = + # No need to traverse our git directory + .git, + # There's no value in checking cache directories + __pycache__, + # This contains our built documentation + build, + # This contains builds of flake8 that we don't want to check + dist +max-complexity = 10 \ No newline at end of file diff --git a/tests/travis.sh b/tests/travis.sh index 1b386d2..3691727 100644 --- a/tests/travis.sh +++ b/tests/travis.sh @@ -225,6 +225,7 @@ echo -e ' various informations ' echo -e "${CGREEN}#############################################${CEND}" wp --allow-root --info wo site info wp.net +wo info echo -e "${CGREEN}#############################################${CEND}" echo -e ' wo stack purge ' @@ -244,6 +245,3 @@ for stack in $stack_purge; do fi done -if [ -n "$1" ]; then - cat /var/log/wo/test.log | ccze -A -p syslog -fi diff --git a/wo/cli/plugins/info.py b/wo/cli/plugins/info.py index cb505fe..da52efc 100644 --- a/wo/cli/plugins/info.py +++ b/wo/cli/plugins/info.py @@ -79,20 +79,22 @@ class WOInfoController(CementBaseController): max_execution_time = config['PHP']['max_execution_time'] config.read('/etc/{0}/fpm/pool.d/www.conf'.format("php/7.2")) - www_listen = config['www-php72']['listen'] - www_ping_path = config['www-php72']['ping.path'] - www_pm_status_path = config['www-php72']['pm.status_path'] - www_pm = config['www-php72']['pm'] - www_pm_max_requests = config['www-php72']['pm.max_requests'] - www_pm_max_children = config['www-php72']['pm.max_children'] - www_pm_start_servers = config['www-php72']['pm.start_servers'] - www_pm_min_spare_servers = config['www-php72']['pm.min_spare_servers'] - www_pm_max_spare_servers = config['www-php72']['pm.max_spare_servers'] - www_request_terminate_time = (config['www-php72'] + wo_sec = (config.sections())[0] + www_listen = config[[wo_sec]]['listen'] + www_ping_path = config[[wo_sec]]['ping.path'] + www_pm_status_path = config[[wo_sec]]['pm.status_path'] + www_pm = config[[wo_sec]]['pm'] + www_pm_max_requests = config[[wo_sec]]['pm.max_requests'] + www_pm_max_children = config[[wo_sec]]['pm.max_children'] + www_pm_start_servers = config[[wo_sec]]['pm.start_servers'] + www_pm_min_spare_servers = config[[wo_sec]]['pm.min_spare_servers'] + www_pm_max_spare_servers = config[[wo_sec]]['pm.max_spare_servers'] + www_request_terminate_time = (config[[wo_sec]] ['request_terminate_timeout']) try: - www_xdebug = (config['www-php72']['php_admin_flag[xdebug.profiler_enable' - '_trigger]']) + www_xdebug = ( + config[[wo_sec]]['php_admin_flag[xdebug.profiler_enable' + '_trigger]']) except Exception as e: Log.debug(self, "{0}".format(e)) www_xdebug = 'off' @@ -156,19 +158,20 @@ class WOInfoController(CementBaseController): max_execution_time = config['PHP']['max_execution_time'] config.read('/etc/php/7.3/fpm/pool.d/www.conf') - www_listen = config['www-php73']['listen'] - www_ping_path = config['www-php73']['ping.path'] - www_pm_status_path = config['www-php73']['pm.status_path'] - www_pm = config['www-php73']['pm'] - www_pm_max_requests = config['www-php73']['pm.max_requests'] - www_pm_max_children = config['www-php73']['pm.max_children'] - www_pm_start_servers = config['www-php73']['pm.start_servers'] - www_pm_min_spare_servers = config['www-php73']['pm.min_spare_servers'] - www_pm_max_spare_servers = config['www-php73']['pm.max_spare_servers'] - www_request_terminate_time = (config['www-php73'] + wo_sec = (config.sections())[0] + www_listen = config[wo_sec]['listen'] + www_ping_path = config[wo_sec]['ping.path'] + www_pm_status_path = config[wo_sec]['pm.status_path'] + www_pm = config[wo_sec]['pm'] + www_pm_max_requests = config[wo_sec]['pm.max_requests'] + www_pm_max_children = config[wo_sec]['pm.max_children'] + www_pm_start_servers = config[wo_sec]['pm.start_servers'] + www_pm_min_spare_servers = config[wo_sec]['pm.min_spare_servers'] + www_pm_max_spare_servers = config[wo_sec]['pm.max_spare_servers'] + www_request_terminate_time = (config[wo_sec] ['request_terminate_timeout']) try: - www_xdebug = (config['www-php73'] + www_xdebug = (config[wo_sec] ['php_admin_flag[xdebug.profiler_enable' '_trigger]']) except Exception as e: @@ -266,7 +269,8 @@ class WOInfoController(CementBaseController): if self.app.pargs.nginx: if (WOAptGet.is_installed(self, 'nginx-custom') or - WOAptGet.is_installed(self, 'nginx-wo')): + WOAptGet.is_installed(self, 'nginx-wo') or + (os.path.exists('/usr/bin/nginx'))): self.info_nginx() else: Log.error(self, "Nginx is not installed") diff --git a/wo/cli/plugins/site.py b/wo/cli/plugins/site.py index 41ee7f3..3f04551 100644 --- a/wo/cli/plugins/site.py +++ b/wo/cli/plugins/site.py @@ -78,7 +78,7 @@ class WOSiteController(CementBaseController): Log.error(self, "service nginx reload failed. " "check issues with `nginx -t` command") else: - Log.error(self, "nginx configuration file does not exist") + Log.error(self, 'nginx configuration file does not exist') @expose(help="Disable site example.com") def disable(self): @@ -106,7 +106,7 @@ class WOSiteController(CementBaseController): if not os.path.isfile('/etc/nginx/sites-enabled/{0}' .format(wo_domain)): Log.debug(self, "Site {0} already disabled".format(wo_domain)) - Log.info(self, "[" + Log.FAIL + "Failed" + Log.OKBLUE+"]") + Log.info(self, "[" + Log.FAIL + "Failed" + Log.OKBLUE + "]") else: WOFileUtils.remove_symlink(self, '/etc/nginx/sites-enabled/{0}' @@ -176,7 +176,7 @@ class WOSiteController(CementBaseController): dbname=wo_db_name, dbuser=wo_db_user, php_version=php_version, dbpass=wo_db_pass, - ssl=ssl, sslprovider=sslprovider, sslexpiry=sslexpiry, + ssl=ssl, sslprovider=sslprovider, sslexpiry=sslexpiry, type=sitetype + " " + cachetype + " ({0})" .format("enabled" if siteinfo.is_enabled else "disabled")) @@ -452,7 +452,7 @@ class WOSiteCreateController(CementBaseController): if stype == 'proxy': data = dict(site_name=wo_domain, www_domain=wo_www_domain, - static=True, basic=False, php73=False, wp=False, + static=True, basic=False, php73=False, wp=False, wpfc=False, wpsc=False, wprocket=False, wpce=False, multisite=False, wpsubdir=False, webroot=wo_site_webroot) @@ -463,7 +463,7 @@ class WOSiteCreateController(CementBaseController): if pargs.php73: data = dict(site_name=wo_domain, www_domain=wo_www_domain, - static=False, basic=False, php73=True, wp=False, + static=False, basic=False, php73=True, wp=False, wpfc=False, wpsc=False, wprocket=False, wpce=False, multisite=False, wpsubdir=False, webroot=wo_site_webroot) @@ -471,7 +471,7 @@ class WOSiteCreateController(CementBaseController): if stype in ['html', 'php']: data = dict(site_name=wo_domain, www_domain=wo_www_domain, - static=True, basic=False, php73=False, wp=False, + static=True, basic=False, php73=False, wp=False, wpfc=False, wpsc=False, wprocket=False, wpce=False, multisite=False, wpsubdir=False, webroot=wo_site_webroot) @@ -483,7 +483,7 @@ class WOSiteCreateController(CementBaseController): elif stype in ['mysql', 'wp', 'wpsubdir', 'wpsubdomain']: data = dict(site_name=wo_domain, www_domain=wo_www_domain, - static=False, basic=True, wp=False, wpfc=False, + static=False, basic=True, wp=False, wpfc=False, wpsc=False, wpredis=False, wprocket=False, wpce=False, multisite=False, wpsubdir=False, webroot=wo_site_webroot, @@ -509,8 +509,7 @@ class WOSiteCreateController(CementBaseController): elif data: data['php73'] = False - if ((not pargs.wpfc) and - (not pargs.wpsc) and + if ((not pargs.wpfc) and (not pargs.wpsc) and (not pargs.wprocket) and (not pargs.wpce) and (not pargs.wpredis)): @@ -1131,7 +1130,7 @@ class WOSiteUpdateController(CementBaseController): if stype == 'php': data = dict( site_name=wo_domain, www_domain=wo_www_domain, - static=False, basic=True, wp=False, wpfc=False, + static=False, basic=True, wp=False, wpfc=False, wpsc=False, wpredis=False, wprocket=False, wpce=False, multisite=False, wpsubdir=False, webroot=wo_site_webroot, currsitetype=oldsitetype, currcachetype=oldcachetype) @@ -1140,7 +1139,7 @@ class WOSiteUpdateController(CementBaseController): data = dict( site_name=wo_domain, www_domain=wo_www_domain, - static=False, basic=True, wp=False, wpfc=False, + static=False, basic=True, wp=False, wpfc=False, wpsc=False, wpredis=False, wprocket=False, wpce=False, multisite=False, wpsubdir=False, webroot=wo_site_webroot, wo_db_name='', wo_db_user='', wo_db_pass='', @@ -1294,6 +1293,11 @@ class WOSiteUpdateController(CementBaseController): letsencrypt = False acme_subdomain = False acme_wildcard = False + else: + data['letsencrypt'] = False + letsencrypt = False + acme_subdomain = False + acme_wildcard = False if not (acme_subdomain is True): if letsencrypt is check_ssl: diff --git a/wo/cli/plugins/stack.py b/wo/cli/plugins/stack.py index 46681d7..b21257e 100644 --- a/wo/cli/plugins/stack.py +++ b/wo/cli/plugins/stack.py @@ -539,6 +539,7 @@ class WOStackController(CementBaseController): Log.info(self, "Successfully installed packages") else: return self.msg + return 0 @expose(help="Remove packages") def remove(self): diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index d55f1d2..91f8233 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -814,14 +814,14 @@ def post_pref(self, apt_packages, packages, upgrade=False): "/etc/mysql/my.cnf.default-pkg") wo_ram = psutil.virtual_memory().total / (1024 * 1024) # set InnoDB variable depending on the RAM available - wo_ram_innodb = int(wo_ram*0.3) - wo_ram_log_buffer = int(wo_ram_innodb*0.25) - wo_ram_log_size = int(wo_ram_log_buffer*0.5) + wo_ram_innodb = int(wo_ram * 0.3) + wo_ram_log_buffer = int(wo_ram_innodb * 0.25) + wo_ram_log_size = int(wo_ram_log_buffer * 0.5) if (wo_ram < 2000): wo_innodb_instance = int(1) tmp_table_size = int(32) elif (wo_ram > 2000) and (wo_ram < 64000): - wo_innodb_instance = int(wo_ram/1000) + wo_innodb_instance = int(wo_ram / 1000) tmp_table_size = int(128) elif (wo_ram > 64000): wo_innodb_instance = int(64) @@ -1015,24 +1015,26 @@ def post_pref(self, apt_packages, packages, upgrade=False): if wo_ram < 1024: Log.debug(self, "Setting maxmemory variable to " "{0} in redis.conf" - .format(int(wo_ram*1024*1024*0.1))) - WOFileUtils.searchreplace(self, - "/etc/redis/redis.conf", - "# maxmemory ", - "maxmemory {0}" - .format - (int(wo_ram*1024*1024*0.1))) + .format(int(wo_ram * 1024 * 1024 * 0.1))) + WOFileUtils.searchreplace + (self, + "/etc/redis/redis.conf", + "# maxmemory ", + "maxmemory {0}" + .format + (int(wo_ram * 1024 * 1024 * 0.1))) else: Log.debug(self, "Setting maxmemory variable to {0} " "in redis.conf" - .format(int(wo_ram*1024*1024*0.2))) - WOFileUtils.searchreplace(self, - "/etc/redis/redis.conf", - "# maxmemory ", - "maxmemory {0}" - .format - (int(wo_ram*1024*1024*0.2))) + .format(int(wo_ram * 1024 * 1024 * 0.2))) + WOFileUtils.searchreplace( + self, + "/etc/redis/redis.conf", + "# maxmemory ", + "maxmemory {0}" + .format + (int(wo_ram * 1024 * 1024 * 0.2))) Log.debug( self, "Setting maxmemory-policy variable to " diff --git a/wo/cli/plugins/stack_upgrade.py b/wo/cli/plugins/stack_upgrade.py index 976aa75..accc0cf 100644 --- a/wo/cli/plugins/stack_upgrade.py +++ b/wo/cli/plugins/stack_upgrade.py @@ -3,7 +3,7 @@ import shutil from cement.core.controller import CementBaseController, expose -from wo.cli.plugins.stack_pref import post_pref, pre_pref +from wo.cli.plugins.stack_pref import post_pref, pre_pref, pre_stack from wo.core.aptget import WOAptGet from wo.core.download import WODownload from wo.core.extract import WOExtract @@ -178,6 +178,7 @@ class WOStackUpgradeController(CementBaseController): if ((not (apt_packages)) and (not(packages))): self.app.args.print_help() else: + pre_stack(self) if (apt_packages): if (("php7.2-fpm" not in apt_packages) and ("php7.3-fpm" not in apt_packages) and diff --git a/wo/core/download.py b/wo/core/download.py index bd7a725..da4fe4f 100644 --- a/wo/core/download.py +++ b/wo/core/download.py @@ -27,8 +27,8 @@ class WODownload(): if req.encoding is None: req.encoding = 'utf-8' out_file.write(req.content) - Log.info(self, "{0}".format("[" + Log.ENDC + "Done" - + Log.OKBLUE + "]")) + Log.info(self, "{0}".format("[" + Log.ENDC + "Done" + + Log.OKBLUE + "]")) except requests.RequestException as e: Log.debug(self, "[{err}]".format(err=str(e.reason))) Log.error(self, "Unable to download file, {0}" diff --git a/wo/core/git.py b/wo/core/git.py index a157282..5bf903f 100644 --- a/wo/core/git.py +++ b/wo/core/git.py @@ -21,7 +21,7 @@ class WOGit: git = git.bake("--git-dir={0}/.git".format(path), "--work-tree={0}".format(path)) if os.path.isdir(path): - if not os.path.isdir(path+"/.git"): + if not os.path.isdir(path + "/.git"): try: Log.debug(self, "WOGit: git init at {0}" .format(path)) @@ -67,11 +67,13 @@ class WOGit: git = git.bake("--git-dir={0}/.git".format(path), "--work-tree={0}".format(path)) if os.path.isdir(path): - if not os.path.isdir(path+"/.git"): - Log.error(self, "Unable to find a git repository at {0}" + if not os.path.isdir(path + "/.git"): + Log.error( + self, "Unable to find a git repository at {0}" .format(path)) try: - Log.debug(self, "WOGit: git stash --include-untracked at {0}" + Log.debug( + self, "WOGit: git stash --include-untracked at {0}" .format(path)) git.stash("push", "--include-untracked", "-m {0}" .format(msg)) diff --git a/wo/core/services.py b/wo/core/services.py index 7b4d748..1c59e63 100644 --- a/wo/core/services.py +++ b/wo/core/services.py @@ -43,7 +43,8 @@ class WOService(): return True else: Log.debug(self, "{0}".format(retcode[1])) - Log.info(self, "[" + Log.FAIL + "Failed" + Log.OKBLUE+"]") + Log.info(self, "[" + Log.FAIL + + "Failed" + Log.OKBLUE + "]") return False except OSError as e: Log.debug(self, "{0}".format(e)) @@ -65,7 +66,7 @@ class WOService(): return True else: Log.debug(self, "{0}".format(retcode[1])) - Log.info(self, "[" + Log.FAIL + "Failed" + Log.OKBLUE+"]") + Log.info(self, "[" + Log.FAIL + "Failed" + Log.OKBLUE + "]") return False except OSError as e: Log.debug(self, "{0}".format(e)) diff --git a/wo/core/variables.py b/wo/core/variables.py index 0bafd50..d0b45ff 100644 --- a/wo/core/variables.py +++ b/wo/core/variables.py @@ -64,7 +64,7 @@ class WOVar(): # WordOps git configuration management config = configparser.ConfigParser() - config.read(os.path.expanduser("~")+'/.gitconfig') + config.read(os.path.expanduser("~") + '/.gitconfig') try: wo_user = config['user']['name'] wo_email = config['user']['email'] @@ -90,7 +90,7 @@ class WOVar(): git.config("--global", "user.email", "{0}".format(wo_email)) if not os.path.isfile('/root/.gitconfig'): - copy2(os.path.expanduser("~")+'/.gitconfig', '/root/.gitconfig') + copy2(os.path.expanduser("~") + '/.gitconfig', '/root/.gitconfig') # MySQL hostname wo_mysql_host = "" @@ -98,7 +98,7 @@ class WOVar(): if os.path.exists('/etc/mysql/conf.d/my.cnf'): cnfpath = "/etc/mysql/conf.d/my.cnf" else: - cnfpath = os.path.expanduser("~")+"/.my.cnf" + cnfpath = os.path.expanduser("~") + "/.my.cnf" if [cnfpath] == config.read(cnfpath): try: wo_mysql_host = config.get('client', 'host') From 089bb64cdb6f4af58316c60328b7ce7bfd8b547b Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 30 Oct 2019 04:28:37 +0100 Subject: [PATCH 56/79] fix `wo info` --- README.md | 2 +- install | 2 +- wo/cli/plugins/info.py | 28 ++++++++++++++-------------- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index f7d0980..fd9d101 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,7 @@ - Ubuntu 16.04 LTS (Xenial) - Ubuntu 19.04 (Disco) - Debian 9 (Stretch) -- Debian 10 (Buster) - Not ready for production +- Debian 10 (Buster) - Raspbian 9 (Stretch) - Raspbian 10 (Buster) - Testing diff --git a/install b/install index 2dd3a85..512615f 100755 --- a/install +++ b/install @@ -9,7 +9,7 @@ # ------------------------------------------------------------------------- # wget -qO wo wops.cc && sudo bash wo # ------------------------------------------------------------------------- -# Version 3.10.0 - 2019-10-28 +# Version 3.10.0 - 2019-10-30 # ------------------------------------------------------------------------- # CONTENTS diff --git a/wo/cli/plugins/info.py b/wo/cli/plugins/info.py index da52efc..0959324 100644 --- a/wo/cli/plugins/info.py +++ b/wo/cli/plugins/info.py @@ -79,22 +79,22 @@ class WOInfoController(CementBaseController): max_execution_time = config['PHP']['max_execution_time'] config.read('/etc/{0}/fpm/pool.d/www.conf'.format("php/7.2")) - wo_sec = (config.sections())[0] - www_listen = config[[wo_sec]]['listen'] - www_ping_path = config[[wo_sec]]['ping.path'] - www_pm_status_path = config[[wo_sec]]['pm.status_path'] - www_pm = config[[wo_sec]]['pm'] - www_pm_max_requests = config[[wo_sec]]['pm.max_requests'] - www_pm_max_children = config[[wo_sec]]['pm.max_children'] - www_pm_start_servers = config[[wo_sec]]['pm.start_servers'] - www_pm_min_spare_servers = config[[wo_sec]]['pm.min_spare_servers'] - www_pm_max_spare_servers = config[[wo_sec]]['pm.max_spare_servers'] - www_request_terminate_time = (config[[wo_sec]] + wo_sec = (config.sections()[0]) + www_listen = config[wo_sec]['listen'] + www_ping_path = config[wo_sec]['ping.path'] + www_pm_status_path = config[wo_sec]['pm.status_path'] + www_pm = config[wo_sec]['pm'] + www_pm_max_requests = config[wo_sec]['pm.max_requests'] + www_pm_max_children = config[wo_sec]['pm.max_children'] + www_pm_start_servers = config[wo_sec]['pm.start_servers'] + www_pm_min_spare_servers = config[wo_sec]['pm.min_spare_servers'] + www_pm_max_spare_servers = config[wo_sec]['pm.max_spare_servers'] + www_request_terminate_time = (config[wo_sec] ['request_terminate_timeout']) try: www_xdebug = ( - config[[wo_sec]]['php_admin_flag[xdebug.profiler_enable' - '_trigger]']) + config[wo_sec]['php_admin_flag[xdebug.profiler_enable' + '_trigger]']) except Exception as e: Log.debug(self, "{0}".format(e)) www_xdebug = 'off' @@ -158,7 +158,7 @@ class WOInfoController(CementBaseController): max_execution_time = config['PHP']['max_execution_time'] config.read('/etc/php/7.3/fpm/pool.d/www.conf') - wo_sec = (config.sections())[0] + wo_sec = (config.sections()[0]) www_listen = config[wo_sec]['listen'] www_ping_path = config[wo_sec]['ping.path'] www_pm_status_path = config[wo_sec]['pm.status_path'] From 3388f2d95d3b954d12394bf073f34ac33495967e Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 30 Oct 2019 04:30:14 +0100 Subject: [PATCH 57/79] Another fix for info --- wo/cli/plugins/info.py | 4 ++-- wo/cli/plugins/site_functions.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/wo/cli/plugins/info.py b/wo/cli/plugins/info.py index 0959324..e72ad84 100644 --- a/wo/cli/plugins/info.py +++ b/wo/cli/plugins/info.py @@ -79,7 +79,7 @@ class WOInfoController(CementBaseController): max_execution_time = config['PHP']['max_execution_time'] config.read('/etc/{0}/fpm/pool.d/www.conf'.format("php/7.2")) - wo_sec = (config.sections()[0]) + wo_sec = (config.sections())[0] www_listen = config[wo_sec]['listen'] www_ping_path = config[wo_sec]['ping.path'] www_pm_status_path = config[wo_sec]['pm.status_path'] @@ -158,7 +158,7 @@ class WOInfoController(CementBaseController): max_execution_time = config['PHP']['max_execution_time'] config.read('/etc/php/7.3/fpm/pool.d/www.conf') - wo_sec = (config.sections()[0]) + wo_sec = (config.sections())[0] www_listen = config[wo_sec]['listen'] www_ping_path = config[wo_sec]['ping.path'] www_pm_status_path = config[wo_sec]['pm.status_path'] diff --git a/wo/cli/plugins/site_functions.py b/wo/cli/plugins/site_functions.py index 38912b6..a592e7b 100644 --- a/wo/cli/plugins/site_functions.py +++ b/wo/cli/plugins/site_functions.py @@ -403,14 +403,14 @@ def setupwordpress(self, data, vhostonly=False): try: Log.debug(self, "Moving file from {0} to {1}".format(os.getcwd( - )+'/wp-config.php', os.path.abspath(os.path.join(os.getcwd(), - os.pardir)))) - shutil.move(os.getcwd()+'/wp-config.php', + ) + '/wp-config.php', os.path.abspath(os.path.join(os.getcwd(), + os.pardir)))) + shutil.move(os.getcwd() + '/wp-config.php', os.path.abspath(os.path.join(os.getcwd(), os.pardir))) except Exception as e: Log.debug(self, str(e)) Log.error(self, 'Unable to move file from {0} to {1}' - .format(os.getcwd()+'/wp-config.php', + .format(os.getcwd() + '/wp-config.php', os.path.abspath(os.path.join(os.getcwd(), os.pardir))), False) raise SiteError("Unable to move wp-config.php") From 97fd871edeb120426c38225f53cc7bad0862606f Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 30 Oct 2019 04:51:56 +0100 Subject: [PATCH 58/79] Another way to proceed for `wo info` --- wo/cli/plugins/info.py | 72 ++++++++++++++++++++++++------------------ 1 file changed, 42 insertions(+), 30 deletions(-) diff --git a/wo/cli/plugins/info.py b/wo/cli/plugins/info.py index e72ad84..928dc1a 100644 --- a/wo/cli/plugins/info.py +++ b/wo/cli/plugins/info.py @@ -79,22 +79,30 @@ class WOInfoController(CementBaseController): max_execution_time = config['PHP']['max_execution_time'] config.read('/etc/{0}/fpm/pool.d/www.conf'.format("php/7.2")) - wo_sec = (config.sections())[0] - www_listen = config[wo_sec]['listen'] - www_ping_path = config[wo_sec]['ping.path'] - www_pm_status_path = config[wo_sec]['pm.status_path'] - www_pm = config[wo_sec]['pm'] - www_pm_max_requests = config[wo_sec]['pm.max_requests'] - www_pm_max_children = config[wo_sec]['pm.max_children'] - www_pm_start_servers = config[wo_sec]['pm.start_servers'] - www_pm_min_spare_servers = config[wo_sec]['pm.min_spare_servers'] - www_pm_max_spare_servers = config[wo_sec]['pm.max_spare_servers'] - www_request_terminate_time = (config[wo_sec] + wo_sec = config.sections() + www_listen = config['{0}' + .format(wo_sec[0])]['listen'] + www_ping_path = config['{0}'.format(wo_sec[0])]['ping.path'] + www_pm_status_path = config['{0}'.format(wo_sec[0])]['pm.status_path'] + www_pm = config['{0}'.format(wo_sec[0])]['pm'] + www_pm_max_requests = config['{0}'.format( + wo_sec[0])]['pm.max_requests'] + www_pm_max_children = config['{0}'.format( + wo_sec[0])]['pm.max_children'] + www_pm_start_servers = config['{0}'.format( + wo_sec[0])]['pm.start_servers'] + www_pm_min_spare_servers = config['{0}'.format( + wo_sec[0])]['pm.min_spare_servers'] + www_pm_max_spare_servers = config['{0}'.format( + wo_sec[0])]['pm.max_spare_servers'] + www_request_terminate_time = (config['{0}'.format(wo_sec[0])] ['request_terminate_timeout']) try: www_xdebug = ( - config[wo_sec]['php_admin_flag[xdebug.profiler_enable' - '_trigger]']) + config['{0}' + .format(wo_sec[0])][ + 'php_admin_flag[xdebug.profiler_enable' + '_trigger]']) except Exception as e: Log.debug(self, "{0}".format(e)) www_xdebug = 'off' @@ -158,20 +166,25 @@ class WOInfoController(CementBaseController): max_execution_time = config['PHP']['max_execution_time'] config.read('/etc/php/7.3/fpm/pool.d/www.conf') - wo_sec = (config.sections())[0] - www_listen = config[wo_sec]['listen'] - www_ping_path = config[wo_sec]['ping.path'] - www_pm_status_path = config[wo_sec]['pm.status_path'] - www_pm = config[wo_sec]['pm'] - www_pm_max_requests = config[wo_sec]['pm.max_requests'] - www_pm_max_children = config[wo_sec]['pm.max_children'] - www_pm_start_servers = config[wo_sec]['pm.start_servers'] - www_pm_min_spare_servers = config[wo_sec]['pm.min_spare_servers'] - www_pm_max_spare_servers = config[wo_sec]['pm.max_spare_servers'] - www_request_terminate_time = (config[wo_sec] + wo_sec = config.sections() + www_listen = config['{0}'.format(wo_sec[0])]['listen'] + www_ping_path = config['{0}'.format(wo_sec[0])]['ping.path'] + www_pm_status_path = config['{0}'.format(wo_sec[0])]['pm.status_path'] + www_pm = config['{0}'.format(wo_sec[0])]['pm'] + www_pm_max_requests = config['{0}'.format( + wo_sec[0])]['pm.max_requests'] + www_pm_max_children = config['{0}'.format( + wo_sec[0])]['pm.max_children'] + www_pm_start_servers = config['{0}'.format( + wo_sec[0])]['pm.start_servers'] + www_pm_min_spare_servers = config['{0}'.format( + wo_sec[0])]['pm.min_spare_servers'] + www_pm_max_spare_servers = config['{0}'.format( + wo_sec[0])]['pm.max_spare_servers'] + www_request_terminate_time = (config['{0}'.format(wo_sec[0])] ['request_terminate_timeout']) try: - www_xdebug = (config[wo_sec] + www_xdebug = (config['{0}'.format(wo_sec[0])] ['php_admin_flag[xdebug.profiler_enable' '_trigger]']) except Exception as e: @@ -268,12 +281,11 @@ class WOInfoController(CementBaseController): self.app.pargs.php73 = True if self.app.pargs.nginx: - if (WOAptGet.is_installed(self, 'nginx-custom') or - WOAptGet.is_installed(self, 'nginx-wo') or - (os.path.exists('/usr/bin/nginx'))): - self.info_nginx() - else: + if ((not WOAptGet.is_installed(self, 'nginx-custom')) and + (not os.path.exists('/usr/bin/nginx'))): Log.error(self, "Nginx is not installed") + else: + self.info_nginx() if self.app.pargs.php: if WOAptGet.is_installed(self, 'php7.2-fpm'): From 1ce64e62df8045b8a3c71572341d0e61d6089c3b Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 30 Oct 2019 05:26:44 +0100 Subject: [PATCH 59/79] Fix wo_ouputhandler --- wo/cli/ext/wo_outputhandler.py | 1 - wo/cli/plugins/info.py | 61 ++++++++++++++-------------------- 2 files changed, 25 insertions(+), 37 deletions(-) diff --git a/wo/cli/ext/wo_outputhandler.py b/wo/cli/ext/wo_outputhandler.py index 47a92f8..c472420 100644 --- a/wo/cli/ext/wo_outputhandler.py +++ b/wo/cli/ext/wo_outputhandler.py @@ -19,4 +19,3 @@ class WOOutputHandler(MustacheOutputHandler): return open(full_path, encoding='utf-8', mode='r').read() else: continue - return 1 diff --git a/wo/cli/plugins/info.py b/wo/cli/plugins/info.py index 928dc1a..5e50d9d 100644 --- a/wo/cli/plugins/info.py +++ b/wo/cli/plugins/info.py @@ -80,29 +80,22 @@ class WOInfoController(CementBaseController): config.read('/etc/{0}/fpm/pool.d/www.conf'.format("php/7.2")) wo_sec = config.sections() - www_listen = config['{0}' - .format(wo_sec[0])]['listen'] - www_ping_path = config['{0}'.format(wo_sec[0])]['ping.path'] - www_pm_status_path = config['{0}'.format(wo_sec[0])]['pm.status_path'] - www_pm = config['{0}'.format(wo_sec[0])]['pm'] - www_pm_max_requests = config['{0}'.format( - wo_sec[0])]['pm.max_requests'] - www_pm_max_children = config['{0}'.format( - wo_sec[0])]['pm.max_children'] - www_pm_start_servers = config['{0}'.format( - wo_sec[0])]['pm.start_servers'] - www_pm_min_spare_servers = config['{0}'.format( - wo_sec[0])]['pm.min_spare_servers'] - www_pm_max_spare_servers = config['{0}'.format( - wo_sec[0])]['pm.max_spare_servers'] - www_request_terminate_time = (config['{0}'.format(wo_sec[0])] + config.read('/etc/{0}/fpm/pool.d/www.conf'.format("php/7.2")) + www_listen = config[wo_sec[0]]['listen'] + www_ping_path = config[wo_sec[0]]['ping.path'] + www_pm_status_path = config[wo_sec[0]]['pm.status_path'] + www_pm = config[wo_sec[0]]['pm'] + www_pm_max_requests = config[wo_sec[0]]['pm.max_requests'] + www_pm_max_children = config[wo_sec[0]]['pm.max_children'] + www_pm_start_servers = config[wo_sec[0]]['pm.start_servers'] + www_pm_min_spare_servers = config[wo_sec[0]]['pm.min_spare_servers'] + www_pm_max_spare_servers = config[wo_sec[0]]['pm.max_spare_servers'] + www_request_terminate_time = (config[wo_sec[0]] ['request_terminate_timeout']) try: www_xdebug = ( - config['{0}' - .format(wo_sec[0])][ - 'php_admin_flag[xdebug.profiler_enable' - '_trigger]']) + config[wo_sec[0]]['php_admin_flag[xdebug.profiler_enable' + '_trigger]']) except Exception as e: Log.debug(self, "{0}".format(e)) www_xdebug = 'off' @@ -167,24 +160,20 @@ class WOInfoController(CementBaseController): config.read('/etc/php/7.3/fpm/pool.d/www.conf') wo_sec = config.sections() - www_listen = config['{0}'.format(wo_sec[0])]['listen'] - www_ping_path = config['{0}'.format(wo_sec[0])]['ping.path'] - www_pm_status_path = config['{0}'.format(wo_sec[0])]['pm.status_path'] - www_pm = config['{0}'.format(wo_sec[0])]['pm'] - www_pm_max_requests = config['{0}'.format( - wo_sec[0])]['pm.max_requests'] - www_pm_max_children = config['{0}'.format( - wo_sec[0])]['pm.max_children'] - www_pm_start_servers = config['{0}'.format( - wo_sec[0])]['pm.start_servers'] - www_pm_min_spare_servers = config['{0}'.format( - wo_sec[0])]['pm.min_spare_servers'] - www_pm_max_spare_servers = config['{0}'.format( - wo_sec[0])]['pm.max_spare_servers'] - www_request_terminate_time = (config['{0}'.format(wo_sec[0])] + config.read('/etc/php/7.3/fpm/pool.d/www.conf') + www_listen = config[wo_sec[0]]['listen'] + www_ping_path = config[wo_sec[0]]['ping.path'] + www_pm_status_path = config[wo_sec[0]]['pm.status_path'] + www_pm = config[wo_sec[0]]['pm'] + www_pm_max_requests = config[wo_sec[0]]['pm.max_requests'] + www_pm_max_children = config[wo_sec[0]]['pm.max_children'] + www_pm_start_servers = config[wo_sec[0]]['pm.start_servers'] + www_pm_min_spare_servers = config[wo_sec[0]]['pm.min_spare_servers'] + www_pm_max_spare_servers = config[wo_sec[0]]['pm.max_spare_servers'] + www_request_terminate_time = (config[wo_sec[0]] ['request_terminate_timeout']) try: - www_xdebug = (config['{0}'.format(wo_sec[0])] + www_xdebug = (config[wo_sec[0]] ['php_admin_flag[xdebug.profiler_enable' '_trigger]']) except Exception as e: From 4652ea6ceb6d8a10c48e71f78f8af556b9ddb729 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 30 Oct 2019 05:30:13 +0100 Subject: [PATCH 60/79] Fix info --- wo/cli/plugins/info.py | 48 +++++++++++++++++++----------------------- 1 file changed, 22 insertions(+), 26 deletions(-) diff --git a/wo/cli/plugins/info.py b/wo/cli/plugins/info.py index 5e50d9d..7f651fa 100644 --- a/wo/cli/plugins/info.py +++ b/wo/cli/plugins/info.py @@ -79,22 +79,20 @@ class WOInfoController(CementBaseController): max_execution_time = config['PHP']['max_execution_time'] config.read('/etc/{0}/fpm/pool.d/www.conf'.format("php/7.2")) - wo_sec = config.sections() - config.read('/etc/{0}/fpm/pool.d/www.conf'.format("php/7.2")) - www_listen = config[wo_sec[0]]['listen'] - www_ping_path = config[wo_sec[0]]['ping.path'] - www_pm_status_path = config[wo_sec[0]]['pm.status_path'] - www_pm = config[wo_sec[0]]['pm'] - www_pm_max_requests = config[wo_sec[0]]['pm.max_requests'] - www_pm_max_children = config[wo_sec[0]]['pm.max_children'] - www_pm_start_servers = config[wo_sec[0]]['pm.start_servers'] - www_pm_min_spare_servers = config[wo_sec[0]]['pm.min_spare_servers'] - www_pm_max_spare_servers = config[wo_sec[0]]['pm.max_spare_servers'] - www_request_terminate_time = (config[wo_sec[0]] + www_listen = config['www-php72']['listen'] + www_ping_path = config['www-php72']['ping.path'] + www_pm_status_path = config['www-php72']['pm.status_path'] + www_pm = config['www-php72']['pm'] + www_pm_max_requests = config['www-php72']['pm.max_requests'] + www_pm_max_children = config['www-php72']['pm.max_children'] + www_pm_start_servers = config['www-php72']['pm.start_servers'] + www_pm_min_spare_servers = config['www-php72']['pm.min_spare_servers'] + www_pm_max_spare_servers = config['www-php72']['pm.max_spare_servers'] + www_request_terminate_time = (config['www-php72'] ['request_terminate_timeout']) try: www_xdebug = ( - config[wo_sec[0]]['php_admin_flag[xdebug.profiler_enable' + config['www-php72']['php_admin_flag[xdebug.profiler_enable' '_trigger]']) except Exception as e: Log.debug(self, "{0}".format(e)) @@ -159,21 +157,19 @@ class WOInfoController(CementBaseController): max_execution_time = config['PHP']['max_execution_time'] config.read('/etc/php/7.3/fpm/pool.d/www.conf') - wo_sec = config.sections() - config.read('/etc/php/7.3/fpm/pool.d/www.conf') - www_listen = config[wo_sec[0]]['listen'] - www_ping_path = config[wo_sec[0]]['ping.path'] - www_pm_status_path = config[wo_sec[0]]['pm.status_path'] - www_pm = config[wo_sec[0]]['pm'] - www_pm_max_requests = config[wo_sec[0]]['pm.max_requests'] - www_pm_max_children = config[wo_sec[0]]['pm.max_children'] - www_pm_start_servers = config[wo_sec[0]]['pm.start_servers'] - www_pm_min_spare_servers = config[wo_sec[0]]['pm.min_spare_servers'] - www_pm_max_spare_servers = config[wo_sec[0]]['pm.max_spare_servers'] - www_request_terminate_time = (config[wo_sec[0]] + www_listen = config['www-php73']['listen'] + www_ping_path = config['www-php73']['ping.path'] + www_pm_status_path = config['www-php73']['pm.status_path'] + www_pm = config['www-php73']['pm'] + www_pm_max_requests = config['www-php73']['pm.max_requests'] + www_pm_max_children = config['www-php73']['pm.max_children'] + www_pm_start_servers = config['www-php73']['pm.start_servers'] + www_pm_min_spare_servers = config['www-php73']['pm.min_spare_servers'] + www_pm_max_spare_servers = config['www-php73']['pm.max_spare_servers'] + www_request_terminate_time = (config['www-php73'] ['request_terminate_timeout']) try: - www_xdebug = (config[wo_sec[0]] + www_xdebug = (config['www-php73'] ['php_admin_flag[xdebug.profiler_enable' '_trigger]']) except Exception as e: From 270fb478a574fe6401c4c546b70bb9b9045f8836 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 30 Oct 2019 06:07:01 +0100 Subject: [PATCH 61/79] Final fix for `wo info` --- wo/cli/plugins/info.py | 52 ++++++++++++++++++++++++------------------ 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/wo/cli/plugins/info.py b/wo/cli/plugins/info.py index 7f651fa..280ba79 100644 --- a/wo/cli/plugins/info.py +++ b/wo/cli/plugins/info.py @@ -79,20 +79,24 @@ class WOInfoController(CementBaseController): max_execution_time = config['PHP']['max_execution_time'] config.read('/etc/{0}/fpm/pool.d/www.conf'.format("php/7.2")) - www_listen = config['www-php72']['listen'] - www_ping_path = config['www-php72']['ping.path'] - www_pm_status_path = config['www-php72']['pm.status_path'] - www_pm = config['www-php72']['pm'] - www_pm_max_requests = config['www-php72']['pm.max_requests'] - www_pm_max_children = config['www-php72']['pm.max_children'] - www_pm_start_servers = config['www-php72']['pm.start_servers'] - www_pm_min_spare_servers = config['www-php72']['pm.min_spare_servers'] - www_pm_max_spare_servers = config['www-php72']['pm.max_spare_servers'] - www_request_terminate_time = (config['www-php72'] + if config.has_section('www'): + wconfig = config['www'] + elif config.has_section['www-php72']: + wconfig = config['www-php72'] + www_listen = wconfig['listen'] + www_ping_path = wconfig['ping.path'] + www_pm_status_path = wconfig['pm.status_path'] + www_pm = wconfig['pm'] + www_pm_max_requests = wconfig['pm.max_requests'] + www_pm_max_children = wconfig['pm.max_children'] + www_pm_start_servers = wconfig['pm.start_servers'] + www_pm_min_spare_servers = wconfig['pm.min_spare_servers'] + www_pm_max_spare_servers = wconfig['pm.max_spare_servers'] + www_request_terminate_time = (wconfig ['request_terminate_timeout']) try: www_xdebug = ( - config['www-php72']['php_admin_flag[xdebug.profiler_enable' + wconfig['php_admin_flag[xdebug.profiler_enable' '_trigger]']) except Exception as e: Log.debug(self, "{0}".format(e)) @@ -157,19 +161,23 @@ class WOInfoController(CementBaseController): max_execution_time = config['PHP']['max_execution_time'] config.read('/etc/php/7.3/fpm/pool.d/www.conf') - www_listen = config['www-php73']['listen'] - www_ping_path = config['www-php73']['ping.path'] - www_pm_status_path = config['www-php73']['pm.status_path'] - www_pm = config['www-php73']['pm'] - www_pm_max_requests = config['www-php73']['pm.max_requests'] - www_pm_max_children = config['www-php73']['pm.max_children'] - www_pm_start_servers = config['www-php73']['pm.start_servers'] - www_pm_min_spare_servers = config['www-php73']['pm.min_spare_servers'] - www_pm_max_spare_servers = config['www-php73']['pm.max_spare_servers'] - www_request_terminate_time = (config['www-php73'] + if config.has_section('www'): + wconfig = config['www'] + elif config.has_section['www-php73']: + wconfig = config['www-php73'] + www_listen = wconfig['listen'] + www_ping_path = wconfig['ping.path'] + www_pm_status_path = wconfig['pm.status_path'] + www_pm = wconfig['pm'] + www_pm_max_requests = wconfig['pm.max_requests'] + www_pm_max_children = wconfig['pm.max_children'] + www_pm_start_servers = wconfig['pm.start_servers'] + www_pm_min_spare_servers = wconfig['pm.min_spare_servers'] + www_pm_max_spare_servers = wconfig['pm.max_spare_servers'] + www_request_terminate_time = (wconfig ['request_terminate_timeout']) try: - www_xdebug = (config['www-php73'] + www_xdebug = (wconfig ['php_admin_flag[xdebug.profiler_enable' '_trigger]']) except Exception as e: From 715b31329a7aeca79b3ad4c6e43385c42b680ea6 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 30 Oct 2019 06:07:40 +0100 Subject: [PATCH 62/79] fix indentation --- wo/cli/plugins/info.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/wo/cli/plugins/info.py b/wo/cli/plugins/info.py index 280ba79..1061ffd 100644 --- a/wo/cli/plugins/info.py +++ b/wo/cli/plugins/info.py @@ -93,11 +93,11 @@ class WOInfoController(CementBaseController): www_pm_min_spare_servers = wconfig['pm.min_spare_servers'] www_pm_max_spare_servers = wconfig['pm.max_spare_servers'] www_request_terminate_time = (wconfig - ['request_terminate_timeout']) + ['request_terminate_timeout']) try: www_xdebug = ( wconfig['php_admin_flag[xdebug.profiler_enable' - '_trigger]']) + '_trigger]']) except Exception as e: Log.debug(self, "{0}".format(e)) www_xdebug = 'off' @@ -175,7 +175,7 @@ class WOInfoController(CementBaseController): www_pm_min_spare_servers = wconfig['pm.min_spare_servers'] www_pm_max_spare_servers = wconfig['pm.max_spare_servers'] www_request_terminate_time = (wconfig - ['request_terminate_timeout']) + ['request_terminate_timeout']) try: www_xdebug = (wconfig ['php_admin_flag[xdebug.profiler_enable' From 04361eecf5c9564340867eec05cdbda1d9e8a3d3 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 30 Oct 2019 06:09:35 +0100 Subject: [PATCH 63/79] Fix has_section --- wo/cli/plugins/info.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wo/cli/plugins/info.py b/wo/cli/plugins/info.py index 1061ffd..488e0db 100644 --- a/wo/cli/plugins/info.py +++ b/wo/cli/plugins/info.py @@ -81,7 +81,7 @@ class WOInfoController(CementBaseController): config.read('/etc/{0}/fpm/pool.d/www.conf'.format("php/7.2")) if config.has_section('www'): wconfig = config['www'] - elif config.has_section['www-php72']: + elif config.has_section('www-php72'): wconfig = config['www-php72'] www_listen = wconfig['listen'] www_ping_path = wconfig['ping.path'] @@ -163,7 +163,7 @@ class WOInfoController(CementBaseController): config.read('/etc/php/7.3/fpm/pool.d/www.conf') if config.has_section('www'): wconfig = config['www'] - elif config.has_section['www-php73']: + elif config.has_section('www-php73'): wconfig = config['www-php73'] www_listen = wconfig['listen'] www_ping_path = wconfig['ping.path'] From d3a49478310bb623cd60f48a8e9ce26dbfb81ac2 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 30 Oct 2019 06:11:57 +0100 Subject: [PATCH 64/79] Add Log.error in case of failure --- wo/cli/plugins/info.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/wo/cli/plugins/info.py b/wo/cli/plugins/info.py index 488e0db..2d04b0e 100644 --- a/wo/cli/plugins/info.py +++ b/wo/cli/plugins/info.py @@ -83,6 +83,8 @@ class WOInfoController(CementBaseController): wconfig = config['www'] elif config.has_section('www-php72'): wconfig = config['www-php72'] + else: + Log.error(self, 'Unable to parse configuration') www_listen = wconfig['listen'] www_ping_path = wconfig['ping.path'] www_pm_status_path = wconfig['pm.status_path'] @@ -165,6 +167,8 @@ class WOInfoController(CementBaseController): wconfig = config['www'] elif config.has_section('www-php73'): wconfig = config['www-php73'] + else: + Log.error(self, 'Unable to parse configuration') www_listen = wconfig['listen'] www_ping_path = wconfig['ping.path'] www_pm_status_path = wconfig['pm.status_path'] From 7c6b60188cecad54d9f30853a62dd0df076e52ce Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 30 Oct 2019 06:14:38 +0100 Subject: [PATCH 65/79] Check if file exist before parsing --- wo/cli/plugins/info.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/wo/cli/plugins/info.py b/wo/cli/plugins/info.py index 2d04b0e..602a6ce 100644 --- a/wo/cli/plugins/info.py +++ b/wo/cli/plugins/info.py @@ -78,7 +78,10 @@ class WOInfoController(CementBaseController): upload_max_filesize = config['PHP']['upload_max_filesize'] max_execution_time = config['PHP']['max_execution_time'] - config.read('/etc/{0}/fpm/pool.d/www.conf'.format("php/7.2")) + if os.path.exists('/etc/php/7.2/fpm/pool.d/www.conf'): + config.read('/etc/php/7.2/fpm/pool.d/www.conf') + else: + Log.error(self, 'php-fpm pool config not found') if config.has_section('www'): wconfig = config['www'] elif config.has_section('www-php72'): @@ -162,7 +165,10 @@ class WOInfoController(CementBaseController): upload_max_filesize = config['PHP']['upload_max_filesize'] max_execution_time = config['PHP']['max_execution_time'] - config.read('/etc/php/7.3/fpm/pool.d/www.conf') + if os.path.exists('/etc/php/7.3/fpm/pool.d/www.conf'): + config.read('/etc/php/7.3/fpm/pool.d/www.conf') + else: + Log.error(self, 'php-fpm pool config not found') if config.has_section('www'): wconfig = config['www'] elif config.has_section('www-php73'): From fdfe60315c487fd2458467d12c1408b77b555d3a Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 30 Oct 2019 06:18:34 +0100 Subject: [PATCH 66/79] Update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e432c45..1c1f1b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), #### Added -- WordOps install is now installed with pip from PyPi (easier, cleaner and safer) inside a wheel +- WordOps is now installed inside a wheel with pip (easier, cleaner and safer) from PyPi - Redis 5.0.6 package backported to Debian 8/9/10 - Custom motd to display a message if a new WordOps release is available - Run `mysql_upgrade` during MySQL upgrade with `wo stack upgrade` to perform migration if needed From fb61a9697f0c60bd60fd87588bff2908a272a78e Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 30 Oct 2019 06:20:22 +0100 Subject: [PATCH 67/79] Fix extra argument --- wo/cli/plugins/site.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/wo/cli/plugins/site.py b/wo/cli/plugins/site.py index 3f04551..71a6fee 100644 --- a/wo/cli/plugins/site.py +++ b/wo/cli/plugins/site.py @@ -308,8 +308,7 @@ class WOSiteEditController(CementBaseController): Log.error(self, "service nginx reload failed. " "check issues with `nginx -t` command") else: - Log.error(self, "nginx configuration file does not exists" - .format(wo_domain)) + Log.error(self, "nginx configuration file does not exists") class WOSiteCreateController(CementBaseController): From 8fd202af3c5c4f5b79673bcc6d188dd062f3c945 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 30 Oct 2019 06:21:49 +0100 Subject: [PATCH 68/79] Fix searchreplace --- wo/cli/plugins/stack_pref.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index 91f8233..50001fd 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -1016,13 +1016,13 @@ def post_pref(self, apt_packages, packages, upgrade=False): Log.debug(self, "Setting maxmemory variable to " "{0} in redis.conf" .format(int(wo_ram * 1024 * 1024 * 0.1))) - WOFileUtils.searchreplace - (self, - "/etc/redis/redis.conf", - "# maxmemory ", - "maxmemory {0}" - .format - (int(wo_ram * 1024 * 1024 * 0.1))) + WOFileUtils.searchreplace( + self, + "/etc/redis/redis.conf", + "# maxmemory ", + "maxmemory {0}" + .format + (int(wo_ram * 1024 * 1024 * 0.1))) else: Log.debug(self, "Setting maxmemory variable to {0} " From 7d28d66d113dd15f306629e1a945e18583645542 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 30 Oct 2019 12:06:42 +0100 Subject: [PATCH 69/79] Add `wo stack upgrade --ngxblocker` --- CHANGELOG.md | 2 + tests/travis.sh | 2 +- wo/cli/plugins/stack_upgrade.py | 140 ++++++++++++++++++++++---------- 3 files changed, 102 insertions(+), 42 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c1f1b6..b9ca680 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Redis 5.0.6 package backported to Debian 8/9/10 - Custom motd to display a message if a new WordOps release is available - Run `mysql_upgrade` during MySQL upgrade with `wo stack upgrade` to perform migration if needed +- `wo stack upgrade --ngxblocker` to update ngxblocker blocklist #### Changed @@ -30,6 +31,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - MySQL backup path changed to `/var/lib/wo-backup/mysql` - Do not check anymore if stack are installed with apt in `wo service` but only if there is a systemd service - Refactored `--letsencrypt=renew`. Require the flag `--force` if certificate expiration is more than 45 days +- Improve netdata stack upgrade with install from source detection and updater fallback #### Fixed diff --git a/tests/travis.sh b/tests/travis.sh index 3691727..6ebaa72 100644 --- a/tests/travis.sh +++ b/tests/travis.sh @@ -145,7 +145,7 @@ if [ -z "$1" ]; then echo -e "${CGREEN}#############################################${CEND}" echo -e ' wo stack upgrade ' echo -e "${CGREEN}#############################################${CEND}" - stack_upgrade='nginx php php73 mysql redis netdata dashboard phpmyadmin composer' + stack_upgrade='nginx php php73 mysql redis netdata dashboard phpmyadmin composer ngxblocker' for stack in $stack_upgrade; do echo -ne " Upgrading $stack [..]\r" if { diff --git a/wo/cli/plugins/stack_upgrade.py b/wo/cli/plugins/stack_upgrade.py index accc0cf..cd860b5 100644 --- a/wo/cli/plugins/stack_upgrade.py +++ b/wo/cli/plugins/stack_upgrade.py @@ -46,6 +46,8 @@ class WOStackUpgradeController(CementBaseController): dict(help='Upgrade Composer', action='store_true')), (['--phpmyadmin'], dict(help='Upgrade phpMyAdmin', action='store_true')), + (['--ngxblocker'], + dict(help='Upgrade phpMyAdmin', action='store_true')), (['--no-prompt'], dict(help="Upgrade Packages without any prompt", action='store_true')), @@ -70,24 +72,29 @@ class WOStackUpgradeController(CementBaseController): (not pargs.phpmyadmin) and (not pargs.dashboard) and (not pargs.redis)): pargs.web = True + pargs.admin = True if pargs.all: pargs.web = True - pargs.netdata = True - pargs.composer = True - pargs.dashboard = True - pargs.phpmyadmin = True + pargs.admin = True pargs.redis = True - pargs.wpcli = True pargs.php73 = True + pargs.ngxblocker = True if pargs.web: - if WOAptGet.is_installed(self, 'nginx-custom'): - pargs.nginx = True + pargs.nginx = True pargs.php = True pargs.mysql = True pargs.wpcli = True + if pargs.admin: + pargs.netdata = True + pargs.composer = True + pargs.dashboard = True + pargs.phpmyadmin = True + pargs.wpcli = True + + # nginx if pargs.nginx: if WOAptGet.is_installed(self, 'nginx-custom'): apt_packages = apt_packages + WOVar.wo_nginx @@ -98,6 +105,7 @@ class WOStackUpgradeController(CementBaseController): else: Log.info(self, "Nginx Stable is not already installed") + # php 7.2 if pargs.php: if WOAptGet.is_installed(self, 'php7.2-fpm'): apt_packages = apt_packages + WOVar.wo_php + \ @@ -105,6 +113,7 @@ class WOStackUpgradeController(CementBaseController): else: Log.info(self, "PHP 7.2 is not installed") + # php 7.3 if pargs.php73: if WOAptGet.is_installed(self, 'php7.3-fpm'): apt_packages = apt_packages + WOVar.wo_php73 + \ @@ -112,18 +121,21 @@ class WOStackUpgradeController(CementBaseController): else: Log.info(self, "PHP 7.3 is not installed") + # mysql if pargs.mysql: if WOShellExec.cmd_exec(self, 'mysqladmin ping'): apt_packages = apt_packages + ['mariadb-server'] else: Log.info(self, "MariaDB is not installed") + # redis if pargs.redis: if WOAptGet.is_installed(self, 'redis-server'): apt_packages = apt_packages + ['redis-server'] else: Log.info(self, "Redis is not installed") + # wp-cli if pargs.wpcli: if os.path.isfile('/usr/local/bin/wp'): packages = packages + [[ @@ -135,37 +147,48 @@ class WOStackUpgradeController(CementBaseController): else: Log.info(self, "WPCLI is not installed with WordOps") + # netdata if pargs.netdata: - if (os.path.isdir('/opt/netdata') or - os.path.isdir('/etc/netdata')): - packages = packages + [['https://my-netdata.io/' - 'kickstart-static64.sh', - '/var/lib/wo/tmp/kickstart.sh', - 'Netdata']] + # detect static binaries install + if os.path.isdir('/opt/netdata'): + packages = packages + [[ + 'https://my-netdata.io/kickstart-static64.sh', + '/var/lib/wo/tmp/kickstart.sh', 'Netdata']] + # detect install from source + elif os.path.isdir('/etc/netdata'): + packages = packages + [[ + 'https://my-netdata.io/kickstart.sh', + '/var/lib/wo/tmp/kickstart.sh', 'Netdata']] + else: + Log.info(self, 'Netdata us not installed') + # wordops dashboard if pargs.dashboard: if (os.path.isfile('/var/www/22222/htdocs/index.php') or os.path.isfile('/var/www/22222/htdocs/index.html')): - packages = packages + \ - [["https://github.com/WordOps/wordops-dashboard/" - "releases/download/v{0}/wordops-dashboard.tar.gz" - .format(WOVar.wo_dashboard), - "/var/lib/wo/tmp/wo-dashboard.tar.gz", - "WordOps Dashboard"]] + packages = packages + [[ + "https://github.com/WordOps/wordops-dashboard/" + "releases/download/v{0}/wordops-dashboard.tar.gz" + .format(WOVar.wo_dashboard), + "/var/lib/wo/tmp/wo-dashboard.tar.gz", + "WordOps Dashboard"]] + else: + Log.info(self, 'WordOps dashboard is not installed') + # phpmyadmin if pargs.phpmyadmin: if os.path.isdir('/var/www/22222/htdocs/db/pma'): - packages = packages + \ - [["https://files.phpmyadmin.net" - "/phpMyAdmin/{0}/" - "phpMyAdmin-{0}-" - "all-languages" - ".tar.gz".format(WOVar.wo_phpmyadmin), - "/var/lib/wo/tmp/pma.tar.gz", - "PHPMyAdmin"]] + packages = packages + [[ + "https://files.phpmyadmin.net" + "/phpMyAdmin/{0}/phpMyAdmin-{0}-" + "all-languages.tar.gz" + .format(WOVar.wo_phpmyadmin), + "/var/lib/wo/tmp/pma.tar.gz", + "PHPMyAdmin"]] else: Log.info(self, "phpMyAdmin isn't installed") + # composer if pargs.composer: if os.path.isfile('/usr/local/bin/composer'): packages = packages + [[ @@ -175,6 +198,18 @@ class WOStackUpgradeController(CementBaseController): else: Log.info(self, "Composer isn't installed") + # ngxblocker + if pargs.ngxblocker: + if os.path.exists('/usr/local/sbin/update-ngxblocker'): + packages = packages + [[ + 'https://raw.githubusercontent.com/mitchellkrogza/' + 'nginx-ultimate-bad-bot-blocker/master/update-ngxblocker', + '/usr/local/sbin/update-ngxblocker', + 'ngxblocker' + ]] + else: + Log.info(self, "ngxblocker is not installed") + if ((not (apt_packages)) and (not(packages))): self.app.args.print_help() else: @@ -198,7 +233,6 @@ class WOStackUpgradeController(CementBaseController): # apt-get update WOAptGet.update(self) Log.valide(self, "Updating APT packages") - Log.wait(self, "Upgrading APT Packages") # additional pre_pref if "nginx-custom" in apt_packages: @@ -215,7 +249,6 @@ class WOStackUpgradeController(CementBaseController): post_pref(self, WOVar.wo_nginx, [], True) # upgrade packages WOAptGet.install(self, apt_packages) - Log.valide(self, "Upgrading APT Packages") Log.wait(self, "Configuring APT Packages") post_pref(self, apt_packages, [], True) if "mariadb-server" in apt_packages: @@ -230,6 +263,9 @@ class WOStackUpgradeController(CementBaseController): if pargs.netdata: WOFileUtils.rm(self, '/var/lib/wo/tmp/kickstart.sh') + if pargs.ngxblocker: + WOFileUtils.rm(self, '/usr/local/sbin/update-ngxblocker') + if pargs.dashboard: if os.path.isfile('/var/www/22222/htdocs/index.php'): WOFileUtils.rm(self, '/var/www/22222/htdocs/index.php') @@ -243,18 +279,38 @@ class WOStackUpgradeController(CementBaseController): if pargs.wpcli: WOFileUtils.chmod(self, "/usr/local/bin/wp", 0o775) + if pargs.ngxblocker: + WOFileUtils.chmod( + self, '/usr/local/sbin/update-ngxblocker', 0o700) + WOShellExec.chmod( + self, '/usr/local/sbin/update-ngxblocker -nq' + ) + + # Netdata if pargs.netdata: Log.wait(self, "Upgrading Netdata") + # detect static binaries install if os.path.isdir('/opt/netdata'): - WOShellExec.cmd_exec( - self, "bash /opt/netdata/usr/" - "libexec/netdata/netdata-" - "updater.sh") + if os.path.exists( + '/opt/netdata/usr/libexec/' + 'netdata/netdata-updater.sh'): + WOShellExec.cmd_exec( + self, "bash /opt/netdata/usr/" + "libexec/netdata/netdata-" + "updater.sh") + else: + WOShellExec.cmd_exec( + self, "bash /var/lib/wo/tmp/kickstart.sh") + # detect install from source elif os.path.isdir('/etc/netdata'): - WOShellExec.cmd_exec( - self, "bash /usr/" - "libexec/netdata/netdata-" - "updater.sh") + if os.path.exists( + '/usr/libexec/netdata/netdata-updater.sh'): + WOShellExec.cmd_exec( + self, + 'bash /usr/libexec/netdata/netdata-updater.sh') + else: + WOShellExec.cmd_exec( + self, "bash /var/lib/wo/tmp/kickstart.sh") Log.valide(self, "Upgrading Netdata") if pargs.dashboard: @@ -269,10 +325,12 @@ class WOStackUpgradeController(CementBaseController): if pargs.composer: Log.wait(self, "Upgrading Composer") - WOShellExec.cmd_exec( - self, "php -q /var/lib/wo" - "/tmp/composer-install " - "--install-dir=/var/lib/wo/tmp/") + if WOShellExec.cmd_exec( + self, '/usr/bin/php -v'): + WOShellExec.cmd_exec( + self, "php -q /var/lib/wo" + "/tmp/composer-install " + "--install-dir=/var/lib/wo/tmp/") shutil.copyfile('/var/lib/wo/tmp/composer.phar', '/usr/local/bin/composer') WOFileUtils.chmod(self, "/usr/local/bin/composer", 0o775) From 7b82aae8ce4750895c38cbde9831529e4bba77c4 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 30 Oct 2019 12:09:25 +0100 Subject: [PATCH 70/79] Fix stack upgrade --- wo/cli/plugins/stack_upgrade.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wo/cli/plugins/stack_upgrade.py b/wo/cli/plugins/stack_upgrade.py index cd860b5..ab12b55 100644 --- a/wo/cli/plugins/stack_upgrade.py +++ b/wo/cli/plugins/stack_upgrade.py @@ -66,7 +66,7 @@ class WOStackUpgradeController(CementBaseController): if ((not pargs.web) and (not pargs.nginx) and (not pargs.php) and (not pargs.php73) and - (not pargs.mysql) and + (not pargs.mysql) and (not pargs.ngxblocker) and (not pargs.all) and (not pargs.wpcli) and (not pargs.netdata) and (not pargs.composer) and (not pargs.phpmyadmin) and (not pargs.dashboard) and From 4c6a0be680ee3f4e8abf24348330d2b7eae302a5 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 30 Oct 2019 12:19:39 +0100 Subject: [PATCH 71/79] Fix shellexec --- wo/cli/plugins/stack_upgrade.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wo/cli/plugins/stack_upgrade.py b/wo/cli/plugins/stack_upgrade.py index ab12b55..401ca44 100644 --- a/wo/cli/plugins/stack_upgrade.py +++ b/wo/cli/plugins/stack_upgrade.py @@ -282,7 +282,7 @@ class WOStackUpgradeController(CementBaseController): if pargs.ngxblocker: WOFileUtils.chmod( self, '/usr/local/sbin/update-ngxblocker', 0o700) - WOShellExec.chmod( + WOShellExec.shellexec( self, '/usr/local/sbin/update-ngxblocker -nq' ) From 250ea66aa42c41201dae2f3e3e7c8217d07673fb Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 30 Oct 2019 12:44:36 +0100 Subject: [PATCH 72/79] Fix cmd_exec --- wo/cli/plugins/stack_upgrade.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wo/cli/plugins/stack_upgrade.py b/wo/cli/plugins/stack_upgrade.py index 401ca44..64e75e1 100644 --- a/wo/cli/plugins/stack_upgrade.py +++ b/wo/cli/plugins/stack_upgrade.py @@ -282,7 +282,7 @@ class WOStackUpgradeController(CementBaseController): if pargs.ngxblocker: WOFileUtils.chmod( self, '/usr/local/sbin/update-ngxblocker', 0o700) - WOShellExec.shellexec( + WOShellExec.cmd_exec( self, '/usr/local/sbin/update-ngxblocker -nq' ) From 8ce4a206a942b480fb65ebf1b66a9abb82647be9 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 30 Oct 2019 12:48:40 +0100 Subject: [PATCH 73/79] Fix slow-log-db --- wo/cli/plugins/log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wo/cli/plugins/log.py b/wo/cli/plugins/log.py index 54b23de..f108f3c 100644 --- a/wo/cli/plugins/log.py +++ b/wo/cli/plugins/log.py @@ -210,7 +210,7 @@ class WOLogResetController(CementBaseController): if ((not self.app.pargs.nginx) and (not self.app.pargs.fpm) and (not self.app.pargs.mysql) and (not self.app.pargs.access) and (not self.app.pargs.wp) and (self.app.pargs.site_name) and - (not self.app.pargs.slow-log-db)): + (not self.app.pargs.slow_log_db)): self.app.pargs.nginx = True self.app.pargs.wp = True self.app.pargs.access = True From 37c937f006a5e324fa98e652c10d5a12c4b56eb1 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 30 Oct 2019 13:05:08 +0100 Subject: [PATCH 74/79] Improve return function --- wo/cli/plugins/site.py | 36 ++++++++---------------------------- wo/core/checkfqdn.py | 5 +---- wo/core/shellexec.py | 11 +++-------- 3 files changed, 12 insertions(+), 40 deletions(-) diff --git a/wo/cli/plugins/site.py b/wo/cli/plugins/site.py index 71a6fee..31373ab 100644 --- a/wo/cli/plugins/site.py +++ b/wo/cli/plugins/site.py @@ -637,10 +637,7 @@ class WOSiteCreateController(CementBaseController): # Setup WordPress if Wordpress site if data['wp']: - if pargs.vhostonly: - vhostonly = True - else: - vhostonly = False + vhostonly = bool(pargs.vhostonly) try: wo_wp_creds = setupwordpress(self, data, vhostonly) # Add database information for site into database @@ -1261,10 +1258,7 @@ class WOSiteUpdateController(CementBaseController): if pargs.letsencrypt == 'on': data['letsencrypt'] = True letsencrypt = True - if (wo_domain_type == 'subdomain'): - acme_subdomain = True - else: - acme_subdomain = False + acme_subdomain = bool(wo_domain_type == 'subdomain') acme_wildcard = False elif pargs.letsencrypt == 'subdomain': data['letsencrypt'] = True @@ -1316,19 +1310,11 @@ class WOSiteUpdateController(CementBaseController): return 0 if data and (not pargs.php73): - if old_php73 is True: - data['php73'] = True - php73 = True - else: - data['php73'] = False - php73 = False + data['php73'] = bool(old_php73 is True) + php73 = bool(old_php73 is True) - if pargs.php73 == "on": - data['php73'] = True - php73 = True - else: - data['php73'] = False - php73 = False + data['php73'] = bool(pargs.php73 == "on") + php73 = bool(pargs.php73 == "on") if pargs.wpredis and data['currcachetype'] != 'wpredis': data['wpredis'] = True @@ -1350,16 +1336,10 @@ class WOSiteUpdateController(CementBaseController): return 1 if pargs.hsts: - if pargs.hsts == "on": - data['hsts'] = True - elif pargs.hsts == "off": - data['hsts'] = False + data['hsts'] = bool(pargs.hsts == "on") if pargs.ngxblocker: - if pargs.ngxblocker == 'on': - ngxblocker = True - elif pargs.ngxblocker == 'off': - ngxblocker = False + ngxblocker = bool(pargs.ngxblocker == 'on') if not data: Log.error(self, "Cannot update {0}, Invalid Options" diff --git a/wo/core/checkfqdn.py b/wo/core/checkfqdn.py index 55f93d5..2ce5533 100644 --- a/wo/core/checkfqdn.py +++ b/wo/core/checkfqdn.py @@ -33,7 +33,4 @@ def check_fqdn_ip(self): y = requests.get('http://v4.wordops.eu/dns/{0}/'.format(wo_fqdn)) ip_fqdn = (y.text).strip() - if ip == ip_fqdn: - return True - else: - return False + return bool(ip == ip_fqdn) diff --git a/wo/core/shellexec.py b/wo/core/shellexec.py index 61b12ea..3fb638c 100644 --- a/wo/core/shellexec.py +++ b/wo/core/shellexec.py @@ -27,14 +27,9 @@ class WOShellExec(): cmd_stderr_bytes.decode('utf-8', "replace")) - if proc.returncode == 0: - Log.debug(self, "Command Output: {0}, \nCommand Error: {1}" - .format(cmd_stdout, cmd_stderr)) - return True - else: - Log.debug(self, "Command Output: {0}, \nCommand Error: {1}" - .format(cmd_stdout, cmd_stderr)) - return False + Log.debug(self, "Command Output: {0}, \nCommand Error: {1}" + .format(cmd_stdout, cmd_stderr)) + return bool(proc.returncode == 0) except OSError as e: Log.debug(self, str(e)) raise CommandExecutionError From 2d478f1d7709ca627a10d28aba3b45a69fa4cf4d Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 30 Oct 2019 14:04:48 +0100 Subject: [PATCH 75/79] update travis --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4398eed..2e8c841 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,4 +32,5 @@ script: - sudo echo "Travis Banch = $TRAVIS_BRANCH" - sudo -E time bash install --travis -b "$TRAVIS_BRANCH" - sudo -E time bash tests/travis.sh - - sudo wo update --travis \ No newline at end of file + - sudo wo update --travis + - sudo python3 setup.py sdist bdist_wheel \ No newline at end of file From 575c5e948937a5331f130da53610892f75924e3e Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 30 Oct 2019 14:06:01 +0100 Subject: [PATCH 76/79] update travis --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2e8c841..08d99e6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ -sudo: required +os: linux dist: bionic -language: bash +language: shell notifications: webhooks: From eb9888d1ce05f9a6a8e88b15fb414a602dfba8b0 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 30 Oct 2019 15:18:19 +0100 Subject: [PATCH 77/79] Fix wo update --- wo/core/download.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/wo/core/download.py b/wo/core/download.py index da4fe4f..c40e8c9 100644 --- a/wo/core/download.py +++ b/wo/core/download.py @@ -47,7 +47,6 @@ class WODownload(): except requests.RequestException as e: Log.debug(self, str(e)) Log.error(self, "Unable to query GitHub API") - else: - github_json = req.json() - github_release = github_json["tag_name"] - return github_release + + github_json = req.json() + return github_json["tag_name"] From 9b3891409c7cc9f3fe88ff20b8df54d3ac1a33df Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 30 Oct 2019 15:23:12 +0100 Subject: [PATCH 78/79] Add info about travis update --- wo/cli/plugins/update.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/wo/cli/plugins/update.py b/wo/cli/plugins/update.py index 606da81..bef21d2 100644 --- a/wo/cli/plugins/update.py +++ b/wo/cli/plugins/update.py @@ -78,6 +78,9 @@ class WOUpdateController(CementBaseController): "update script"]]) if os.path.isfile('install'): + Log.info(self, "updating WordOps from local install\n" + "Latest public release = {0}".format(wo_latest)) + try: Log.info(self, "updating WordOps, please wait...") os.system("/bin/bash install --travis") From 0b6e627306b87b63eae0db5daf7ffd41fb708684 Mon Sep 17 00:00:00 2001 From: VirtuBox Date: Wed, 30 Oct 2019 16:14:14 +0100 Subject: [PATCH 79/79] update travis --- .travis.yml | 5 +++-- wo/cli/plugins/site.py | 7 ++----- wo/cli/plugins/update.py | 9 +++------ wo/core/download.py | 2 +- 4 files changed, 9 insertions(+), 14 deletions(-) diff --git a/.travis.yml b/.travis.yml index 08d99e6..ebeaa10 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,7 +23,7 @@ before_script: after_script: - - sudo bash install --purge + - sudo -E bash install --purge script: @@ -31,6 +31,7 @@ script: - sudo bash -c 'echo -e "[user]\n\tname = abc\n\temail = root@localhost.com" > /home/travis/.gitconfig' - sudo echo "Travis Banch = $TRAVIS_BRANCH" - sudo -E time bash install --travis -b "$TRAVIS_BRANCH" + - sudo -E python3 -m pip install -U -r requirements.txt - sudo -E time bash tests/travis.sh - - sudo wo update --travis + - sudo -E wo update --travis - sudo python3 setup.py sdist bdist_wheel \ No newline at end of file diff --git a/wo/cli/plugins/site.py b/wo/cli/plugins/site.py index 31373ab..cb65342 100644 --- a/wo/cli/plugins/site.py +++ b/wo/cli/plugins/site.py @@ -999,10 +999,7 @@ class WOSiteUpdateController(CementBaseController): check_ssl = check_site.is_ssl check_php_version = check_site.php_version - if check_php_version == "7.3": - old_php73 = True - else: - old_php73 = False + old_php73 = bool(check_php_version == "7.3") if (pargs.password and not (pargs.html or pargs.php or pargs.php73 or pargs.mysql or @@ -1378,7 +1375,7 @@ class WOSiteUpdateController(CementBaseController): if 'proxy' in data.keys() and data['proxy']: updateSiteInfo(self, wo_domain, stype=stype, cache=cache, - ssl=True if check_site.is_ssl else False) + ssl=(bool(check_site.is_ssl))) Log.info(self, "Successfully updated site" " http://{0}".format(wo_domain)) return 0 diff --git a/wo/cli/plugins/update.py b/wo/cli/plugins/update.py index bef21d2..f0fcad7 100644 --- a/wo/cli/plugins/update.py +++ b/wo/cli/plugins/update.py @@ -45,9 +45,6 @@ class WOUpdateController(CementBaseController): pargs = self.app.pargs filename = "woupdate" + time.strftime("%Y%m%d-%H%M%S") - wo_current = WOVar.wo_version - wo_latest = WODownload.latest_release(self, "WordOps/WordOps") - install_args = "" if pargs.mainline or pargs.beta: wo_branch = "mainline" @@ -64,6 +61,8 @@ class WOUpdateController(CementBaseController): if ((not pargs.force) and (not pargs.travis) and (not pargs.mainline) and (not pargs.beta) and (not pargs.branch)): + wo_current = WOVar.wo_version + wo_latest = WODownload.latest_release(self, "WordOps/WordOps") if wo_current == wo_latest: Log.error( self, "WordOps {0} is already installed" @@ -78,9 +77,7 @@ class WOUpdateController(CementBaseController): "update script"]]) if os.path.isfile('install'): - Log.info(self, "updating WordOps from local install\n" - "Latest public release = {0}".format(wo_latest)) - + Log.info(self, "updating WordOps from local install\n") try: Log.info(self, "updating WordOps, please wait...") os.system("/bin/bash install --travis") diff --git a/wo/core/download.py b/wo/core/download.py index c40e8c9..a67c486 100644 --- a/wo/core/download.py +++ b/wo/core/download.py @@ -44,9 +44,9 @@ class WODownload(): 'https://api.github.com/repos/{0}/releases/latest' .format(repository), timeout=(5, 30)) + github_json = req.json() except requests.RequestException as e: Log.debug(self, str(e)) Log.error(self, "Unable to query GitHub API") - github_json = req.json() return github_json["tag_name"]