diff --git a/.travis.yml b/.travis.yml index 5445688..d7c342b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,9 +26,9 @@ before_script: after_script: - sudo cat /etc/nginx/nginx.conf | ccze -A - - sudo cat /var/log/wo/wordops.log | ccze -A - sudo cat /etc/mysql/my.cnf | ccze -A - sudo bash install --purge + - sudo curl --progress-bar --upload-file /var/log/wo/wordops.log https://transfer.vtbox.net/"$(basename wordops.log)" && echo "" | sudo tee -a $HOME/.transfer.log && echo "" script: diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c64539..855376d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,21 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### v3.9.x - [Unreleased] +#### Added + +- Rate limiter on wp-cron.php and xmlrpc.php +- mime.types template to handle missing extension ttf +- try_files directive for favicon +- additional settings for fail2ban +- asynchronous installer to decrease install/update duration + +#### Fixed + +- Several typo or syntax errors +- `wo site` errors due to broken symlinks for access.log or error.log +- `wo clean` error due to unused memcached flag +- MySQL database and user variables overwrited in `wo site` + ### v3.9.8.8 - 2019-09-02 #### Added diff --git a/install b/install index d10db07..2fb4750 100755 --- a/install +++ b/install @@ -9,7 +9,7 @@ # ------------------------------------------------------------------------- # wget -qO wo wops.cc && sudo bash wo # ------------------------------------------------------------------------- -# Version 3.9.8.5 - 2019-08-28 +# Version 3.9.8.9 - 2019-09-03 # ------------------------------------------------------------------------- # CONTENTS @@ -107,16 +107,16 @@ unset LANG export LANG='en_US.UTF-8' export LC_ALL='C.UTF-8' -[ -z "$wo_travis" ] && { - apt-get update -qq -} - command_exists() { command -v "$@" > /dev/null 2>&1 } -if ! command_exists curl; then - apt-get -y install curl -qq +if [ -z "$wo_travis" ]; then + if command_exists curl; then + apt-get update -qq & + else + apt-get update -qq && apt-get -y install curl -qq > /dev/null 2>&1 + fi fi if [ -f ./setup.py ]; then @@ -133,7 +133,7 @@ echo "" # 1- Check whether lsb_release is installed, and if not, install it ### if ! command_exists lsb_release; then - wo_lib_echo "Installing lsb-release, please wait..." + wo_lib_echo "Installing lsb-release, please /bin/bash --init-file <(echo 'source /etc/bash_completion.d/wo_auto.rc')..." apt-get install lsb-release -qq fi @@ -179,7 +179,7 @@ if [ -z "$wo_force_install" ]; then else check_wo_linux_distro=$(lsb_release -sc | grep -E "xenial|bionic|disco|jessie|stretch|buster") if [ -z "$check_wo_linux_distro" ]; then - wo_lib_echo_fail "WordOps (wo) only supports Ubuntu 16.04/18.04/19.04 LTS, Debian 9.x/10.x and Raspbian 9.x" + wo_lib_echo_fail "WordOps (wo) only supports Ubuntu 16.04/18.04/19.04 LTS, Debian 9.x/10.x and Raspbian 9.x/10x" exit 100 fi fi @@ -190,7 +190,7 @@ fi ### if [ ! -d "$wo_log_dir" ] || [ ! -d "$wo_backup_dir" ] || [ ! -d "$wo_tmp_dir" ]; then - wo_lib_echo "Creating WordOps backup, tmp & log directory, just a second..." + wo_lib_echo "Creating WordOps directory" mkdir -p "$wo_backup_dir" "$wo_log_dir" "$wo_tmp_dir" || wo_lib_error "Whoops - seems we are unable to create the log directory $wo_log_dir, exit status " $? # create wordops log files @@ -217,7 +217,7 @@ wo_install_dep() { apt-get -option=Dpkg::options::=--force-confmiss --option=Dpkg::options::=--force-confold --assume-yes install \ build-essential curl gzip python3 python3-apt python3-setuptools python3-requests python3-dev sqlite3 git tar software-properties-common pigz \ gnupg2 cron ccze rsync tree haveged ufw unattended-upgrades tzdata ntp > /dev/null 2>&1 - add-apt-repository ppa:wordops/nginx-wo -yu + add-apt-repository ppa:wordops/nginx-wo -yn else # install dependencies apt-get -option=Dpkg::options::=--force-confmiss --option=Dpkg::options::=--force-confold --assume-yes install \ @@ -806,13 +806,14 @@ wo_cheat_install() { ### # 4 - WO MAIN SETUP ### - +wait if [ "$wo_purge" = "y" ]; then wo_lib_echo "Backing-up WO install" | tee -ai $wo_install_log wo_backup_wo | tee -ai $wo_install_log wo_lib_echo "Uninstalling WordOps" | tee -ai $wo_install_log wo_uninstall | tee -ai $wo_install_log wo_lib_echo "The WordOps backup files can be found in $WO_BACKUP_FILE" + exit 0 else # 1 - WO already installed if [ -x /usr/local/bin/wo ]; then @@ -822,43 +823,45 @@ else fi fi wo_lib_echo "Installing wo dependencies " | tee -ai $wo_install_log - wo_install_dep | tee -ai $wo_install_log - wo_timesync | tee -ai $wo_install_log + wo_install_dep & + wo_timesync & wo_lib_echo "Backing-up WO install" | tee -ai $wo_install_log - wo_backup_wo | tee -ai $wo_install_log - secure_wo_db | tee -ai $wo_install_log + wo_backup_wo & + secure_wo_db & wo_lib_echo "Installing WordOps " | tee -ai $wo_install_log - wo_clean | tee -ai $wo_install_log + wo_clean & + wait if [ "$wo_travis" = "y" ]; then - wo_install_travis | tee -ai $wo_install_log + wo_install_travis & else if [ -f "$HOME/.gitconfig" ]; then - wo_install >> $wo_install_log 2>&1 + wo_install >> $wo_install_log 2>&1 & else - wo_install | tee -ai $wo_install_log + wo_install fi fi - wo_update_latest | tee -ai $wo_install_log + wo_update_latest & if [ ! -d /opt/acme/.sh ]; then wo_lib_echo "Updating acme.sh" | tee -ai $wo_install_log - wo_install_acme_sh | tee -ai $wo_install_log + wo_install_acme_sh & fi wo_lib_echo "Applying Kernel tweaks" | tee -ai $wo_install_log - wo_tweak_kernel | tee -ai $wo_install_log + wo_tweak_kernel & if [ ! -f /opt/wo-kernel.sh ]; then wo_lib_echo "Adding systemd service tweak" | tee -ai $wo_install_log - wo_systemd_tweak | tee -ai $wo_install_log + wo_systemd_tweak & fi if [ -x /usr/sbin/nginx ]; then - wo_nginx_tweak | tee -ai $wo_install_log + wo_nginx_tweak & fi if [ -d /etc/systemd/system/mariadb.service.d ]; then - wo_mariadb_tweak | tee -ai $wo_install_log + wo_mariadb_tweak & fi - wo_cheat_install | tee -ai $wo_install_log - wo_domain_suffix | tee -ai $wo_install_log + wo_cheat_install & + wo_domain_suffix & wo_lib_echo "Running post-install steps " | tee -ai $wo_install_log - wo_update_wp_cli | tee -ai $wo_install_log + wo_update_wp_cli & + wait else # 2 - Migration from EEv3 if [ -x /usr/local/bin/ee ]; then @@ -869,83 +872,92 @@ else fi fi wo_lib_echo "Installing wo dependencies " | tee -ai $wo_install_log - wo_install_dep | tee -ai $wo_install_log - wo_timesync | tee -ai $wo_install_log + wo_install_dep >> $wo_install_log 2>&1 & + wo_timesync >> $wo_install_log 2>&1 & wo_lib_echo "Backing-up EE install" | tee -ai $wo_install_log - wo_backup_ee | tee -ai $wo_install_log + wo_backup_ee >> $wo_install_log 2>&1 & wo_lib_echo "Removing EasyEngine cronjob" | tee -ai $wo_install_log - wo_remove_ee_cron | tee -ai $wo_install_log + wo_remove_ee_cron >> $wo_install_log 2>&1 & wo_lib_echo "Syncing WO database" | tee -ai $wo_install_log - wo_sync_db | tee -ai $wo_install_log - secure_wo_db | tee -ai $wo_install_log + wo_sync_db >> $wo_install_log 2>&1 & + secure_wo_db >> $wo_install_log 2>&1 & + wait wo_lib_echo "Installing WordOps " | tee -ai $wo_install_log if [ -f "$HOME/.gitconfig" ]; then - wo_install >> $wo_install_log 2>&1 + wo_install >> $wo_install_log 2>&1 & else wo_install | tee -ai $wo_install_log fi if command_exists nginx; then wo_lib_echo "Upgrading Nginx" | tee -ai $wo_install_log - wo_upgrade_nginx | tee -ai $wo_install_log + wo_upgrade_nginx >> $wo_install_log 2>&1 & fi - wo_update_latest | tee -ai $wo_install_log + wait + wo_update_latest >> $wo_install_log 2>&1 & wo_lib_echo "Installing acme.sh" | tee -ai $wo_install_log - wo_install_acme_sh | tee -ai $wo_install_log + wo_install_acme_sh >> $wo_install_log 2>&1 & wo_lib_echo "Applying Kernel tweaks" | tee -ai $wo_install_log - wo_tweak_kernel | tee -ai $wo_install_log + wo_tweak_kernel >> $wo_install_log 2>&1 & if [ ! -f /opt/wo-kernel.sh ]; then wo_lib_echo "Adding systemd service tweak" | tee -ai $wo_install_log - wo_systemd_tweak | tee -ai $wo_install_log + wo_systemd_tweak & fi if command_exists nginx; then - wo_nginx_tweak | tee -ai $wo_install_log + wo_nginx_tweak & fi if [ -d /etc/systemd/system/mariadb.service.d ]; then - wo_mariadb_tweak | tee -ai $wo_install_log + wo_mariadb_tweak & fi - wo_domain_suffix | tee -ai $wo_install_log + wo_domain_suffix >> $wo_install_log & wo_lib_echo "Running post-install steps " | tee -ai $wo_install_log - wo_git_init | tee -ai $wo_install_log - wo_update_wp_cli | tee -ai $wo_install_log - wo_cheat_install | tee -ai $wo_install_log + { + wo_git_init & + wo_update_wp_cli & + wo_cheat_install & + } >> $wo_install_log + wait wo_lib_echo "Cleaning-up EE previous install" | tee -ai $wo_install_log - wo_clean_ee | tee -ai $wo_install_log + wo_clean_ee >> $wo_install_log else # 3 - Fresh WO setup wo_lib_echo "Installing wo dependencies " | tee -ai $wo_install_log [ -z "$wo_travis" ] && { - wo_dist_upgrade | tee -ai $wo_install_log + wo_dist_upgrade >> $wo_install_log } - wo_install_dep | tee -ai $wo_install_log - wo_timesync | tee -ai $wo_install_log + wo_install_dep >> $wo_install_log & + wo_timesync >> $wo_install_log & + wait wo_lib_echo "Installing WordOps " | tee -ai $wo_install_log if [ "$wo_travis" = "y" ]; then - wo_install_travis | tee -ai $wo_install_log + wo_install_travis | tee -ai $wo_install_log & else if [ -f "$HOME/.gitconfig" ]; then - wo_install >> $wo_install_log 2>&1 + wo_install >> $wo_install_log 2>&1 & else wo_install | tee -ai $wo_install_log fi fi if [ "$ufw" = "y" ]; then wo_lib_echo "Configuring UFW" | tee -ai $wo_install_log - wo_ufw_setup | tee -ai $wo_install_log + wo_ufw_setup & fi wo_lib_echo "Applying Kernel tweaks" | tee -ai $wo_install_log - wo_tweak_kernel | tee -ai $wo_install_log + wo_tweak_kernel >> $wo_install_log 2>&1 & if [ ! -f /opt/wo-kernel.sh ]; then wo_lib_echo "Adding systemd service tweak" | tee -ai $wo_install_log - wo_systemd_tweak | tee -ai $wo_install_log + wo_systemd_tweak >> $wo_install_log 2>&1 & fi wo_lib_echo "Installing acme.sh" | tee -ai $wo_install_log - wo_install_acme_sh | tee -ai $wo_install_log wo_lib_echo "Running post-install steps " | tee -ai $wo_install_log - secure_wo_db | tee -ai $wo_install_log - wo_cheat_install | tee -ai $wo_install_log - wo_domain_suffix | tee -ai $wo_install_log - wo_git_init | tee -ai $wo_install_log - wo_update_wp_cli | tee -ai $wo_install_log + { + wo_install_acme_sh & + secure_wo_db & + wo_cheat_install & + wo_domain_suffix & + wo_git_init & + wo_update_wp_cli & + } >> $wo_install_log + wait fi fi @@ -967,11 +979,14 @@ else else wo_lib_echo "WordOps (wo) installed successfully" echo - wo_lib_echo "For autocompletion, run the following command:" - wo_lib_echo_info "source /etc/bash_completion.d/wo_auto.rc" + wo_lib_echo "To enable bash-completion, just use the command:" + wo_lib_echo_info "bash" echo - wo_lib_echo "To install WordOps main stacks, use the command:" + wo_lib_echo "To install WordOps recommended stacks, you can use the command:" wo_lib_echo_info "wo stack install" + echo + wo_lib_echo "To create a first WordPress site, you can use the command:" + wo_lib_echo_info "wo site create site.tld --wp" fi echo wo_lib_echo "WordOps Documentation : https://docs.wordops.net" @@ -979,5 +994,7 @@ else echo wo_lib_echo "Give WordOps a GitHub star : https://github.com/WordOps/WordOps/" echo - fi +# if [ -z "$wo_travis" ]; then +# /bin/bash --init-file "/var/lib/wo/tmp/WordOps-$wo_branch/tests/init-file" +# fi diff --git a/tests/init-file b/tests/init-file new file mode 100644 index 0000000..d8a3a07 --- /dev/null +++ b/tests/init-file @@ -0,0 +1,3 @@ +#!/bin/bash --init-file +source /etc/bash_completion.d/wo_auto.rc +source ~/.bashrc diff --git a/tests/travis.sh b/tests/travis.sh index d4b74b3..8c8a065 100644 --- a/tests/travis.sh +++ b/tests/travis.sh @@ -10,8 +10,7 @@ CGREEN="${CSI}1;32m" CEND="${CSI}0m" exit_script() { - tar -I pigz -cf wordops.tar.gz /var/log/wo - curl --progress-bar --upload-file wordops.tar.gz https://transfer.vtbox.net/"$(basename wordops.tar.gz)" && echo "" | sudo tee -a $HOME/.transfer.log && echo "" + curl --progress-bar --upload-file /var/log/wo/wordops.log https://transfer.vtbox.net/"$(basename wordops.log)" && echo "" | sudo tee -a $HOME/.transfer.log && echo "" exit 1 } diff --git a/wo/cli/plugins/clean.py b/wo/cli/plugins/clean.py index 2638dc3..a8e0052 100644 --- a/wo/cli/plugins/clean.py +++ b/wo/cli/plugins/clean.py @@ -36,8 +36,8 @@ class WOCleanController(CementBaseController): @expose(hide=True) def default(self): pargs = self.app.pargs - if (not (pargs.all or pargs.fastcgi or - pargs.memcached or pargs.opcache or + if (not (pargs.all or pargs.fastcgi + or pargs.opcache or pargs.redis)): self.clean_fastcgi() if pargs.all: diff --git a/wo/cli/plugins/debug.py b/wo/cli/plugins/debug.py index 649cfea..391b3fc 100644 --- a/wo/cli/plugins/debug.py +++ b/wo/cli/plugins/debug.py @@ -15,7 +15,6 @@ import os import configparser import glob import signal -import subprocess def wo_debug_hook(app): diff --git a/wo/cli/plugins/log.py b/wo/cli/plugins/log.py index 7baf633..d080368 100644 --- a/wo/cli/plugins/log.py +++ b/wo/cli/plugins/log.py @@ -1,17 +1,18 @@ """Logfile Plugin for WordOps""" -from cement.core.controller import CementBaseController, expose -from cement.core import handler, hook -from wo.core.logging import Log -from wo.cli.plugins.site_functions import logwatch -from wo.core.variables import WOVariables -from wo.core.fileutils import WOFileUtils -from wo.core.shellexec import WOShellExec -from wo.core.sendmail import WOSendMail -from wo.core.mysql import WOMysql -import os import glob import gzip +import os + +from cement.core import handler, hook +from cement.core.controller import CementBaseController, expose +from wo.cli.plugins.site_functions import logwatch +from wo.core.fileutils import WOFileUtils +from wo.core.logging import Log +from wo.core.mysql import WOMysql +from wo.core.sendmail import WOSendMail +from wo.core.shellexec import WOShellExec +from wo.core.variables import WOVariables def wo_log_hook(app): diff --git a/wo/cli/plugins/maintenance.py b/wo/cli/plugins/maintenance.py index 06a9071..ad65841 100644 --- a/wo/cli/plugins/maintenance.py +++ b/wo/cli/plugins/maintenance.py @@ -1,9 +1,9 @@ """Maintenance Plugin for WordOps""" -from cement.core.controller import CementBaseController, expose from cement.core import handler, hook -from wo.core.logging import Log +from cement.core.controller import CementBaseController, expose from wo.core.aptget import WOAptGet +from wo.core.logging import Log def wo_maintenance_hook(app): diff --git a/wo/cli/plugins/site.py b/wo/cli/plugins/site.py index 2a1ec33..1e18b19 100644 --- a/wo/cli/plugins/site.py +++ b/wo/cli/plugins/site.py @@ -1,23 +1,24 @@ # """WordOps site controller.""" -from cement.core.controller import CementBaseController, expose +import glob +import json +import os +import subprocess +from subprocess import Popen + from cement.core import handler, hook -from wo.core.sslutils import SSL -from wo.core.variables import WOVariables -from wo.core.shellexec import WOShellExec -from wo.core.domainvalidate import ValidateDomain, GetDomainlevel -from wo.core.fileutils import WOFileUtils +from cement.core.controller import CementBaseController, expose from wo.cli.plugins.site_functions import * -from wo.core.services import WOService -from wo.cli.plugins.sitedb import (addNewSite, getSiteInfo, - updateSiteInfo, deleteSiteInfo, getAllsites) +from wo.cli.plugins.sitedb import (addNewSite, deleteSiteInfo, getAllsites, + getSiteInfo, updateSiteInfo) +from wo.core.domainvalidate import GetDomainlevel, ValidateDomain +from wo.core.fileutils import WOFileUtils from wo.core.git import WOGit from wo.core.logging import Log -from subprocess import Popen from wo.core.nginxhashbucket import hashbucket -import os -import glob -import subprocess -import json +from wo.core.services import WOService +from wo.core.shellexec import WOShellExec +from wo.core.sslutils import SSL +from wo.core.variables import WOVariables def wo_site_hook(app): @@ -31,7 +32,6 @@ class WOSiteController(CementBaseController): label = 'site' stacked_on = 'base' stacked_type = 'nested' - exit_on_close = True description = ('Performs website specific operations') arguments = [ (['site_name'], @@ -79,8 +79,7 @@ class WOSiteController(CementBaseController): Log.error(self, "service nginx reload failed. " "check issues with `nginx -t` command") else: - Log.error(self, "nginx configuration file does not exist" - .format(wo_domain)) + Log.error(self, "nginx configuration file does not exist") @expose(help="Disable site example.com") def disable(self): @@ -121,8 +120,7 @@ class WOSiteController(CementBaseController): Log.error(self, "service nginx reload failed. " "check issues with `nginx -t` command") else: - Log.error(self, "nginx configuration file does not exist" - .format(wo_domain)) + Log.error(self, "nginx configuration file does not exist") @expose(help="Get example.com information") def info(self): @@ -182,8 +180,7 @@ class WOSiteController(CementBaseController): "disabled")) self.app.render((data), 'siteinfo.mustache') else: - Log.error(self, "nginx configuration file does not exist" - .format(wo_domain)) + Log.error(self, "nginx configuration file does not exist") @expose(help="Monitor example.com logs") def log(self): @@ -430,7 +427,7 @@ class WOSiteCreateController(CementBaseController): pargs.site_name = pargs.site_name.strip() (wo_domain, wo_www_domain) = ValidateDomain(pargs.site_name) if not wo_domain.strip(): - Log.error("Invalid domain name, " + Log.error(self, "Invalid domain name, " "Provide valid domain name") wo_site_webroot = WOVariables.wo_webroot + wo_domain diff --git a/wo/cli/plugins/site_functions.py b/wo/cli/plugins/site_functions.py index ccf7575..99d11d3 100644 --- a/wo/cli/plugins/site_functions.py +++ b/wo/cli/plugins/site_functions.py @@ -81,10 +81,10 @@ def setupdomain(self, data): out=wo_site_nginx_conf) wo_site_nginx_conf.close() except IOError as e: - Log.debug(self, "{0}".format(e)) + Log.debug(self, str(e)) raise SiteError("create nginx configuration failed for site") except Exception as e: - Log.debug(self, "{0}".format(e)) + Log.debug(self, str(e)) raise SiteError("create nginx configuration failed for site") finally: # Check nginx -t and return status over it @@ -126,7 +126,7 @@ def setupdomain(self, data): '{0}/logs/error.log' .format(wo_site_webroot)]) except Exception as e: - Log.debug(self, "{0}".format(e)) + Log.debug(self, str(e)) raise SiteError("setup webroot failed for site") finally: # TODO Check if directories are setup @@ -160,6 +160,7 @@ def setupdatabase(self, data): if not wo_db_name: wo_db_name = wo_replace_dot + wo_db_name = (wo_db_name[0:8] + generate_random()) if prompt_dbuser == 'True' or prompt_dbuser == 'true': try: @@ -173,12 +174,10 @@ def setupdatabase(self, data): if not wo_db_username: wo_db_username = wo_replace_dot + wo_db_username = (wo_db_name[0:8] + generate_random()) if not wo_db_password: wo_db_password = wo_random_pass - wo_db_username = (wo_db_name[0:8] + generate_random()) - wo_db_name = (wo_db_name[0:8] + generate_random()) - # create MySQL database Log.info(self, "Setting up database\t\t", end='') Log.debug(self, "Creating database {0}".format(wo_db_name)) @@ -241,9 +240,6 @@ def setupwordpress(self, data, vhostonly=False): wo_random_pass = (''.join(random.sample(string.ascii_uppercase + string.ascii_lowercase + string.digits, 24))) - wo_random = (''.join(random.sample(string.ascii_uppercase + - string.ascii_lowercase + - string.digits, 8))) wo_wp_prefix = '' # wo_wp_user = '' # wo_wp_pass = '' @@ -267,7 +263,7 @@ def setupwordpress(self, data, vhostonly=False): raise SiteError("download WordPress core failed") except CommandExecutionError: Log.info(self, "[" + Log.ENDC + Log.FAIL + "Fail" + Log.OKBLUE + "]") - raise SiteError(self, "download WordPress core failed") + raise SiteError("download WordPress core failed") Log.info(self, "[" + Log.ENDC + "Done" + Log.OKBLUE + "]") @@ -730,6 +726,7 @@ def setupwp_plugin(self, plugin_name, plugin_option, plugin_data, data): def setwebrootpermissions(self, webroot): Log.debug(self, "Setting up permissions") try: + WOFileUtils.findBrokenSymlink(self, '/var/www/') WOFileUtils.chown(self, webroot, WOVariables.wo_php_user, WOVariables.wo_php_user, recursive=True) except Exception as e: @@ -917,7 +914,7 @@ def updatewpuserpassword(self, wo_domain, wo_site_webroot): try: wo_wp_user = input("Provide WordPress user name [admin]: ") except Exception as e: - Log.debug(self, "{0}".format(e)) + Log.debug(self, str(e)) Log.error(self, "\nCould not update password") if wo_wp_user == "?": @@ -951,7 +948,7 @@ def updatewpuserpassword(self, wo_domain, wo_site_webroot): "{0} user: " .format(wo_wp_user)) except Exception as e: - Log.debug(self, "{0}".format(e)) + Log.debug(self, str(e)) raise SiteError("failed to read password input ") try: @@ -1345,15 +1342,15 @@ def doCleanupAction(self, domain='', webroot='', dbname='', dbuser='', if dbname: if not dbuser: raise SiteError("dbuser not provided") - if not dbhost: - raise SiteError("dbhost not provided") + if not dbhost: + raise SiteError("dbhost not provided") deleteDB(self, dbname, dbuser, dbhost) # setup letsencrypt for domain + www.domain def setupLetsEncrypt(self, wo_domain_name, subdomain=False, wildcard=False, - wo_dns=False, wo_acme_dns='dns_cf'): + wo_dns=False, wo_acme_dns='dns_cf', backend=False): if os.path.isfile("/etc/letsencrypt/" "renewal/{0}_ecc/" @@ -1446,8 +1443,7 @@ def setupLetsEncrypt(self, wo_domain_name, subdomain=False, wildcard=False, '/etc/letsencrypt'): Log.info(self, "Securing WordOps backend with {0} certificate" .format(wo_domain_name)) - sslconf = open("/var/www/22222/conf/nginx/ssl.conf" - .format(wo_domain_name), + sslconf = open("/var/www/22222/conf/nginx/ssl.conf", encoding='utf-8', mode='w') sslconf.write("ssl_certificate {0}/{1}/fullchain.pem;\n" "ssl_certificate_key {0}/{1}/key.pem;\n" diff --git a/wo/cli/plugins/stack.py b/wo/cli/plugins/stack.py index 833750e..02585fb 100644 --- a/wo/cli/plugins/stack.py +++ b/wo/cli/plugins/stack.py @@ -1,27 +1,25 @@ """Stack Plugin for WordOps""" -from cement.core.controller import CementBaseController, expose -from cement.core import handler, hook - import codecs import configparser import os import pwd import random +import re import shutil import string -import re + import requests import psutil - -# from pynginxconfig import NginxConfig +from cement.core import handler, hook +from cement.core.controller import CementBaseController, expose from wo.cli.plugins.site_functions import * from wo.cli.plugins.sitedb import * from wo.cli.plugins.stack_migrate import WOStackMigrateController +from wo.cli.plugins.stack_pref import post_pref, pre_pref from wo.cli.plugins.stack_services import WOStackStatusController from wo.cli.plugins.stack_upgrade import WOStackUpgradeController -from wo.cli.plugins.stack_pref import pre_pref, post_pref from wo.core.apt_repo import WORepo from wo.core.aptget import WOAptGet from wo.core.cron import WOCron @@ -33,8 +31,8 @@ from wo.core.logging import Log from wo.core.mysql import WOMysql from wo.core.services import WOService from wo.core.shellexec import CommandExecutionError, WOShellExec -from wo.core.variables import WOVariables from wo.core.template import WOTemplate +from wo.core.variables import WOVariables def wo_stack_hook(app): @@ -202,8 +200,8 @@ class WOStackController(CementBaseController): Log.debug(self, "Setting apt_packages variable for PHP 7.2") if not (WOAptGet.is_installed(self, 'php7.2-fpm')): if not (WOAptGet.is_installed(self, 'php7.3-fpm')): - apt_packages = apt_packages + WOVariables.wo_php + \ - WOVariables.wo_php_extra + apt_packages = (apt_packages + WOVariables.wo_php + + WOVariables.wo_php_extra) else: apt_packages = apt_packages + WOVariables.wo_php else: @@ -215,8 +213,9 @@ class WOStackController(CementBaseController): Log.debug(self, "Setting apt_packages variable for PHP 7.3") if not WOAptGet.is_installed(self, 'php7.3-fpm'): if not (WOAptGet.is_installed(self, 'php7.2-fpm')): - apt_packages = apt_packages + WOVariables.wo_php + \ - WOVariables.wo_php73 + WOVariables.wo_php_extra + apt_packages = (apt_packages + WOVariables.wo_php + + WOVariables.wo_php73 + + WOVariables.wo_php_extra) else: apt_packages = apt_packages + WOVariables.wo_php73 else: @@ -706,7 +705,8 @@ class WOStackController(CementBaseController): else: WOShellExec.cmd_exec(self, "bash /opt/netdata/usr/" "libexec/netdata/" - "netdata-uninstaller.sh -y -f") + "netdata-uninstaller.sh - y - f", + errormsg='', log=False) if (packages): Log.info(self, "Removing packages, please wait...") @@ -926,7 +926,8 @@ class WOStackController(CementBaseController): if WOVariables.wo_distro == 'Raspbian': WOShellExec.cmd_exec(self, "bash /usr/" "libexec/netdata/netdata-" - "uninstaller.sh -y -f") + "uninstaller.sh -y -f", + errormsg='', log=False) else: WOShellExec.cmd_exec(self, "bash /opt/netdata/usr/" "libexec/netdata/netdata-" diff --git a/wo/cli/plugins/stack_migrate.py b/wo/cli/plugins/stack_migrate.py index 4b61074..63a940b 100644 --- a/wo/cli/plugins/stack_migrate.py +++ b/wo/cli/plugins/stack_migrate.py @@ -1,14 +1,15 @@ -from cement.core.controller import CementBaseController, expose -from cement.core import handler, hook -from wo.core.mysql import WOMysql -from wo.core.logging import Log -from wo.core.variables import WOVariables -from wo.core.aptget import WOAptGet -from wo.core.shellexec import WOShellExec -from wo.core.apt_repo import WORepo import configparser import os +from cement.core import handler, hook +from cement.core.controller import CementBaseController, expose +from wo.core.apt_repo import WORepo +from wo.core.aptget import WOAptGet +from wo.core.logging import Log +from wo.core.mysql import WOMysql +from wo.core.shellexec import WOShellExec +from wo.core.variables import WOVariables + class WOStackMigrateController(CementBaseController): class Meta: diff --git a/wo/cli/plugins/stack_pref.py b/wo/cli/plugins/stack_pref.py index e3fab80..b9ce87e 100644 --- a/wo/cli/plugins/stack_pref.py +++ b/wo/cli/plugins/stack_pref.py @@ -4,6 +4,7 @@ import os import random import shutil import string + import psutil import requests @@ -11,15 +12,17 @@ from wo.cli.plugins.site_functions import * from wo.cli.plugins.stack_services import WOStackStatusController from wo.core.apt_repo import WORepo from wo.core.aptget import WOAptGet +from wo.core.checkfqdn import check_fqdn_ip from wo.core.cron import WOCron +from wo.core.domainvalidate import GetDomainlevel from wo.core.extract import WOExtract from wo.core.fileutils import WOFileUtils from wo.core.git import WOGit -from wo.core.template import WOTemplate from wo.core.logging import Log from wo.core.mysql import WOMysql from wo.core.services import WOService from wo.core.shellexec import CommandExecutionError, WOShellExec +from wo.core.template import WOTemplate from wo.core.variables import WOVariables @@ -70,7 +73,7 @@ def pre_pref(self, apt_packages): log=False) except CommandExecutionError as e: Log.debug(self, "{0}".format(e)) - Log.error("Failed to initialize MySQL package") + Log.error(self, "Failed to initialize MySQL package") Log.debug(self, "echo \"mariadb-server-{0} " "mysql-server/root_password_again " @@ -86,7 +89,7 @@ def pre_pref(self, apt_packages): log=False) except CommandExecutionError as e: Log.debug(self, "{0}".format(e)) - Log.error("Failed to initialize MySQL package") + Log.error(self, "Failed to initialize MySQL package") # generate my.cnf root credentials mysql_config = """ [client] @@ -359,16 +362,16 @@ def post_pref(self, apt_packages, packages, upgrade=False): '/etc/nginx/sites-available') os.makedirs('/etc/nginx/sites-enabled') - # 22222 port settings - if not os.path.isfile('/etc/nginx/sites-available/22222'): - data = dict(webroot=ngxroot) - WOTemplate.render( - self, - '/etc/nginx/sites-available/22222', - '22222.mustache', data, overwrite=False) - passwd = ''.join([random.choice - (string.ascii_letters + string.digits) - for n in range(24)]) + # 22222 port settings + data = dict(webroot=ngxroot) + WOTemplate.render( + self, + '/etc/nginx/sites-available/22222', + '22222.mustache', data, overwrite=False) + passwd = ''.join([random.choice + (string.ascii_letters + string.digits) + for n in range(24)]) + if not os.path.isfile('/etc/nginx/htpasswd-wo'): try: WOShellExec.cmd_exec( self, "printf \"WordOps:" @@ -380,8 +383,8 @@ def post_pref(self, apt_packages, packages, upgrade=False): except CommandExecutionError as e: Log.debug(self, "{0}".format(e)) Log.error(self, "Failed to save HTTP Auth") - - # Create Symbolic link for 22222 + if not os.path.islink('/etc/nginx/sites-enabled/22222'): + # Create Symbolic link for 22222 WOFileUtils.create_symlink( self, ['/etc/nginx/' 'sites-available/' @@ -1280,7 +1283,7 @@ def post_pref(self, apt_packages, packages, upgrade=False): self, "/usr/local/bin/composer update " "--no-plugins --no-scripts " "-n --no-dev -d " - "/var/www/22222/htdocs/db/pma/") + "/var/www/22222/htdocs/db/pma/ &") WOFileUtils.chown( self, '{0}22222/htdocs/db/pma' .format(WOVariables.wo_webroot), @@ -1302,7 +1305,7 @@ def post_pref(self, apt_packages, packages, upgrade=False): "--no-scripts -n -s dev " "erik-dubbelboer/php-redis-admin " "/var/www/22222/htdocs/cache" - "/redis/phpRedisAdmin ") + "/redis/phpRedisAdmin &") WOFileUtils.chown(self, '{0}22222/htdocs' .format(WOVariables.wo_webroot), 'www-data', @@ -1321,7 +1324,8 @@ def post_pref(self, apt_packages, packages, upgrade=False): Log.info(self, "Installing Netdata, please wait...") WOShellExec.cmd_exec(self, "bash /var/lib/wo/tmp/" "kickstart.sh " - "--dont-wait") + "--dont-wait", + errormsg='', log=False) if os.path.isdir('/etc/netdata'): wo_netdata = "/" elif os.path.isdir('/opt/netdata'): diff --git a/wo/cli/plugins/stack_services.py b/wo/cli/plugins/stack_services.py index 4c5aa2a..695ee1d 100644 --- a/wo/cli/plugins/stack_services.py +++ b/wo/cli/plugins/stack_services.py @@ -1,11 +1,12 @@ -from cement.core.controller import CementBaseController, expose -from cement.core import handler, hook -from wo.core.services import WOService -from wo.core.logging import Log -from wo.core.variables import WOVariables -from wo.core.aptget import WOAptGet import os +from cement.core import handler, hook +from cement.core.controller import CementBaseController, expose +from wo.core.aptget import WOAptGet +from wo.core.logging import Log +from wo.core.services import WOService +from wo.core.variables import WOVariables + class WOStackStatusController(CementBaseController): class Meta: diff --git a/wo/cli/plugins/stack_upgrade.py b/wo/cli/plugins/stack_upgrade.py index 606998f..abfc912 100644 --- a/wo/cli/plugins/stack_upgrade.py +++ b/wo/cli/plugins/stack_upgrade.py @@ -1,8 +1,9 @@ import os import shutil -from cement.core.controller import CementBaseController, expose from cement.core import handler, hook +from cement.core.controller import CementBaseController, expose +from wo.cli.plugins.stack_pref import post_pref, pre_pref from wo.core.aptget import WOAptGet from wo.core.download import WODownload from wo.core.extract import WOExtract @@ -11,7 +12,6 @@ from wo.core.logging import Log from wo.core.services import WOService from wo.core.shellexec import WOShellExec from wo.core.variables import WOVariables -from wo.cli.plugins.stack_pref import pre_pref, post_pref class WOStackUpgradeController(CementBaseController): diff --git a/wo/cli/plugins/sync.py b/wo/cli/plugins/sync.py index 1f9da5b..5039a03 100644 --- a/wo/cli/plugins/sync.py +++ b/wo/cli/plugins/sync.py @@ -1,11 +1,12 @@ -from cement.core.controller import CementBaseController, expose -from cement.core import handler, hook -from wo.core.fileutils import WOFileUtils -from wo.cli.plugins.sitedb import updateSiteInfo, getAllsites -from wo.core.mysql import WOMysql, StatementExcecutionError -from wo.core.logging import Log import glob +from cement.core import handler, hook +from cement.core.controller import CementBaseController, expose +from wo.cli.plugins.sitedb import getAllsites, updateSiteInfo +from wo.core.fileutils import WOFileUtils +from wo.core.logging import Log +from wo.core.mysql import StatementExcecutionError, WOMysql + def wo_sync_hook(app): pass diff --git a/wo/cli/templates/acl.mustache b/wo/cli/templates/acl.mustache index c8ff43b..b75c91b 100644 --- a/wo/cli/templates/acl.mustache +++ b/wo/cli/templates/acl.mustache @@ -5,4 +5,5 @@ auth_basic "Restricted Area"; auth_basic_user_file htpasswd-wo; # Allowed IP Address List allow 127.0.0.1; +allow ::1; deny all; diff --git a/wo/cli/templates/fail2ban.mustache b/wo/cli/templates/fail2ban.mustache index 554cc6e..2cf4d83 100644 --- a/wo/cli/templates/fail2ban.mustache +++ b/wo/cli/templates/fail2ban.mustache @@ -1,3 +1,6 @@ +[DEFAULT] +ignoreip = 127.0.0.1/8 ::1 + [recidive] enabled = true @@ -19,6 +22,3 @@ enabled = true filter = nginx-forbidden action = iptables-multiport[name="wo-wordpress", port="http,https"] logpath = /var/log/nginx/*error*.log -findtime = 60 -bantime = 6000 -maxretry = 5 \ No newline at end of file diff --git a/wo/cli/templates/locations.mustache b/wo/cli/templates/locations.mustache index 4e75fdc..ff9da19 100644 --- a/wo/cli/templates/locations.mustache +++ b/wo/cli/templates/locations.mustache @@ -2,20 +2,26 @@ # DO NOT MODIFY, ALL CHANGES WILL BE LOST AFTER AN WordOps (wo) UPDATE # Basic locations files location = /favicon.ico { - access_log off; - log_not_found off; - expires max; + try_files /wp-content/uploads/fbrfg/favicon.ico $uri $uri/ /index.php?$args @empty_gif; + access_log off; + log_not_found off; + expires max; +} +location @empty_gif { + empty_gif; } # Cache static files location ~* \.(ogg|ogv|svg|svgz|eot|otf|woff|woff2|ttf|m4a|mp4|ttf|rss|atom|jpe?g|gif|cur|heic|png|tiff|ico|webm|mp3|aac|tgz|doc|xls|exe|ppt|tar|mid|midi|wav|bmp|rtf|swf|webp|json|webmanifest)$ { - add_header "Access-Control-Allow-Origin" "*"; + more_set_headers 'Access-Control-Allow-Origin : "*"'; + more_set_headers "Cache-Control : public, no-transform"; access_log off; log_not_found off; expires max; } # Cache css & js files location ~* \.(?:css(\.map)?|js(\.map)?)$ { - add_header "Access-Control-Allow-Origin" "*"; + more_set_headers 'Access-Control-Allow-Origin : "*"'; + more_set_headers "Cache-Control : public, no-transform"; access_log off; log_not_found off; expires 30d; diff --git a/wo/cli/templates/mime.mustache b/wo/cli/templates/mime.mustache new file mode 100644 index 0000000..0eac2b4 --- /dev/null +++ b/wo/cli/templates/mime.mustache @@ -0,0 +1,98 @@ + +types { + text/html html htm shtml; + text/css css; + text/xml xml; + image/gif gif; + image/jpeg jpeg jpg; + application/javascript js; + application/atom+xml atom; + application/rss+xml rss; + + text/mathml mml; + text/plain txt; + text/vnd.sun.j2me.app-descriptor jad; + text/vnd.wap.wml wml; + text/x-component htc; + + image/png png; + image/svg+xml svg svgz; + image/tiff tif tiff; + image/vnd.wap.wbmp wbmp; + image/webp webp; + image/x-icon ico; + image/x-jng jng; + image/x-ms-bmp bmp; + + font/woff woff; + font/woff2 woff2; + font/ttf ttf; + + application/java-archive jar war ear; + application/json json; + application/mac-binhex40 hqx; + application/msword doc; + application/pdf pdf; + application/postscript ps eps ai; + application/rtf rtf; + application/vnd.apple.mpegurl m3u8; + application/vnd.google-earth.kml+xml kml; + application/vnd.google-earth.kmz kmz; + application/vnd.ms-excel xls; + application/vnd.ms-fontobject eot; + application/vnd.ms-powerpoint ppt; + application/vnd.oasis.opendocument.graphics odg; + application/vnd.oasis.opendocument.presentation odp; + application/vnd.oasis.opendocument.spreadsheet ods; + application/vnd.oasis.opendocument.text odt; + application/vnd.openxmlformats-officedocument.presentationml.presentation + pptx; + application/vnd.openxmlformats-officedocument.spreadsheetml.sheet + xlsx; + application/vnd.openxmlformats-officedocument.wordprocessingml.document + docx; + application/vnd.wap.wmlc wmlc; + application/x-7z-compressed 7z; + application/x-cocoa cco; + application/x-java-archive-diff jardiff; + application/x-java-jnlp-file jnlp; + application/x-makeself run; + application/x-perl pl pm; + application/x-pilot prc pdb; + application/x-rar-compressed rar; + application/x-redhat-package-manager rpm; + application/x-sea sea; + application/x-shockwave-flash swf; + application/x-stuffit sit; + application/x-tcl tcl tk; + application/x-x509-ca-cert der pem crt; + application/x-xpinstall xpi; + application/xhtml+xml xhtml; + application/xspf+xml xspf; + application/zip zip; + + application/octet-stream bin exe dll; + application/octet-stream deb; + application/octet-stream dmg; + application/octet-stream iso img; + application/octet-stream msi msp msm; + + audio/midi mid midi kar; + audio/mpeg mp3; + audio/ogg ogg; + audio/x-m4a m4a; + audio/x-realaudio ra; + + video/3gpp 3gpp 3gp; + video/mp2t ts; + video/mp4 mp4; + video/mpeg mpeg mpg; + video/quicktime mov; + video/webm webm; + video/x-flv flv; + video/x-m4v m4v; + video/x-mng mng; + video/x-ms-asf asx asf; + video/x-ms-wmv wmv; + video/x-msvideo avi; +} diff --git a/wo/cli/templates/nginx-core.mustache b/wo/cli/templates/nginx-core.mustache index d79b947..9d205c5 100644 --- a/wo/cli/templates/nginx-core.mustache +++ b/wo/cli/templates/nginx-core.mustache @@ -32,6 +32,7 @@ http { # Limit Request limit_req_status 403; limit_req_zone $binary_remote_addr zone=one:10m rate=1r/s; + limit_req_zone $binary_remote_addr zone=two:10m rate=10r/s; # Proxy Settings # set_real_ip_from proxy-server-ip; diff --git a/wo/cli/templates/wpcommon.mustache b/wo/cli/templates/wpcommon.mustache index 89d3e1c..ccc10d7 100644 --- a/wo/cli/templates/wpcommon.mustache +++ b/wo/cli/templates/wpcommon.mustache @@ -6,6 +6,18 @@ location = /wp-login.php { include fastcgi_params; fastcgi_pass {{upstream}}; } +# Prevent DoS attacks on wp-cron +location = /wp-cron.php { + limit_req zone=two burst=1 nodelay; + include fastcgi_params; + fastcgi_pass {{upstream}}; +} +# Prevent Dos attacks with xmlrpc.php +location = /xmlrpc.php { + limit_req zone=two burst=1 nodelay; + include fastcgi_params; + fastcgi_pass {{upstream}}; +} # Disable wp-config.txt location = /wp-config.txt { deny all; diff --git a/wo/core/checkfqdn.py b/wo/core/checkfqdn.py index c9d4ff1..933c020 100644 --- a/wo/core/checkfqdn.py +++ b/wo/core/checkfqdn.py @@ -1,5 +1,6 @@ from wo.core.shellexec import WOShellExec from wo.core.variables import WOVariables +import requests def check_fqdn(self, wo_host): @@ -20,3 +21,18 @@ def check_fqdn(self, wo_host): else: wo_host = input("Enter hostname [fqdn]:") check_fqdn(self, wo_host) + + +def check_fqdn_ip(self): + """Check if server hostname resolved server IP""" + x = requests.get('http://v4.wordops.eu') + ip = (x.text).strip() + + wo_fqdn = WOVariables.wo_fqdn + y = requests.get('http://v4.wordops.eu/dns/{0}/'.format(wo_fqdn)) + ip_fqdn = (y.text).strip() + + if ip == ip_fqdn: + return True + else: + return False diff --git a/wo/core/logwatch.py b/wo/core/logwatch.py index 36baf16..6440bdd 100644 --- a/wo/core/logwatch.py +++ b/wo/core/logwatch.py @@ -49,9 +49,9 @@ class LogWatcher(object): # assert (os.path.isdir(self.folder), "%s does not exists" # % self.folder) for file in self.filelist: - assert (os.path.isfile(file)) - assert callable(callback) - self.update_files() + if not os.path.isfile(file): + if not callable(callback): + self.update_files() # The first time we run the script we move all file markers at EOF. # In case of files created afterwards we don't do this. for id, file in list(iter(self.files_map.items())): diff --git a/wo/core/shellexec.py b/wo/core/shellexec.py index 77ba8a5..adf9241 100644 --- a/wo/core/shellexec.py +++ b/wo/core/shellexec.py @@ -48,15 +48,15 @@ class WOShellExec(): try: subprocess.call(['sensible-editor', filepath]) except OSError as e: - Log.debug(self, "{0}{1}".format(e.errno, e.strerror)) - raise CommandExecutionError + Log.debug(self, "{0}{1}".format(e.errno, e.strerror)) + raise CommandExecutionError def cmd_exec_stdout(self, command, errormsg='', log=True): """Run shell command from Python""" try: - log and Log.debug(self, "Running command: {0}".format(command)) - - with subprocess.Popen([command], stdout=subprocess.PIPE, + log and Log.debug(self, "Running command: command -v {0}".format(command)) + check_command = 'command -v' + command + with subprocess.Popen([check_command], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) as proc: (cmd_stdout_bytes, cmd_stderr_bytes) = proc.communicate() (cmd_stdout, cmd_stderr) = (cmd_stdout_bytes.decode('utf-8', @@ -73,8 +73,8 @@ class WOShellExec(): .format(cmd_stdout, cmd_stderr)) return cmd_stdout except OSError as e: - Log.debug(self, str(e)) - raise CommandExecutionError + Log.debug(self, str(e)) + raise CommandExecutionError except Exception as e: - Log.debug(self, str(e)) - raise CommandExecutionError + Log.debug(self, str(e)) + raise CommandExecutionError